modelflowib 2.73__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
modelestimator_new.py ADDED
@@ -0,0 +1,2613 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ modelestimator_new.py
4
+
5
+ High-level helpers for estimating econometric equations and exporting rich
6
+ reports for ModelFlow / MFMod workflows.
7
+
8
+ Overview
9
+ --------
10
+ This module is organized around a small contract that any estimation backend
11
+ must satisfy. The shared plumbing (equation parsing, mfcalc helpers, sample
12
+ handling, HTML report assembly, ModelFlow integration) lives in base classes;
13
+ each backend supplies only what is genuinely backend-specific.
14
+
15
+ Provided estimators
16
+ -------------------
17
+ - :class:`Estimate_ols` — OLS via ``statsmodels``.
18
+ - :class:`Estimate_nls_lmfit` — Nonlinear least squares via ``lmfit``.
19
+ - :class:`Estimate_nls_eviews` — Nonlinear least squares via ``EViews``
20
+ (using ``py2eviews`` to round-trip a temporary workfile).
21
+
22
+ A factory class :class:`Estimate_nls` keeps the older single-class API
23
+ working — ``Estimate_nls(eq, solver='lmfit'|'eviews', ...)`` dispatches
24
+ to the appropriate concrete backend, and supports the same
25
+ ``with_defaults`` factory pattern as the concrete backends.
26
+
27
+ Adding a new estimator
28
+ ----------------------
29
+ To add a new estimator backend (for example, GLS, IV, scipy.optimize, or a
30
+ Bayesian sampler):
31
+
32
+ 1. Subclass :class:`EstimatorBackend`.
33
+ 2. Implement :meth:`EstimatorBackend._fit` — run the estimation and return
34
+ ``(regression_model, coef_estimate_dict)``.
35
+ 3. Implement :meth:`EstimatorBackend._render_summary_html` — return the
36
+ regression summary as an HTML fragment. The title, equation listing and
37
+ actual-vs-fitted plot are added by :class:`LSResult` and must NOT be
38
+ included here.
39
+ 4. Optionally override :meth:`EstimatorBackend._validate_equation` to reject
40
+ equation forms the backend cannot handle.
41
+ 5. Add dataclass fields for any extra user-supplied input the backend needs
42
+ (priors, instruments, initial guesses, ...).
43
+
44
+ Worked example::
45
+
46
+ @dataclass
47
+ class Estimate_gls(EstimatorBackend):
48
+ '''GLS via statsmodels.'''
49
+ sigma: Optional[np.ndarray] = None
50
+ ecm: bool = False # GLS is linear-in-params, like OLS
51
+
52
+ def _fit(self):
53
+ df = self.estimation_df
54
+ y, X = df["LHS"], df.drop(columns=["LHS"])
55
+ result = sm.GLS(y, X, sigma=self.sigma).fit()
56
+ coef_dict = self._map_statsmodels_params(result.params.to_dict())
57
+ return result, coef_dict
58
+
59
+ def _render_summary_html(self) -> str:
60
+ return self.regression_model.summary().as_html()
61
+
62
+ That is the entire backend. Sample handling, A/F plots, normalized equations,
63
+ :class:`EqContainer` integration, and HTML reports all work for free.
64
+
65
+ Dependencies
66
+ ------------
67
+ - ``modelclass.model`` (ModelFlow)
68
+ - ``modelnormalize`` (imported as ``nz``); especially ``normal()`` and
69
+ ``endovar()``
70
+ - pandas DataFrames with the time dimension on the index
71
+ - ``statsmodels``, ``lmfit``; ``py2eviews`` is imported lazily by the EViews
72
+ backend only.
73
+
74
+ Author
75
+ ------
76
+ ibhan (refactor 2026)
77
+ """
78
+
79
+ from __future__ import annotations
80
+
81
+ import ast
82
+ import base64
83
+ import re
84
+ import tempfile
85
+ import webbrowser
86
+ from abc import ABC, abstractmethod
87
+ from dataclasses import dataclass, field
88
+ from functools import cached_property, reduce
89
+ from io import BytesIO
90
+ from pathlib import Path
91
+ from typing import Any, Callable, ClassVar, Dict, List, Optional, Set, Tuple, Union
92
+
93
+ import matplotlib.pyplot as plt
94
+ import numpy as np
95
+ import pandas as pd
96
+ import statsmodels.api as sm
97
+ from IPython.display import HTML, display
98
+ from lmfit import Parameters, minimize
99
+ from matplotlib.gridspec import GridSpec
100
+
101
+ from modelclass import model
102
+ import modelnormalize as nz
103
+
104
+
105
+ # =============================================================================
106
+ # Variable-description shim
107
+ # =============================================================================
108
+
109
+ @dataclass
110
+ class DummyOModel:
111
+ """Minimal carrier for ``var_description`` when a real ModelFlow model
112
+ isn't available yet.
113
+
114
+ Attributes
115
+ ----------
116
+ var_description : dict
117
+ Mapping ``variable_name -> human-readable description``. Defaults to
118
+ ``model.defsub({})`` — a defaultdict-like object that returns ``""``
119
+ for unknown keys.
120
+ """
121
+ var_description: dict = field(default_factory=model.defsub)
122
+
123
+
124
+ def dummy_omodel() -> DummyOModel:
125
+ """Return a fresh, empty :class:`DummyOModel`."""
126
+ return DummyOModel()
127
+
128
+
129
+ # =============================================================================
130
+ # Free helpers: parameter token normalization and coefficient expansion
131
+ # =============================================================================
132
+
133
+ def replace_c_params(equation: str, est_param: str = "C") -> str:
134
+ """Convert EViews-style ``C(n)`` (and ``{est_param}(n)``) tokens to the
135
+ mfcalc-style ``{est_param}__n`` form.
136
+
137
+ Examples
138
+ --------
139
+ >>> replace_c_params("Y = C(1) + C(2)*X", est_param="B")
140
+ 'Y = B__1 + B__2*X'
141
+ >>> replace_c_params("Y = B(1) + B(2)*X", est_param="B")
142
+ 'Y = B__1 + B__2*X'
143
+
144
+ Notes
145
+ -----
146
+ Both ``C(...)`` (the EViews convention) and ``{est_param}(...)`` are
147
+ accepted. Negative indices are preserved. When ``est_param == "C"`` the
148
+ second pass is a no-op.
149
+ """
150
+ equation = re.sub(r'C\((-?\d+)\)', rf'{est_param}__\1', equation)
151
+ if est_param != "C":
152
+ equation = re.sub(rf'{re.escape(est_param)}\((-?\d+)\)',
153
+ rf'{est_param}__\1', equation)
154
+ return equation
155
+
156
+
157
+ def restore_c_params(equation: str, est_param: str = "C") -> str:
158
+ """Convert mfcalc-style ``{est_param}__n`` tokens back to EViews ``C(n)``.
159
+
160
+ Used by the EViews backend regardless of the active ``est_param``, because
161
+ EViews always names its coefficient vector ``C``.
162
+ """
163
+ return re.sub(rf'{re.escape(est_param)}__(-?\d+)', r'C(\1)', equation)
164
+
165
+
166
+ def expand_equation_with_coefficients(
167
+ equation_text: str,
168
+ coefficients: Optional[Dict[str, float]] = None,
169
+ decimals: int = 6,
170
+ ) -> str:
171
+ """Replace coefficient placeholders in an equation with numeric values.
172
+
173
+ Parameters
174
+ ----------
175
+ equation_text : str
176
+ Equation containing placeholders such as ``C__1``, ``C__2``, ...
177
+ coefficients : dict[str, float], optional
178
+ Mapping ``placeholder -> value``. ``None`` (default) returns
179
+ ``equation_text`` unchanged.
180
+ decimals : int, default 6
181
+ Number of decimals to keep when formatting each value.
182
+
183
+ Returns
184
+ -------
185
+ str
186
+ Equation with placeholders substituted by ``(<value>)``.
187
+ """
188
+ if not coefficients:
189
+ return equation_text
190
+ # Use word-boundary-anchored regex so a key like "C__1" doesn't match
191
+ # inside variable names like "C__12" or "C__1_LAGGED". Sort by length
192
+ # descending as a defensive belt-and-braces measure (word boundaries
193
+ # already prevent partial matches, but if a future caller passes
194
+ # placeholder keys whose ends sit at word/non-word transitions in
195
+ # surprising ways, longest-first still avoids overlap surprises).
196
+ for key in sorted(coefficients.keys(), key=lambda x: -len(x)):
197
+ value = coefficients[key]
198
+ formatted = f"({value:.{decimals}f})"
199
+ equation_text = re.sub(
200
+ rf"\b{re.escape(key)}\b", formatted, equation_text
201
+ )
202
+ return equation_text
203
+
204
+
205
+ # =============================================================================
206
+ # Constraint parsing — ST. clause and per-parameter dicts
207
+ # =============================================================================
208
+
209
+ _ST_CLAUSE_RE = re.compile(r'\s+ST\.\s+', flags=re.IGNORECASE)
210
+
211
+
212
+ def _split_st_clause(equation: str) -> Tuple[str, str]:
213
+ """Split ``LHS = RHS ST. constraints`` into ``(equation, constraint_text)``.
214
+
215
+ Returns ``(equation, '')`` if no ``ST.`` clause is present. Matching is
216
+ case-insensitive and requires whitespace on both sides so ``ST.`` does
217
+ not collide with variable names.
218
+ """
219
+ parts = _ST_CLAUSE_RE.split(equation, maxsplit=1)
220
+ if len(parts) == 1:
221
+ return equation, ''
222
+ return parts[0], parts[1]
223
+
224
+
225
+ def _normalize_param_tokens(
226
+ text: str,
227
+ est_param: str,
228
+ param_names: Optional[List[str]],
229
+ ) -> str:
230
+ """Apply ``replace_c_params`` and ``param_names`` substitution.
231
+
232
+ Shared between the equation body and the ``ST.`` clause so both end up
233
+ with the same ``{prefix}__n`` form.
234
+ """
235
+ if not text:
236
+ return text
237
+ out = replace_c_params(text, est_param=est_param)
238
+ for _pname in (param_names or []):
239
+ _up = _pname.upper()
240
+ out = re.sub(rf'\b{re.escape(_up)}\((-?\d+)\)', rf'{_up}__\1', out)
241
+ out = re.sub(rf'\b{re.escape(_up)}\b(?!\()', rf'{_up}__1', out)
242
+ return out
243
+
244
+
245
+ def _parse_constraints(text: str) -> Dict[str, Dict[str, Any]]:
246
+ """Parse a semicolon-separated constraint string into a per-parameter dict.
247
+
248
+ Names are assumed already normalized to ``{prefix}__n`` form (callers
249
+ should run :func:`_normalize_param_tokens` first).
250
+
251
+ Recognized forms::
252
+
253
+ NAME=[lo, hi] bounds -> {'min': lo, 'max': hi}
254
+ NAME > x | NAME >= x lower bound -> {'min': x}
255
+ NAME < x | NAME <= x upper bound -> {'max': x}
256
+ NAME := <expr> algebraic -> {'expr': '<expr>'}
257
+ NAME = <value> fixed value -> {'value': v, 'vary': False}
258
+ NAME ~ <value> initial value -> {'value': v, 'vary': True}
259
+
260
+ Raises ``ValueError`` for any line that does not match one of the above.
261
+ """
262
+ out: Dict[str, Dict[str, Any]] = {}
263
+ if not text or not text.strip():
264
+ return out
265
+
266
+ num = r'-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?'
267
+
268
+ for raw in text.split(';'):
269
+ line = raw.strip().rstrip('$').strip()
270
+ if not line:
271
+ continue
272
+
273
+ # `:=` must be checked before plain `=`
274
+ m = re.match(r'^(\w+)\s*:=\s*(.+)$', line)
275
+ if m:
276
+ name = m.group(1).upper()
277
+ out.setdefault(name, {})['expr'] = m.group(2).strip()
278
+ continue
279
+
280
+ # Bounds: NAME = [lo, hi]
281
+ m = re.match(rf'^(\w+)\s*=\s*\[\s*({num})\s*,\s*({num})\s*\]$', line)
282
+ if m:
283
+ name = m.group(1).upper()
284
+ out.setdefault(name, {}).update(
285
+ {'min': float(m.group(2)), 'max': float(m.group(3))}
286
+ )
287
+ continue
288
+
289
+ # Inequalities: >=, <=, >, < (order matters: longest first)
290
+ m = re.match(rf'^(\w+)\s*(>=|<=|>|<)\s*({num})$', line)
291
+ if m:
292
+ name = m.group(1).upper()
293
+ key = 'min' if m.group(2) in ('>', '>=') else 'max'
294
+ out.setdefault(name, {})[key] = float(m.group(3))
295
+ continue
296
+
297
+ # Initial value (free to change): NAME ~ value
298
+ # Explicit vary=True so a later ``~`` can override an earlier ``=``.
299
+ m = re.match(rf'^(\w+)\s*~\s*({num})$', line)
300
+ if m:
301
+ name = m.group(1).upper()
302
+ out.setdefault(name, {}).update(
303
+ {'value': float(m.group(2)), 'vary': True}
304
+ )
305
+ continue
306
+
307
+ # Fixed value: NAME = value
308
+ m = re.match(rf'^(\w+)\s*=\s*({num})$', line)
309
+ if m:
310
+ name = m.group(1).upper()
311
+ out.setdefault(name, {}).update(
312
+ {'value': float(m.group(2)), 'vary': False}
313
+ )
314
+ continue
315
+
316
+ raise ValueError(
317
+ f"Cannot parse constraint {line!r}. Expected one of: "
318
+ "NAME=[lo,hi], NAME>x, NAME>=x, NAME<x, NAME<=x, "
319
+ "NAME=value, NAME~value, NAME:=expr."
320
+ )
321
+
322
+ return out
323
+
324
+
325
+ def _merge_constraint_dicts(
326
+ primary: Dict[str, Dict[str, Any]],
327
+ secondary: Dict[str, Dict[str, Any]],
328
+ ) -> Dict[str, Dict[str, Any]]:
329
+ """Per-key merge: ``primary`` wins on key collisions inside each param."""
330
+ out: Dict[str, Dict[str, Any]] = {}
331
+ for name in set(secondary) | set(primary):
332
+ out[name] = {**secondary.get(name, {}), **primary.get(name, {})}
333
+ return out
334
+
335
+
336
+ def process_string_eq(eqs: str) -> List["Eq"]:
337
+ """Split a multi-line block of identity equations into a list of :class:`Eq`."""
338
+ out: List["Eq"] = []
339
+ for line in eqs.split("\n"):
340
+ stripped = line.strip()
341
+ if stripped:
342
+ out.append(Eq(stripped))
343
+ return out
344
+
345
+
346
+ def _trim_invalid_rows(
347
+ df: pd.DataFrame,
348
+ requested_smpl: Tuple[Any, Any],
349
+ label: str = "estimation",
350
+ ) -> Tuple[pd.DataFrame, Tuple[Any, Any]]:
351
+ """Drop rows containing any non-finite value (NaN or ±inf) and report.
352
+
353
+ Used by :class:`Estimate_ols` to clean up the regressor frame produced
354
+ by mfcalc before handing it to ``sm.OLS``. Without this, lags and
355
+ transformations introduce non-finite rows that statsmodels would
356
+ silently propagate into a meaningless coefficient vector — NaNs from
357
+ boundary lags or data gaps, infinities from things like ``LOG(0)``
358
+ or division by zero.
359
+
360
+ Parameters
361
+ ----------
362
+ df : pandas.DataFrame
363
+ Frame indexed by time, with one column per regressor and one for
364
+ the LHS.
365
+ requested_smpl : tuple
366
+ The user-requested ``(start, end)`` sample; used purely for messages.
367
+ label : str
368
+ Short identifier used in printed messages (e.g. the equation name).
369
+
370
+ Returns
371
+ -------
372
+ cleaned : pandas.DataFrame
373
+ ``df`` with non-finite rows removed.
374
+ effective_smpl : tuple
375
+ ``(first_index, last_index)`` of the cleaned frame, or the requested
376
+ sample if cleaning produced an empty frame (the caller will raise).
377
+
378
+ Behavior
379
+ --------
380
+ - **Boundary** non-finite rows (at the very start and/or end of the
381
+ sample) are dropped with a brief one-line diagnostic showing the
382
+ requested vs. effective sample. These are expected from lags /
383
+ differences, but the user still benefits from seeing the effective
384
+ window.
385
+ - **Interior** non-finite rows (in the middle of the sample, surrounded
386
+ by valid data) are dropped *with a prominent warning* listing the
387
+ affected rows — these usually indicate data gaps or numerical issues
388
+ (``LOG(0)``, division by zero) the user should investigate.
389
+ """
390
+ if df.empty:
391
+ return df, requested_smpl
392
+
393
+ # np.isfinite is False for NaN, +inf, and -inf — exactly the rows we
394
+ # don't want statsmodels to see.
395
+ is_valid = pd.Series(
396
+ np.isfinite(df.to_numpy()).all(axis=1), index=df.index
397
+ )
398
+ if is_valid.all():
399
+ first, last = df.index[0], df.index[-1]
400
+ return df, (first, last)
401
+
402
+ # Identify boundary vs. interior invalid rows.
403
+ n = len(is_valid)
404
+ leading = 0
405
+ while leading < n and not is_valid.iloc[leading]:
406
+ leading += 1
407
+ trailing = 0
408
+ while trailing < n - leading and not is_valid.iloc[n - 1 - trailing]:
409
+ trailing += 1
410
+ interior_mask = ~is_valid.iloc[leading:n - trailing]
411
+ interior_count = int(interior_mask.sum())
412
+
413
+ cleaned = df.loc[is_valid]
414
+ if cleaned.empty:
415
+ return cleaned, requested_smpl
416
+
417
+ eff_first, eff_last = cleaned.index[0], cleaned.index[-1]
418
+ req_first, req_last = requested_smpl
419
+
420
+ if leading or trailing:
421
+ # Boundary trimming is expected (lags, differences); just print a
422
+ # short diagnostic so the user knows the effective sample.
423
+ print(
424
+ f"[{label}] sample {req_first}..{req_last} -> "
425
+ f"{eff_first}..{eff_last} after dropping "
426
+ f"{leading} leading and {trailing} trailing non-finite row(s)."
427
+ )
428
+
429
+ if interior_count:
430
+ # Interior gaps are unusual and worth surfacing prominently.
431
+ gap_rows = list(interior_mask[interior_mask].index)
432
+ print(
433
+ f"[{label}] WARNING: {interior_count} interior row(s) with "
434
+ f"NaN or inf dropped from the estimation sample: {gap_rows}. "
435
+ "This may indicate data gaps, LOG of non-positive values, or "
436
+ "division by zero you didn't intend to ignore."
437
+ )
438
+
439
+ return cleaned, (eff_first, eff_last)
440
+
441
+
442
+ # =============================================================================
443
+ # Equation parsing -> mfcalc lines + validation
444
+ # =============================================================================
445
+
446
+ @dataclass
447
+ class EquationParse:
448
+ """Parse and validate an equation into mfcalc-friendly sub-equations.
449
+
450
+ Walks the right-hand-side AST to extract parameterized terms of the form
451
+ ``{est_param}__n * <expr>``, validates that parameter tokens are not
452
+ nested incorrectly, and emits a list of mfcalc code lines.
453
+
454
+ Emitted mfcalc lines (in order)
455
+ -------------------------------
456
+ - ``LHS = <original LHS>``
457
+ - ``{est_param}__1 = 1.0`` (if ``const=True`` and ``__1`` appears)
458
+ - ``EC_TERM = <expr>`` (if ``ecm=True`` and ``__2`` appears)
459
+ - one ``c__<n> = <expr>`` line per parameterized regressor (skipping the
460
+ ECM term, which is handled separately)
461
+ - ``LHS_ORG_EQ = <full RHS>``
462
+
463
+ Parameters
464
+ ----------
465
+ org_eq_clean : str
466
+ Canonicalized equation containing ``{est_param}__n`` placeholders.
467
+ ecm : bool, default True
468
+ If True, require the LHS to be of the form ``DLOG(VAR)``.
469
+ const : bool, default True
470
+ If True and ``{est_param}__1`` appears in the equation, emit a line
471
+ ``{est_param}__1 = 1.0``.
472
+ est_param : str, default "C"
473
+ Parameter prefix.
474
+ function_vars : list[str]
475
+ Function names treated as operators (excluded from variable
476
+ discovery). Defaults to ``["LOG", "DLOG"]``.
477
+
478
+ Raises
479
+ ------
480
+ ValueError
481
+ If ECM is required but the LHS doesn't match, or if a parameter token
482
+ appears in a disallowed position (nested expression, or preceded by a
483
+ unary minus).
484
+ """
485
+ org_eq_clean: str
486
+ ecm: bool = True
487
+ const: bool = True
488
+ est_param: str = "C"
489
+ param_names: Optional[List[str]] = None
490
+ function_vars: List[str] = field(default_factory=lambda: ["LOG", "DLOG"])
491
+
492
+ mfcalc_code: List[str] = field(init=False)
493
+ rhs_ast: ast.AST = field(init=False)
494
+ lhs_ast: ast.AST = field(init=False)
495
+ endo_var: str = ""
496
+
497
+ def __post_init__(self) -> None:
498
+ _prefixes = [self.est_param] + [n.upper() for n in (self.param_names or [])]
499
+ self.param_regex = (
500
+ r"(?:" + "|".join(re.escape(p) for p in _prefixes) + r")__(\d+)"
501
+ )
502
+ (self.mfcalc_code,
503
+ self.lhs_raw,
504
+ self.lhs_ast,
505
+ self.rhs_raw,
506
+ self.rhs_ast,
507
+ self.endo_var) = self._parse_equation(self.org_eq_clean)
508
+ self.term_dict = {
509
+ k.strip(): v.strip()
510
+ for k, v in (line.split("=", 1) for line in self.mfcalc_code)
511
+ }
512
+ self.used_vars = self._extract_variable_names_from_ast(
513
+ self.rhs_ast, self.lhs_ast
514
+ )
515
+
516
+ # -- equation splitting + AST building ------------------------------------
517
+
518
+ def _parse_equation(self, eq: str):
519
+ lhs_raw, rhs_raw = eq.strip().split("=", 1)
520
+ lhs_var, endo_var = self._sanitize_lhs(lhs_raw.strip())
521
+
522
+ tree = ast.parse(f"{lhs_var} = {rhs_raw}", mode="exec")
523
+ rhs_ast = ast.parse(rhs_raw.strip(), mode="eval")
524
+ lhs_ast = ast.parse(lhs_raw.strip(), mode="eval")
525
+ lines = self._extract_subterms_from_ast(tree, lhs_raw)
526
+ return lines, lhs_raw, lhs_ast, rhs_raw, rhs_ast, endo_var
527
+
528
+ def _sanitize_lhs(self, lhs: str) -> Tuple[str, str]:
529
+ if self.ecm:
530
+ match = re.match(r"\s*DLOG\((\w+)\)", lhs)
531
+ if not match:
532
+ raise ValueError(
533
+ "LHS must be of the form DLOG(VAR) when ecm=True. "
534
+ "Pass ecm=False if this isn't an error-correction equation."
535
+ )
536
+ return "lhs", nz.endovar(lhs)
537
+
538
+ # -- main AST walk + validation -------------------------------------------
539
+
540
+ def _extract_subterms_from_ast(self, tree, lhs_raw) -> List[str]:
541
+ """Walk the RHS, validate parameter positions, and emit mfcalc lines.
542
+
543
+ Negative signs in front of parameter terms are accepted and folded
544
+ into the regressor expression. ``- C__n * <expr>`` becomes a
545
+ regressor column equal to ``-(<expr>)``, fitted against ``C__n``.
546
+ The estimated value of ``C__n`` then matches the user's mental
547
+ model — the same number EViews would produce — and substituting
548
+ it back into the original equation reproduces what the user wrote.
549
+
550
+ For the conventional intercept slot, ``C__1`` becomes a column of
551
+ ``1.0``; ``-C__1`` becomes a column of ``-1.0``. Either way the
552
+ coefficient slot is ``C__1``.
553
+ """
554
+ rhs_expr = tree.body[0].value
555
+ subexprs = [f"LHS = {lhs_raw}"]
556
+ model_expr = f"LHS_ORG_EQ = {self._ast_to_source(rhs_expr)}"
557
+
558
+ # Annotate parents so we can do simple ancestry checks.
559
+ def annotate_parents(node, parent=None):
560
+ for child in ast.iter_child_nodes(node):
561
+ child._parent = node
562
+ annotate_parents(child, node)
563
+ annotate_parents(rhs_expr)
564
+
565
+ def is_at_top_level(target: ast.AST) -> bool:
566
+ """True iff ``target`` sits in a chain of allowed link nodes
567
+ (+/- BinOps and UnaryOp(USub)) that reaches the very top of
568
+ the RHS.
569
+
570
+ If ``target is rhs_expr``, trivially True. Otherwise we walk
571
+ up through allowed link nodes until we either:
572
+ - hit a non-allowed node → not at top level, OR
573
+ - reach ``rhs_expr`` itself → top level only if ``rhs_expr``
574
+ is also an allowed link node (so that the whole RHS is
575
+ an additive/unary expression that ``target`` is one term
576
+ of).
577
+
578
+ For ``Y = C__1 + C__2 * X`` the param is at top level via the
579
+ +/- chain: rhs_expr is an Add, target's parent is the Add.
580
+ For ``Y = -C__2 * X`` the param's parent is a UnaryOp, but
581
+ then the UnaryOp's parent is a Mult (which IS rhs_expr) — so
582
+ the param is NOT at top level: the Mult breaks the chain.
583
+ """
584
+ if target is rhs_expr:
585
+ return True
586
+
587
+ def is_allowed_link(node: ast.AST) -> bool:
588
+ if isinstance(node, ast.BinOp) and isinstance(
589
+ node.op, (ast.Add, ast.Sub)
590
+ ):
591
+ return True
592
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
593
+ return True
594
+ return False
595
+
596
+ cur = target
597
+ while True:
598
+ parent = getattr(cur, "_parent", None)
599
+ if parent is None:
600
+ return cur is rhs_expr
601
+ if parent is rhs_expr:
602
+ # Reaching rhs_expr counts only if rhs_expr is itself
603
+ # an allowed link node (we're one term in it).
604
+ return is_allowed_link(rhs_expr)
605
+ if not is_allowed_link(parent):
606
+ return False
607
+ cur = parent
608
+
609
+ def effective_sign(target: ast.AST) -> int:
610
+ """Return +1 or -1: the effective sign of ``target`` in the
611
+ additive expansion of the RHS.
612
+
613
+ Each time we are the right child of a binary ``Sub`` or the
614
+ operand of a ``UnaryOp(USub)``, the sign flips. The walk
615
+ stops at ``rhs_expr`` — but if ``rhs_expr`` itself is a Sub
616
+ (and we're its right child) or a UnaryOp(USub), the flip
617
+ still applies to that final step.
618
+ """
619
+ sign = 1
620
+ cur = target
621
+ while True:
622
+ parent = getattr(cur, "_parent", None)
623
+ if parent is None or parent is rhs_expr:
624
+ # Apply the final-step flip if rhs_expr itself is a
625
+ # node whose position relative to cur changes the sign.
626
+ if (isinstance(rhs_expr, ast.BinOp)
627
+ and isinstance(rhs_expr.op, ast.Sub)
628
+ and rhs_expr.right is cur):
629
+ sign = -sign
630
+ elif (isinstance(rhs_expr, ast.UnaryOp)
631
+ and isinstance(rhs_expr.op, ast.USub)
632
+ and rhs_expr.operand is cur):
633
+ sign = -sign
634
+ return sign
635
+ if isinstance(parent, ast.BinOp) and isinstance(parent.op, ast.Sub):
636
+ if parent.right is cur:
637
+ sign = -sign
638
+ elif isinstance(parent, ast.UnaryOp) and isinstance(
639
+ parent.op, ast.USub
640
+ ):
641
+ sign = -sign
642
+ cur = parent
643
+
644
+ # --- Validation: parameter tokens may only appear at the top level
645
+ # of the RHS, where "top level" includes +/- chains AND unary
646
+ # minus prefixes. Allowed positions for a parameter Name:
647
+ #
648
+ # (a) the parameter itself is at the top level
649
+ # Y = C__1
650
+ # Y = C__1 + C__3 * X
651
+ # Y = -C__1 (negative intercept)
652
+ # (b) the parameter is the left operand of a Mult that is
653
+ # itself at the top level
654
+ # Y = C__2 * X
655
+ # Y = -C__2 * X (negative slope)
656
+ # Y = C__1 - C__2 * X (binary minus before Mult)
657
+ # Y = C__1 + C__2 * (X / Z)
658
+ #
659
+ # Anything else — parameter inside a function call, inside a
660
+ # division, inside a power, inside a parenthesized sub-expression
661
+ # that is multiplied or otherwise combined — is rejected.
662
+ #
663
+ # A bare parameter at the top level is only accepted for the
664
+ # conventional intercept slot ({prefix}__1), regardless of sign.
665
+ # A standalone non-intercept parameter (e.g. "Y = C__1 + C__2")
666
+ # would be silently dropped by the regressor-collection step,
667
+ # so we reject rather than fit a smaller model than the user wrote.
668
+ intercept_slot = f"{self.est_param}__1"
669
+
670
+ def parameterized_term(node: ast.AST) -> Optional[Tuple[ast.Name, ast.AST, int]]:
671
+ """If ``node`` is a parameterized term — ``Name(C__n) * expr`` or
672
+ ``-Name(C__n) * expr`` — return ``(name_node, rhs_expr, sign)``.
673
+ Otherwise return None.
674
+
675
+ The ``sign`` here is just the local sign from a UnaryOp wrapper
676
+ on the left operand (–1 if present, +1 otherwise). Effective
677
+ sign in the surrounding additive chain is computed separately.
678
+ """
679
+ if not (isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mult)):
680
+ return None
681
+ left = node.left
682
+ local_sign = 1
683
+ if isinstance(left, ast.UnaryOp) and isinstance(left.op, ast.USub):
684
+ left = left.operand
685
+ local_sign = -1
686
+ if isinstance(left, ast.Name) and re.match(self.param_regex, left.id):
687
+ return left, node.right, local_sign
688
+ return None
689
+
690
+ for node in ast.walk(rhs_expr):
691
+ if isinstance(node, ast.Name) and re.match(self.param_regex, node.id):
692
+ parent = getattr(node, "_parent", None)
693
+
694
+ # Case (a): the Name itself sits at the top level.
695
+ if is_at_top_level(node):
696
+ if node.id == intercept_slot:
697
+ continue
698
+ raise ValueError(
699
+ f"Invalid usage: coefficient {node.id} appears as "
700
+ "a standalone term at the top level. Only "
701
+ f"{self.est_param}(1) may appear bare (it is the "
702
+ "intercept slot). Other coefficients must appear "
703
+ f"as '{self.est_param}(n) * <expr>' — if you meant "
704
+ f"this as an extra constant, write it as "
705
+ f"{self.est_param}(1) instead, or attach a "
706
+ "regressor."
707
+ )
708
+
709
+ # Case (b): the Name is the left operand of a Mult, possibly
710
+ # wrapped in a unary minus, and that Mult itself sits at
711
+ # the top level.
712
+ # Direct: parent = Mult, Mult.left is Name
713
+ # Negated: parent = UnaryOp(USub), grandparent = Mult,
714
+ # Mult.left is UnaryOp
715
+ mult_node: Optional[ast.AST] = None
716
+ if (isinstance(parent, ast.BinOp)
717
+ and isinstance(parent.op, ast.Mult)
718
+ and parent.left is node):
719
+ mult_node = parent
720
+ elif (isinstance(parent, ast.UnaryOp)
721
+ and isinstance(parent.op, ast.USub)):
722
+ grand = getattr(parent, "_parent", None)
723
+ if (isinstance(grand, ast.BinOp)
724
+ and isinstance(grand.op, ast.Mult)
725
+ and grand.left is parent):
726
+ mult_node = grand
727
+ if mult_node is not None and is_at_top_level(mult_node):
728
+ continue
729
+
730
+ raise ValueError(
731
+ f"Invalid nesting: coefficient {node.id} must appear at "
732
+ "the top level of the equation — alone, or as the left "
733
+ "side of a multiplication, possibly within a +/- chain. "
734
+ "It cannot appear inside a function call, a division, "
735
+ "an exponentiation, or a parenthesized sub-expression "
736
+ "that is then combined with another operator."
737
+ )
738
+
739
+ # --- Collect parameterized regressor terms with effective signs.
740
+ # For each parameterized Mult term found, record the index, the
741
+ # right-hand expression, and the effective sign of the whole term.
742
+ # The effective sign combines (a) any local unary minus on the
743
+ # parameter (e.g. ``-C__n * X``) with (b) the sign accumulated from
744
+ # the surrounding additive chain. The sign gets folded into the
745
+ # emitted regressor so the estimated coefficient matches the user's
746
+ # written equation.
747
+ c_terms: Dict[int, Tuple[ast.AST, int]] = {}
748
+ for node in ast.walk(rhs_expr):
749
+ term = parameterized_term(node)
750
+ if term is None:
751
+ continue
752
+ name_node, rhs_node, local_sign = term
753
+ n = int(re.match(self.param_regex, name_node.id).group(1))
754
+ sign = local_sign * effective_sign(node)
755
+ c_terms[n] = (rhs_node, sign)
756
+
757
+ # --- Optional explicit constant placeholder.
758
+ # Determine the effective sign of the C__1 token if it appears bare
759
+ # at the top level (it can also appear inside a Mult, in which case
760
+ # it's collected above instead).
761
+ if self.const:
762
+ for node in ast.walk(rhs_expr):
763
+ if (isinstance(node, ast.Name)
764
+ and node.id == intercept_slot
765
+ and is_at_top_level(node)):
766
+ c1_sign = effective_sign(node)
767
+ subexprs.append(
768
+ f"{intercept_slot} = {1.0 * c1_sign}"
769
+ )
770
+ break
771
+
772
+ # --- ECM term (parameter #2 is the speed-of-adjustment by convention).
773
+ if self.ecm and 2 in c_terms:
774
+ expr_node, sign = c_terms[2]
775
+ ec_src = self._ast_to_source(expr_node)
776
+ if sign < 0:
777
+ ec_src = f"-({ec_src})"
778
+ subexprs.append(f"EC_TERM = {ec_src}")
779
+
780
+ # --- Remaining parameterized regressors.
781
+ for n in sorted(k for k in c_terms if not (self.ecm and k == 2)):
782
+ expr_node, sign = c_terms[n]
783
+ src = self._ast_to_source(expr_node)
784
+ if sign < 0:
785
+ src = f"-({src})"
786
+ subexprs.append(f"c__{n} = {src}")
787
+
788
+ subexprs.append(model_expr)
789
+ return [line.upper() for line in subexprs]
790
+
791
+ @staticmethod
792
+ def _ast_to_source(node) -> str:
793
+ return ast.unparse(node)
794
+
795
+ def _extract_variable_names_from_ast(
796
+ self, rhs_ast: ast.AST, lhs_ast: ast.AST
797
+ ) -> Set[str]:
798
+ """Collect variable names referenced in either side, excluding the
799
+ function-operator names (LOG, DLOG, ...)."""
800
+ vars_used: Set[str] = set()
801
+ function_vars = set(self.function_vars)
802
+
803
+ class VarCollector(ast.NodeVisitor):
804
+ def visit_Call(self, node):
805
+ if isinstance(node.func, ast.Name):
806
+ if node.func.id not in function_vars:
807
+ vars_used.add(node.func.id)
808
+ for arg in node.args:
809
+ self.visit(arg)
810
+
811
+ def visit_Name(self, node):
812
+ if node.id not in function_vars:
813
+ vars_used.add(node.id)
814
+
815
+ VarCollector().visit(rhs_ast)
816
+ VarCollector().visit(lhs_ast)
817
+ return vars_used
818
+
819
+
820
+ # =============================================================================
821
+ # Eq_parent — equation holder shared by Eq and every estimator backend
822
+ # =============================================================================
823
+
824
+ @dataclass
825
+ class Eq_parent:
826
+ """Common base for equation holders.
827
+
828
+ Normalizes the original equation, builds the mfcalc helper equations
829
+ (``actual``, ``fitted``, ``residuals``), and discovers the variables
830
+ referenced. This class does NOT estimate; subclasses do.
831
+
832
+ Parameters
833
+ ----------
834
+ org_eq : str
835
+ Original equation (EViews-style accepted, e.g. ``Y = C(1) + C(2)*X``).
836
+ smpl : tuple[int, int], default (2002, 2018)
837
+ Estimation sample (inclusive).
838
+ input_df : pandas.DataFrame
839
+ Source data with time on the index.
840
+ est_param : str, default "C"
841
+ Parameter prefix. Setting ``"B"`` converts both ``C(1)`` and ``B(1)``
842
+ to ``B__1``.
843
+ caption : str, default "Estimation of "
844
+ Human-readable caption used in exports.
845
+ var_description : dict, optional
846
+ Mapping ``var -> description``. If non-empty it overrides ``omodel``.
847
+ coef_dict : dict[str, float], optional
848
+ Initial numeric substitutions for placeholders (applied before
849
+ parsing).
850
+ frml_name : str, default ""
851
+ FRML header used by :class:`EqContainer` when emitting normalized code.
852
+ add_add_factor, make_fixable, make_fitted : bool
853
+ Flags forwarded to ``modelnormalize.normal``.
854
+ ecm : bool, default True
855
+ If True, the LHS must be ``DLOG(VAR)`` and ``C__2`` (if present) is
856
+ treated as the speed-of-adjustment (``EC_TERM``). Estimator subclasses
857
+ that don't use ECM forms (like :class:`Estimate_ols`) override this.
858
+
859
+ Attributes
860
+ ----------
861
+ org_eq : str
862
+ Canonicalized (upper-cased, whitespace-normalized) equation.
863
+ org_eq_clean : str
864
+ Equation with any initial numeric substitutions applied.
865
+ lhs_actual_eq, rhs_fit_eq, residual_eq : str
866
+ mfcalc helper equations for ``actual``, ``fitted``, ``residuals``.
867
+ endo_var : str
868
+ The endogenous (LHS) variable name.
869
+ eq_var_df : pandas.DataFrame
870
+ Working dataframe with only the variables referenced by the equation.
871
+ estimation_df : pandas.DataFrame
872
+ Default estimation dataset (equal to ``eq_var_df`` in the base; the
873
+ OLS backend overrides this with the mfcalc-built regressor frame).
874
+ """
875
+ org_eq: str = ""
876
+ smpl: Tuple[int, int] = (2002, 2018)
877
+ input_df: Optional[pd.DataFrame] = None
878
+ est_param: str = "C"
879
+ param_names: Optional[List[str]] = field(default=None,
880
+ metadata={"description": "Named parameter prefixes: 'lambda' allows LAMBDA(1), LAMBDA(2) etc."})
881
+
882
+ caption: str = "Estimation of "
883
+ var_description: dict = field(default_factory=dict)
884
+ omodel: DummyOModel = field(default_factory=dummy_omodel)
885
+ coef_dict: Dict[str, Union[int, float]] = field(default_factory=dict)
886
+ constraints: Dict[str, Dict[str, Any]] = field(
887
+ default_factory=dict,
888
+ metadata={
889
+ "description": (
890
+ "Per-parameter lmfit-style kwargs: "
891
+ "{'C__1': {'min': 0, 'max': 1}, 'C__4': {'expr': 'C__1 + C__2'}}. "
892
+ "Also populated by parsing an inline ``ST.`` clause in org_eq."
893
+ )
894
+ },
895
+ )
896
+ frml_name: str = ""
897
+ add_add_factor: bool = False
898
+ make_fixable: bool = False
899
+ make_fitted: bool = False
900
+ ecm: bool = True
901
+
902
+ def __post_init__(self) -> None:
903
+ # 1) Uppercase and split off any inline ``ST.`` constraint clause.
904
+ raw = self.org_eq.upper()
905
+ eq_body, constraint_text = _split_st_clause(raw)
906
+
907
+ # 2) Normalize parameter tokens in both equation body AND constraints
908
+ # so names align downstream (e.g. ``C(1)`` -> ``C__1``).
909
+ eq = _normalize_param_tokens(eq_body, self.est_param, self.param_names)
910
+ constraint_text = _normalize_param_tokens(
911
+ constraint_text, self.est_param, self.param_names
912
+ )
913
+
914
+ eq = eq.replace("@ABS(", "ABS(")
915
+
916
+ # 3) Merge inline ST. constraints with any explicitly supplied dict.
917
+ # Explicit ``constraints=`` kwargs win on key collisions.
918
+ if constraint_text:
919
+ inline = _parse_constraints(constraint_text)
920
+ self.constraints = _merge_constraint_dicts(self.constraints, inline)
921
+
922
+ if self.var_description:
923
+ self.omodel = DummyOModel(
924
+ var_description=model.defsub(self.var_description)
925
+ )
926
+ self.org_eq = " ".join(eq.strip().split()).upper()
927
+ self.org_eq_clean = expand_equation_with_coefficients(
928
+ self.org_eq, self.coef_dict
929
+ )
930
+
931
+ # 4) Build the mfcalc helper equations and identify the endo var.
932
+ lhs_expression, rhs_expression = self.org_eq_clean.split("=", 1)
933
+ self.lhs_actual_eq = nz.normal(
934
+ f"actual = {lhs_expression}", add_add_factor=False
935
+ ).normalized
936
+ self.rhs_fit_eq = nz.normal(
937
+ f"fitted = {rhs_expression}", add_add_factor=False
938
+ ).normalized
939
+ self.residual_eq = nz.normal(
940
+ f"residuals = ({lhs_expression}) - ({rhs_expression})",
941
+ add_add_factor=False,
942
+ ).normalized
943
+ self.endo_var = nz.endovar(lhs_expression)
944
+
945
+ # 5) Build a working dataframe with only the referenced variables.
946
+ # Subclasses that need a custom dataframe layout (e.g. mfcalc-built
947
+ # regressors for OLS) handle that themselves.
948
+ if self.input_df is not None:
949
+ self.eq_var_df = (
950
+ self.mdummy.insertModelVar(self.input_df)
951
+ .loc[:, self.varname_all]
952
+ .copy()
953
+ )
954
+ self.estimation_df = self.eq_var_df
955
+
956
+ # -- variable discovery (cached, derived from mdummy) ---------------------
957
+
958
+ @property
959
+ def mdummy(self):
960
+ """Small temporary ModelFlow model used to collect every variable
961
+ name referenced by the helper equations."""
962
+ fdummy = "\n".join(
963
+ [self.lhs_actual_eq, self.rhs_fit_eq, self.residual_eq]
964
+ )
965
+ return model(fdummy)
966
+
967
+ @property
968
+ def varname_all(self) -> List[str]:
969
+ """Sorted list of every variable name referenced by this equation."""
970
+ return sorted(self.mdummy.allvar_set)
971
+
972
+ @cached_property
973
+ def c_params(self) -> List[str]:
974
+ """Sorted parameter placeholders for the active prefix and named param prefixes.
975
+
976
+ E.g. ``['ALPHA__1', 'BETA__1', 'C__1', 'C__2', 'LAMBDA__1']`` (sorted by prefix, then numerically).
977
+ """
978
+ prefixes = tuple(
979
+ [f'{self.est_param}__'] +
980
+ [f'{n.upper()}__' for n in (self.param_names or [])]
981
+ )
982
+ return sorted(
983
+ [v for v in self.varname_all if v.startswith(prefixes)],
984
+ key=lambda x: (x.rsplit('__', 1)[0], int(x.rsplit('__', 1)[1])),
985
+ )
986
+
987
+ @cached_property
988
+ def eq__var(self) -> List[str]:
989
+ """Sorted list of *data* variables (everything except parameter
990
+ placeholders and the helper slots)."""
991
+ prefix = f"{self.est_param}__"
992
+ return sorted(
993
+ v for v in self.varname_all
994
+ if not (v.startswith(prefix)
995
+ or v in {"ACTUAL", "FITTED", "RESIDUALS"})
996
+ )
997
+
998
+ # -- container arithmetic sugar -------------------------------------------
999
+
1000
+ def __add__(self, other) -> "EqContainer":
1001
+ if isinstance(other, EqContainer):
1002
+ return EqContainer([self] + other.equations)
1003
+ if isinstance(other, Eq_parent):
1004
+ return EqContainer([self, other])
1005
+ if isinstance(other, str):
1006
+ return EqContainer([self] + process_string_eq(other))
1007
+ raise TypeError(
1008
+ f"Cannot add object of type {type(other).__name__} to Eq_parent."
1009
+ )
1010
+
1011
+ def __radd__(self, other) -> "EqContainer":
1012
+ if isinstance(other, str):
1013
+ return EqContainer(process_string_eq(other) + [self])
1014
+ raise TypeError(
1015
+ f"Cannot add object of type {type(other).__name__} to Eq_parent."
1016
+ )
1017
+
1018
+
1019
+ # =============================================================================
1020
+ # Eq — light identity wrapper (no estimation)
1021
+ # =============================================================================
1022
+
1023
+ @dataclass
1024
+ class Eq(Eq_parent):
1025
+ """Wrapper for identity / auxiliary equations.
1026
+
1027
+ If a multi-line string is passed, each non-empty line becomes a separate
1028
+ :class:`Eq` and the result is wrapped in an :class:`EqContainer`.
1029
+ """
1030
+ frml_name: str = "<IDENT>"
1031
+ caption: str = "Identity"
1032
+ ecm: bool = False # identity equations are not ECM forms
1033
+
1034
+ def __new__(cls, org_eq: str = "", *args, **kwargs):
1035
+ lines = [l.strip() for l in org_eq.strip().splitlines() if l.strip()]
1036
+ if len(lines) > 1:
1037
+ return EqContainer([cls(line, *args, **kwargs) for line in lines])
1038
+ return super().__new__(cls)
1039
+
1040
+ def __post_init__(self) -> None:
1041
+ super().__post_init__()
1042
+ # Identity equations have no estimated coefficients to substitute.
1043
+ self.org_eq_baked = self.org_eq_clean
1044
+
1045
+
1046
+ # =============================================================================
1047
+ # EstimatorBackend — the contract every estimator implements
1048
+ # =============================================================================
1049
+
1050
+ @dataclass
1051
+ class EstimatorBackend(Eq_parent, ABC):
1052
+ """Abstract base class for every estimation backend.
1053
+
1054
+ Subclasses implement two methods (:meth:`_fit` and
1055
+ :meth:`_render_summary_html`) and may override one optional hook
1056
+ (:meth:`_validate_equation`). Everything else — equation parsing, sample
1057
+ handling, A/F and residuals computation, ``org_eq_baked`` substitution,
1058
+ HTML report assembly, ``EqContainer`` integration — is inherited.
1059
+
1060
+ Required overrides
1061
+ ------------------
1062
+ :meth:`_fit`
1063
+ Run the actual estimation. Must return
1064
+ ``(regression_model, coef_estimate_dict)`` where:
1065
+
1066
+ - ``regression_model`` is the backend's native fitted-result object.
1067
+ Stored on ``self.regression_model``. Treated as opaque by the base.
1068
+ - ``coef_estimate_dict`` is ``{<prefix>__n: float}`` covering every
1069
+ entry in ``self.c_params``.
1070
+
1071
+ :meth:`_render_summary_html`
1072
+ Return the regression summary as an HTML fragment. Do NOT include the
1073
+ title, equation listing, or actual-vs-fitted plot — those are added
1074
+ by :class:`LSResult`.
1075
+
1076
+ Optional overrides
1077
+ ------------------
1078
+ :meth:`_validate_equation`
1079
+ Default is a no-op. Override to reject equation forms the backend
1080
+ cannot handle.
1081
+
1082
+ Standard attributes set during ``__post_init__``
1083
+ ------------------------------------------------
1084
+ regression_model : Any
1085
+ Backend-native fitted-result object.
1086
+ coef_estimate_dict : dict[str, float]
1087
+ Estimated coefficients keyed by ``{est_param}__n``.
1088
+ org_eq_baked : str
1089
+ Equation with the estimated values substituted in place.
1090
+ mfresult : LSResult
1091
+ Reporting wrapper.
1092
+ coef_ser : pandas.Series
1093
+ Estimated coefficients as a pandas Series (named after ``caption``).
1094
+ """
1095
+
1096
+ fit_kws: dict = field(default_factory=dict)
1097
+ method: str = ""
1098
+
1099
+ # Class-level capability flags. Subclasses opt in by overriding.
1100
+ _supports_constraints: ClassVar[bool] = False
1101
+
1102
+ # Late-initialized attributes (set during __post_init__).
1103
+ regression_model: Any = field(init=False, default=None)
1104
+ coef_estimate_dict: Dict[str, float] = field(init=False, default_factory=dict)
1105
+ org_eq_baked: str = field(init=False, default="")
1106
+ mfresult: Any = field(init=False, default=None)
1107
+ coef_ser: pd.Series = field(init=False, default=None)
1108
+
1109
+ # ---- the contract ------------------------------------------------------
1110
+
1111
+ @abstractmethod
1112
+ def _fit(self) -> Tuple[Any, Dict[str, float]]:
1113
+ """Run the estimation. See class docstring for the contract."""
1114
+
1115
+ @abstractmethod
1116
+ def _render_summary_html(self) -> str:
1117
+ """Return the regression summary as an HTML fragment."""
1118
+
1119
+ def _validate_equation(self) -> None:
1120
+ """Reject equation forms this backend cannot handle.
1121
+
1122
+ Default implementation is a no-op. Override to raise ``ValueError``
1123
+ with a helpful message if the equation is unsupported.
1124
+ """
1125
+ return None
1126
+
1127
+ def _validate_constraints(self) -> None:
1128
+ """Reject constraint specifications when the backend cannot use them.
1129
+
1130
+ Subclasses opt in by setting ``_supports_constraints = True``. The
1131
+ default raises ``ValueError`` if any constraint is supplied (whether
1132
+ from an inline ``ST.`` clause or via a ``constraints=`` kwarg).
1133
+
1134
+ Also verifies that every constrained name is a known parameter
1135
+ (present in ``self.c_params``) — catches typos like ``ST. C(7)=...``
1136
+ when the equation only uses ``C(1)..C(3)``.
1137
+ """
1138
+ if self.constraints and not self._supports_constraints:
1139
+ raise ValueError(
1140
+ f"{type(self).__name__} does not support constraints. "
1141
+ f"Constraints specified: {sorted(self.constraints)}. "
1142
+ "Use Estimate_nls_lmfit, or remove the ST. clause / "
1143
+ "constraints= kwarg."
1144
+ )
1145
+ if self.constraints:
1146
+ unknown = sorted(set(self.constraints) - set(self.c_params))
1147
+ if unknown:
1148
+ raise ValueError(
1149
+ f"{type(self).__name__}: constraints reference unknown "
1150
+ f"parameters {unknown}. Known parameters: {self.c_params}."
1151
+ )
1152
+
1153
+ def _prepare_estimation_df(self) -> None:
1154
+ """Backend-specific data preparation, run before :meth:`_fit`.
1155
+
1156
+ Default implementation does nothing — ``self.estimation_df`` is
1157
+ already set by :class:`Eq_parent` to ``self.eq_var_df``, which is
1158
+ what the residual-evaluating backends (lmfit, EViews) need.
1159
+
1160
+ :class:`Estimate_ols` overrides this to materialize regressor columns
1161
+ via mfcalc before fitting.
1162
+ """
1163
+ return None
1164
+
1165
+ # ---- shared orchestration (do not override) ----------------------------
1166
+
1167
+ def __post_init__(self) -> None:
1168
+ # 1) Parse the equation, build helper equations, build eq_var_df.
1169
+ super().__post_init__()
1170
+ # 2) Reject constraints the backend can't apply; verify names are known.
1171
+ self._validate_constraints()
1172
+ # 3) Backend-specific validation.
1173
+ self._validate_equation()
1174
+ # 4) Backend-specific data prep (e.g. OLS materializes regressors).
1175
+ self._prepare_estimation_df()
1176
+ # 5) Run the backend.
1177
+ self.regression_model, self.coef_estimate_dict = self._fit()
1178
+ # 5) Verify the backend covered every parameter.
1179
+ missing = [p for p in self.c_params if p not in self.coef_estimate_dict]
1180
+ if missing:
1181
+ raise RuntimeError(
1182
+ f"{type(self).__name__}._fit did not return estimates for "
1183
+ f"every parameter in c_params; missing: {missing}"
1184
+ )
1185
+ # 6) Populate eq_var_df with estimates so af_df / residuals_df work.
1186
+ for p in self.c_params:
1187
+ self.eq_var_df[p] = self.coef_estimate_dict[p]
1188
+ # 7) Build the baked (numeric) equation string.
1189
+ self.org_eq_baked = expand_equation_with_coefficients(
1190
+ self.org_eq_clean,
1191
+ coefficients=self.coef_estimate_dict,
1192
+ decimals=10,
1193
+ )
1194
+ # 8) Reporting wrapper + coefficient series.
1195
+ self.mfresult = LSResult(self)
1196
+ self.coef_ser = pd.Series(self.coef_estimate_dict, name=self.caption)
1197
+
1198
+ # ---- shared helpers backends can use -----------------------------------
1199
+
1200
+ def _map_statsmodels_params(
1201
+ self, raw_params: Dict[str, float]
1202
+ ) -> Dict[str, float]:
1203
+ """Map statsmodels-style parameter names to ``{est_param}__n`` tokens.
1204
+
1205
+ statsmodels exposes regressor columns by name (e.g. ``'c__2'`` for the
1206
+ column built from the parser's ``c__2 = <expr>`` line). This helper
1207
+ translates those names back to the canonical ``{est_param}__n`` form
1208
+ used elsewhere in the module.
1209
+ """
1210
+ mapped: Dict[str, float] = {}
1211
+ for k, v in raw_params.items():
1212
+ m = re.search(r"\bc__([0-9]+)\b", k, flags=re.IGNORECASE)
1213
+ if m:
1214
+ mapped[f"{self.est_param}__{m.group(1)}"] = v
1215
+ continue
1216
+ m2 = re.search(
1217
+ rf"\b{re.escape(self.est_param)}__([0-9]+)\b",
1218
+ k, flags=re.IGNORECASE,
1219
+ )
1220
+ if m2:
1221
+ mapped[f"{self.est_param}__{m2.group(1)}"] = v
1222
+ continue
1223
+ for _pname in (self.param_names or []):
1224
+ _up = _pname.upper()
1225
+ m3 = re.search(
1226
+ rf"\b{re.escape(_up)}__([0-9]+)\b", k, flags=re.IGNORECASE
1227
+ )
1228
+ if m3:
1229
+ mapped[f"{_up}__{m3.group(1)}"] = v
1230
+ break
1231
+ return mapped
1232
+
1233
+ # ---- A/F and residuals (computed once, cached) -------------------------
1234
+
1235
+ @cached_property
1236
+ def af_df(self) -> pd.DataFrame:
1237
+ """Actual and fitted series over the estimation sample."""
1238
+ start, end = self.smpl
1239
+ res = self.eq_var_df.mfcalc(f"<{start},{end}> {self.rhs_fit_eq}")
1240
+ res = res.mfcalc(f"<{start},{end}> {self.lhs_actual_eq}")
1241
+ return (
1242
+ res.loc[start:end, ["ACTUAL", "FITTED"]]
1243
+ .rename(columns=lambda s: s.lower().capitalize())
1244
+ )
1245
+
1246
+ @cached_property
1247
+ def residuals_df(self) -> pd.DataFrame:
1248
+ """Residuals over the estimation sample."""
1249
+ start, end = self.smpl
1250
+ res = self.eq_var_df.mfcalc(f"<{start},{end}> {self.residual_eq}")
1251
+ return (
1252
+ res.loc[start:end, ["RESIDUALS"]]
1253
+ .rename(columns=lambda s: s.lower().capitalize())
1254
+ )
1255
+
1256
+ @cached_property
1257
+ def estimation_smpl(self) -> Tuple[Any, Any]:
1258
+ """The usable sample, derived from non-missing residuals.
1259
+
1260
+ Returns the ``(first, last)`` index labels with non-missing residuals.
1261
+ Backends that prefer a different definition can override.
1262
+ """
1263
+ df = self.residuals_df
1264
+ mask = df["Residuals"].notna()
1265
+ if not mask.any():
1266
+ print("Not all data are available")
1267
+ print(self.eq_var_df.loc[self.smpl[0]:self.smpl[1], self.eq__var])
1268
+ raise ValueError("Can't run estimation: residuals are all NaN.")
1269
+ first_idx = mask.idxmax()
1270
+ last_idx = mask[::-1].idxmax()
1271
+ return first_idx, last_idx
1272
+
1273
+ # ---- HTML report convenience -------------------------------------------
1274
+
1275
+ def _repr_html_(self) -> str:
1276
+ return self.mfresult.get_html_report(plot_format="svg")
1277
+
1278
+ def get_html_report(self, plot_format: str = "svg") -> str:
1279
+ return self.mfresult.get_html_report(plot_format=plot_format)
1280
+
1281
+ # ---- factory: pre-fill defaults ----------------------------------------
1282
+
1283
+ @classmethod
1284
+ def with_defaults(cls, input_df=None, **default_kwargs):
1285
+ """Return a callable that constructs instances with pre-filled defaults.
1286
+
1287
+ The returned callable accepts the equation either positionally or as
1288
+ ``org_eq=...``. Any keyword passed to it overrides the stored default.
1289
+
1290
+ Examples
1291
+ --------
1292
+ >>> ls = Estimate_nls_lmfit.with_defaults(
1293
+ ... smpl=(2012, 2019), input_df=npl,
1294
+ ... )
1295
+ >>> m1 = ls("DLOG(Y) = C(1) + C(2)*DLOG(X)")
1296
+ >>> m2 = ls(org_eq="DLOG(Z) = C(1) + C(3)*DLOG(W)", caption="Alt spec")
1297
+ """
1298
+ def factory(*args, **overrides):
1299
+ if args:
1300
+ if len(args) > 1:
1301
+ raise TypeError(
1302
+ f"{cls.__name__}.with_defaults factory accepts at "
1303
+ "most one positional argument (the equation string)."
1304
+ )
1305
+ org_eq = args[0]
1306
+ else:
1307
+ try:
1308
+ org_eq = overrides.pop("org_eq")
1309
+ except KeyError:
1310
+ raise TypeError(
1311
+ "Missing equation. Provide it positionally or as "
1312
+ "org_eq='...'."
1313
+ )
1314
+ params = {
1315
+ "input_df": input_df,
1316
+ **default_kwargs,
1317
+ **overrides,
1318
+ "org_eq": org_eq,
1319
+ }
1320
+ # Light pre-processing matching the previous helper.
1321
+ params["org_eq"] = (
1322
+ params["org_eq"].upper().replace("@ABS", "ABS")
1323
+ )
1324
+ return cls(**params)
1325
+
1326
+ # Metadata for callers such as Makemodel. Reading these attributes
1327
+ # lets Makemodel resolve equation-local samples against the factory's
1328
+ # captured dataframe without constructing a dummy estimator, which
1329
+ # would run a real fit in EstimatorBackend.__post_init__.
1330
+ factory._estimator_defaults = {
1331
+ "input_df": input_df,
1332
+ **default_kwargs,
1333
+ }
1334
+ factory._estimator_class = cls
1335
+
1336
+ return factory
1337
+
1338
+
1339
+ # =============================================================================
1340
+ # Estimate_ols — OLS via statsmodels
1341
+ # =============================================================================
1342
+
1343
+ @dataclass
1344
+ class Estimate_ols(EstimatorBackend):
1345
+ """Ordinary Least Squares estimation via ``statsmodels``.
1346
+
1347
+ The equation is parsed by :class:`EquationParse` (with ``ecm=False``),
1348
+ each parameterized term ``C__n * <expr>`` is materialized as a regressor
1349
+ column via mfcalc, and the resulting frame is fed to ``sm.OLS``.
1350
+
1351
+ Including an intercept
1352
+ ----------------------
1353
+ Write the intercept directly into the equation, e.g.
1354
+ ``Y = C(1) + C(2) * X``. The parser emits ``C__1 = 1.0`` as a
1355
+ regressor column, statsmodels estimates it, and the value flows
1356
+ through every downstream artifact (``coef_estimate_dict``,
1357
+ ``org_eq_baked``, the FRML emitted by :class:`EqContainer`, the
1358
+ HTML report). To fit a regression through the origin, simply omit
1359
+ ``C(1)``.
1360
+
1361
+ Parameters
1362
+ ----------
1363
+ org_eq : str
1364
+ Linear-in-parameters equation, e.g. ``Y = C(1) + C(2)*X + C(3)*Z``.
1365
+ input_df : pandas.DataFrame
1366
+ Source data.
1367
+ smpl : tuple[int, int], default (2002, 2018)
1368
+ Estimation sample.
1369
+
1370
+ Attributes
1371
+ ----------
1372
+ regression_model : statsmodels.regression.linear_model.RegressionResultsWrapper
1373
+ The fitted statsmodels result object (stored, not refit on access).
1374
+ """
1375
+ method: str = "OLS"
1376
+ ecm: bool = False # OLS is linear-in-params, not ECM
1377
+
1378
+ # OLS-specific: the parser builds regressor columns we feed to statsmodels.
1379
+ mfcalc_code: List[str] = field(init=False, default_factory=list)
1380
+
1381
+ def _validate_equation(self) -> None:
1382
+ """OLS requires a linear-in-parameters equation. The parser will
1383
+ raise for nested parameter usage; we leave the LHS form to whatever
1384
+ the parser emits as ``LHS = <expr>`` so transformed dependent
1385
+ variables (e.g. ``LOG(Y)``) work.
1386
+
1387
+ Two additional checks:
1388
+
1389
+ - **Error** if the equation has no parameter placeholders at all
1390
+ (``Y = X`` with no ``C(...)``). There is nothing to estimate.
1391
+ - **Warn** if the intercept slot ``{est_param}__1`` is missing.
1392
+ This is a regression through the origin, which is occasionally
1393
+ intentional but more often an oversight.
1394
+ """
1395
+ intercept_slot = f"{self.est_param}__1"
1396
+ if not self.c_params:
1397
+ raise ValueError(
1398
+ f"{type(self).__name__}: equation has no parameters to "
1399
+ f"estimate ({self.org_eq!r}). Add at least one "
1400
+ f"{self.est_param}(n) placeholder, e.g. "
1401
+ f"'Y = {self.est_param}(1) + {self.est_param}(2)*X'."
1402
+ )
1403
+ if intercept_slot not in self.c_params:
1404
+ print(
1405
+ f"[{type(self).__name__}({self.endo_var})] WARNING: no "
1406
+ f"intercept ({self.est_param}(1)) in the equation — fitting "
1407
+ "through the origin. If that wasn't intentional, add "
1408
+ f"'{self.est_param}(1) + ' to the right-hand side."
1409
+ )
1410
+
1411
+ def _prepare_estimation_df(self) -> None:
1412
+ """Materialize the LHS and per-parameter regressor columns via mfcalc.
1413
+
1414
+ After this runs, ``self.estimation_df`` has columns
1415
+ ``["LHS", "C__1", "C__2", ...]`` (or ``["LHS", "c__1", ...]``,
1416
+ depending on what the parser emits) ready for ``sm.OLS``.
1417
+
1418
+ Rows containing NaN in any column are dropped — otherwise statsmodels
1419
+ would silently produce an all-NaN coefficient vector. The effective
1420
+ sample is recorded on ``self._effective_smpl`` (consumed by the
1421
+ :attr:`estimation_smpl` property) so the report reflects what was
1422
+ actually fit, not the user-requested window.
1423
+ """
1424
+ parser = EquationParse(
1425
+ org_eq_clean=self.org_eq_clean,
1426
+ ecm=self.ecm,
1427
+ est_param=self.est_param,
1428
+ param_names=self.param_names,
1429
+ )
1430
+ self.mfcalc_code = parser.mfcalc_code
1431
+
1432
+ # Restrict eq_var_df to columns the parser actually saw.
1433
+ used_cols = list(parser.used_vars & set(self.input_df.columns))
1434
+ self.eq_var_df = self.input_df[used_cols].copy()
1435
+
1436
+ # Run the mfcalc lines (except the trailing LHS_ORG_EQ) to build the
1437
+ # estimation frame; drop the original input columns.
1438
+ raw = self._run_mfcalc().loc[self.smpl[0]:self.smpl[1], :]
1439
+
1440
+ # Drop NaN rows and report what happened.
1441
+ self.estimation_df, self._effective_smpl = _trim_invalid_rows(
1442
+ raw,
1443
+ requested_smpl=self.smpl,
1444
+ label=f"{type(self).__name__}({self.endo_var})",
1445
+ )
1446
+
1447
+ def _run_mfcalc(self) -> pd.DataFrame:
1448
+ """Execute the parser's mfcalc lines (except the trailing
1449
+ ``LHS_ORG_EQ``) to materialize the LHS and regressor columns."""
1450
+ start, end = self.smpl
1451
+ df = self.eq_var_df.copy()
1452
+ to_drop = list(df.columns)
1453
+ failures: List[str] = []
1454
+ for line in self.mfcalc_code[:-1]:
1455
+ try:
1456
+ df = df.mfcalc(f"<{start},{end}> {line}")
1457
+ except Exception as e: # pragma: no cover -- parser-dependent
1458
+ failures.append(f"{line} -> {e}")
1459
+ if failures:
1460
+ raise RuntimeError(
1461
+ "OLS pre-fit mfcalc failures:\n " + "\n ".join(failures)
1462
+ )
1463
+ return df.drop(columns=to_drop)
1464
+
1465
+ # ---- contract ----------------------------------------------------------
1466
+
1467
+ def _fit(self) -> Tuple[Any, Dict[str, float]]:
1468
+ df = self.estimation_df
1469
+ if df.empty:
1470
+ raise ValueError(
1471
+ f"Estimate_ols: no usable rows in sample {self.smpl}. "
1472
+ "Every row contained at least one NaN or inf after mfcalc — "
1473
+ "check for data gaps, lags reaching outside the data, LOG "
1474
+ "of non-positive values, or division by zero."
1475
+ )
1476
+ y = df["LHS"]
1477
+ X = df.drop(columns=["LHS"])
1478
+ result = sm.OLS(y, X).fit(**self.fit_kws)
1479
+ coefs = self._map_statsmodels_params(result.params.to_dict())
1480
+ return result, coefs
1481
+
1482
+ def _render_summary_html(self) -> str:
1483
+ html_text = self.regression_model.summary().as_html()
1484
+ return html_text.replace(
1485
+ "<caption>OLS Regression Results</caption>",
1486
+ "<caption><h3>OLS Regression Results</h3></caption>",
1487
+ )
1488
+
1489
+ # ---- override estimation_smpl: OLS reports its EFFECTIVE sample --------
1490
+
1491
+ @cached_property
1492
+ def estimation_smpl(self) -> Tuple[Any, Any]:
1493
+ """The sample actually used by the fit, after dropping NaN rows.
1494
+
1495
+ May be narrower than ``self.smpl`` when lags or transformations
1496
+ introduce boundary NaNs.
1497
+ """
1498
+ return getattr(self, "_effective_smpl", self.smpl)
1499
+
1500
+
1501
+ # =============================================================================
1502
+ # Estimate_nls_lmfit — nonlinear least squares via lmfit
1503
+ # =============================================================================
1504
+
1505
+ @dataclass
1506
+ class Estimate_nls_lmfit(EstimatorBackend):
1507
+ """Nonlinear Least Squares via ``lmfit``.
1508
+
1509
+ Iterates over the parameter vector, evaluating the residual equation
1510
+ through ModelFlow's mfcalc each step and minimizing via
1511
+ :func:`lmfit.minimize`.
1512
+
1513
+ Parameters
1514
+ ----------
1515
+ default_params : dict, optional
1516
+ Per-parameter initialization for lmfit, e.g.
1517
+ ``{'C__2': {'value': -0.1, 'min': -1.0, 'max': 0.0}}``. Parameters
1518
+ without an entry default to ``value=0.1``.
1519
+ constraints : dict, optional
1520
+ Per-parameter lmfit kwargs (``min``, ``max``, ``vary``, ``expr``).
1521
+ Wins over ``default_params`` on key collisions. Also populated by
1522
+ parsing an inline ``ST.`` clause in ``org_eq`` — see notes.
1523
+ method : str, default "least_squares"
1524
+ The lmfit minimizer method (passed to :func:`lmfit.minimize`).
1525
+ fit_kws : dict, optional
1526
+ Additional keyword arguments forwarded to :func:`lmfit.minimize`.
1527
+
1528
+ Notes
1529
+ -----
1530
+ For ECM-style equations the speed-of-adjustment coefficient typically
1531
+ lives in :math:`(-1, 0)`. Pass an explicit initial value via
1532
+ ``default_params`` (``{'C__2': {'value': -0.1, 'min': -1.0, 'max': 0.0}}``)
1533
+ to help the solver.
1534
+
1535
+ **Inline constraints via ``ST.`` clause.** Constraints can also be
1536
+ written directly in the equation string, separated from the equation
1537
+ by ``ST.`` and from each other by ``;``::
1538
+
1539
+ Y = C(1) + C(2)*X + ALFA*Z ST. C(1)~0.5; C(2)>0; ALFA:=C(1)+C(2)
1540
+
1541
+ Recognized forms:
1542
+
1543
+ - ``NAME=[lo, hi]`` — bounds (``min``/``max``)
1544
+ - ``NAME > x`` / ``NAME >= x`` — lower bound
1545
+ - ``NAME < x`` / ``NAME <= x`` — upper bound
1546
+ - ``NAME = value`` — fixed value (``vary=False``)
1547
+ - ``NAME ~ value`` — initial value (free to change)
1548
+ - ``NAME := expr`` — algebraic relation (``expr``)
1549
+ """
1550
+ default_params: dict = field(default_factory=dict)
1551
+ method: str = "least_squares"
1552
+ frml_name: str = "<STOC,DAMP>"
1553
+ add_add_factor: bool = True
1554
+ make_fixable: bool = True
1555
+
1556
+ # Constraints (bounds / fixed values / algebraic expressions) are
1557
+ # supported through lmfit's Parameter kwargs (min, max, vary, expr).
1558
+ _supports_constraints: ClassVar[bool] = True
1559
+
1560
+ # Stored for inspection by callers.
1561
+ lmfit_params: Optional[Parameters] = field(init=False, default=None)
1562
+
1563
+ # ---- contract ----------------------------------------------------------
1564
+
1565
+ def _fit(self) -> Tuple[Any, Dict[str, float]]:
1566
+ # Merge per-parameter kwargs from (priority order, last wins):
1567
+ # 1. default_params (initial values / bounds from caller)
1568
+ # 2. constraints (parsed from inline ST. clause or kwarg)
1569
+ # Parameters with an ``expr`` are derived — add them last so any
1570
+ # referenced free parameters already exist in the Parameters set.
1571
+ params = Parameters()
1572
+ derived: List[str] = []
1573
+ for name in self.c_params:
1574
+ kwargs = dict(self.default_params.get(name, {}))
1575
+ kwargs.update(self.constraints.get(name, {}))
1576
+ if 'expr' in kwargs:
1577
+ derived.append(name)
1578
+ continue
1579
+ kwargs.setdefault("value", 0.1)
1580
+ params.add(name=name, **kwargs)
1581
+ for name in derived:
1582
+ kwargs = dict(self.default_params.get(name, {}))
1583
+ kwargs.update(self.constraints.get(name, {}))
1584
+ kwargs.setdefault("value", 0.1)
1585
+ params.add(name=name, **kwargs)
1586
+ self.lmfit_params = params
1587
+
1588
+ mresidual = model(self.residual_eq)
1589
+ start, end = self.smpl
1590
+
1591
+ def residual(p):
1592
+ values = [p.valuesdict()[name] for name in self.c_params]
1593
+ self.eq_var_df.loc[:, self.c_params] = values
1594
+ res = mresidual(self.eq_var_df, start, end, silent=True)
1595
+ return res.loc[start:end, "RESIDUALS"].to_numpy()
1596
+
1597
+ result = minimize(
1598
+ residual,
1599
+ params,
1600
+ nan_policy="omit",
1601
+ method=self.method,
1602
+ calc_covar=True,
1603
+ **self.fit_kws,
1604
+ )
1605
+ return result, dict(result.params.valuesdict())
1606
+
1607
+ def _render_summary_html(self) -> str:
1608
+ html_text = self.regression_model._repr_html_()
1609
+ return html_text.replace(
1610
+ "<h2>Fit Result</h2>", "<h3>NLS Regression Results</h3>"
1611
+ )
1612
+
1613
+
1614
+ # =============================================================================
1615
+ # Estimate_nls_eviews — nonlinear least squares via EViews (py2eviews)
1616
+ # =============================================================================
1617
+
1618
+ @dataclass
1619
+ class Estimate_nls_eviews(EstimatorBackend):
1620
+ """Nonlinear Least Squares via ``EViews``.
1621
+
1622
+ Round-trips the working dataframe to EViews, runs ``equation eq1.nls``
1623
+ followed by ``eq1.ls <equation>``, and reads the estimated coefficient
1624
+ vector back. The ``py2eviews`` package is imported lazily so that other
1625
+ backends do not depend on EViews being installed.
1626
+
1627
+ Parameters
1628
+ ----------
1629
+ fit_kws : dict, optional
1630
+ Currently unused; kept for parity with other backends.
1631
+
1632
+ Attributes
1633
+ ----------
1634
+ regression_model : str
1635
+ The raw EViews spool text (output of ``ib.save``). Treated as opaque
1636
+ and rendered to HTML by :func:`eviews_output_to_html`.
1637
+ """
1638
+ frml_name: str = "<STOC,DAMP>"
1639
+ add_add_factor: bool = True
1640
+ make_fixable: bool = True
1641
+
1642
+ def _fit(self) -> Tuple[Any, Dict[str, float]]:
1643
+ import py2eviews as evp # lazy: only this backend needs it
1644
+
1645
+ start, end = self.smpl
1646
+ eviews_eq = restore_c_params(
1647
+ self.org_eq_clean, est_param=self.est_param
1648
+ )
1649
+ eviews_eq = re.sub(r"\bABS\(", "@ABS(", eviews_eq)
1650
+
1651
+ eviewsapp = evp.GetEViewsApp(instance="new", showwindow=True)
1652
+ try:
1653
+ df_here = self.eq_var_df.copy()
1654
+ df_here.index = pd.to_datetime(df_here.index, format="%Y")
1655
+ evp.PutPythonAsWF(df_here, app=eviewsapp)
1656
+
1657
+ with tempfile.NamedTemporaryFile(delete=False) as tf:
1658
+ temp_path = Path(tf.name)
1659
+
1660
+ runlines = (
1661
+ f"smpl {start} {end}\n"
1662
+ f"cd {temp_path.parent}\n"
1663
+ f"equation eq1.nls\n"
1664
+ f"eq1.ls {eviews_eq}\n"
1665
+ f"spool ib\n"
1666
+ f"eq1.output\n"
1667
+ f"ib.append eq1.output\n"
1668
+ f"ib.display\n"
1669
+ f"ib.save(t=txt) {temp_path}\n"
1670
+ )
1671
+ for line in runlines.splitlines():
1672
+ evp.Run(line, app=eviewsapp)
1673
+
1674
+ # Read back the C vector. We index by the parameter numbers in
1675
+ # self.c_params rather than filtering on non-zero values, so a
1676
+ # legitimate zero estimate is not silently dropped.
1677
+ c_vector = evp.Get("C", app=eviewsapp)
1678
+ coefs: Dict[str, float] = {}
1679
+ for p in self.c_params:
1680
+ # Parameter token is "{prefix}__n"; n is 1-based, c_vector
1681
+ # is 0-indexed.
1682
+ n = int(p.split("__")[1])
1683
+ idx = n - 1
1684
+ if idx < 0 or idx >= len(c_vector):
1685
+ raise RuntimeError(
1686
+ f"EViews C vector has length {len(c_vector)} but "
1687
+ f"parameter {p} requires index {idx}."
1688
+ )
1689
+ coefs[p] = float(c_vector[idx])
1690
+
1691
+ with open(temp_path.with_suffix(".txt"), "rt") as f:
1692
+ eviews_spool = f.read()
1693
+ finally:
1694
+ try:
1695
+ eviewsapp.Hide()
1696
+ except Exception:
1697
+ pass
1698
+ evp.Cleanup()
1699
+
1700
+ return eviews_spool, coefs
1701
+
1702
+ def _render_summary_html(self) -> str:
1703
+ return eviews_output_to_html(self.regression_model)
1704
+
1705
+
1706
+ # =============================================================================
1707
+ # Estimate_nls — factory class dispatching on solver
1708
+ # =============================================================================
1709
+
1710
+ class Estimate_nls:
1711
+ """Factory class dispatching on ``solver`` to the right NLS backend.
1712
+
1713
+ Calling ``Estimate_nls(eq, solver='lmfit', ...)`` returns an
1714
+ :class:`Estimate_nls_lmfit` instance; ``solver='eviews'`` returns an
1715
+ :class:`Estimate_nls_eviews`. The factory itself is never instantiated
1716
+ — ``__new__`` returns the concrete subclass directly.
1717
+
1718
+ This class is the backwards-compatible entry point for code written
1719
+ against the older single-class API. New code can call the concrete
1720
+ classes directly. Either way, :meth:`with_defaults` is available and
1721
+ behaves identically to the version on the concrete backends — so
1722
+ ``Estimate_nls.with_defaults(input_df=df, smpl=(...), solver='lmfit')``
1723
+ works the same as ``Estimate_nls_lmfit.with_defaults(...)``.
1724
+
1725
+ Examples
1726
+ --------
1727
+ >>> ls = Estimate_nls.with_defaults(input_df=df, smpl=(2012, 2019))
1728
+ >>> m1 = ls("DLOG(Y) = C(1) + C(2)*DLOG(X)")
1729
+ >>> # Pass solver='eviews' as a default if you want EViews:
1730
+ >>> ev = Estimate_nls.with_defaults(input_df=df, solver='eviews')
1731
+ >>> m2 = ev("DLOG(Y) = C(1) - C(2) * (LOG(Y(-1)) - LOG(X(-1)))")
1732
+ """
1733
+
1734
+ def __new__(cls, org_eq: str = "", *, solver: str = "lmfit", **kwargs):
1735
+ # The factory never produces an instance of itself; it picks a
1736
+ # concrete backend and returns *that*. Python skips __init__ on
1737
+ # the calling class when __new__ returns a different class, so
1738
+ # the concrete backend's dataclass __init__ runs normally.
1739
+ if solver == "lmfit":
1740
+ return Estimate_nls_lmfit(org_eq=org_eq, **kwargs)
1741
+ if solver == "eviews":
1742
+ return Estimate_nls_eviews(org_eq=org_eq, **kwargs)
1743
+ raise ValueError(
1744
+ f"Unknown solver {solver!r}. Use 'lmfit' or 'eviews', or "
1745
+ "instantiate the concrete class (Estimate_nls_lmfit / "
1746
+ "Estimate_nls_eviews) directly."
1747
+ )
1748
+
1749
+ @classmethod
1750
+ def with_defaults(cls, input_df=None, **default_kwargs):
1751
+ """Return a callable that constructs NLS estimators with pre-filled
1752
+ defaults.
1753
+
1754
+ Behaves identically to :meth:`EstimatorBackend.with_defaults` —
1755
+ the only difference is that ``cls(**params)`` here goes through
1756
+ :meth:`__new__`, so a ``solver=...`` default (or override on a
1757
+ per-call basis) is honored.
1758
+
1759
+ Examples
1760
+ --------
1761
+ >>> ls = Estimate_nls.with_defaults(
1762
+ ... input_df=df, smpl=(2012, 2019), # solver defaults to 'lmfit'
1763
+ ... )
1764
+ >>> m1 = ls("DLOG(Y) = C(1) + C(2)*DLOG(X)")
1765
+ >>>
1766
+ >>> # Override solver per call:
1767
+ >>> m2 = ls("DLOG(Y) = C(1) - C(2)*X", solver="eviews")
1768
+ """
1769
+ def factory(*args, **overrides):
1770
+ if args:
1771
+ if len(args) > 1:
1772
+ raise TypeError(
1773
+ f"{cls.__name__}.with_defaults factory accepts at "
1774
+ "most one positional argument (the equation string)."
1775
+ )
1776
+ org_eq = args[0]
1777
+ else:
1778
+ try:
1779
+ org_eq = overrides.pop("org_eq")
1780
+ except KeyError:
1781
+ raise TypeError(
1782
+ "Missing equation. Provide it positionally or as "
1783
+ "org_eq='...'."
1784
+ )
1785
+ params = {
1786
+ "input_df": input_df,
1787
+ **default_kwargs,
1788
+ **overrides,
1789
+ "org_eq": org_eq,
1790
+ }
1791
+ # Light pre-processing matching the concrete backends' helpers.
1792
+ params["org_eq"] = (
1793
+ params["org_eq"].upper().replace("@ABS", "ABS")
1794
+ )
1795
+ return cls(**params)
1796
+
1797
+ # Metadata for callers such as Makemodel. Reading these attributes
1798
+ # lets Makemodel resolve equation-local samples against the factory's
1799
+ # captured dataframe without constructing a dummy estimator, which
1800
+ # would run a real fit in EstimatorBackend.__post_init__.
1801
+ factory._estimator_defaults = {
1802
+ "input_df": input_df,
1803
+ **default_kwargs,
1804
+ }
1805
+ factory._estimator_class = cls
1806
+
1807
+ return factory
1808
+
1809
+
1810
+ # =============================================================================
1811
+ # LSResult — backend-agnostic HTML report wrapper
1812
+ # =============================================================================
1813
+
1814
+ @dataclass
1815
+ class LSResult:
1816
+ """Backend-agnostic wrapper that builds the HTML report for an estimator.
1817
+
1818
+ The report consists of:
1819
+
1820
+ 1. A title block with caption, endogenous variable, sample, and the
1821
+ original / expanded / normalized equations.
1822
+ 2. The backend's regression summary (delegated to
1823
+ :meth:`EstimatorBackend._render_summary_html`).
1824
+ 3. A responsive Actual-vs-Fitted plot.
1825
+
1826
+ :class:`LSResult` doesn't know which backend produced the estimator —
1827
+ it asks the estimator to render its own summary.
1828
+ """
1829
+ olsmodel: EstimatorBackend # named for backwards compatibility
1830
+
1831
+ def __post_init__(self) -> None:
1832
+ # LSResult exposes the report-facing view of an estimator. We pull
1833
+ # forward the handful of fields downstream consumers (export
1834
+ # helpers, `_repr_html_`, the report builder) need so they don't
1835
+ # have to reach through to `olsmodel` for trivia.
1836
+ self.result = self.olsmodel.regression_model
1837
+ self.af_df = self.olsmodel.af_df
1838
+ self.residuals_df = self.olsmodel.residuals_df
1839
+ self.omodel = self.olsmodel.omodel
1840
+ self.caption = self.olsmodel.caption
1841
+ self.endo_var = self.olsmodel.endo_var
1842
+ self.estimator = self.olsmodel.__class__.__name__
1843
+ self.normal = nz.normal(self.olsmodel.org_eq_baked)
1844
+
1845
+ def get_html_report(self, plot_format: str = "svg") -> str:
1846
+ """Build the full HTML report.
1847
+
1848
+ Parameters
1849
+ ----------
1850
+ plot_format : {"svg", "png"}, default "svg"
1851
+ Format for the embedded actual-vs-fitted plot.
1852
+ """
1853
+ if plot_format not in ("svg", "png"):
1854
+ raise ValueError("plot_format must be 'svg' or 'png'")
1855
+
1856
+ title_html = self._build_title_html()
1857
+ summary_html = self.olsmodel._render_summary_html()
1858
+ plot_html = self._build_plot_html(plot_format)
1859
+ return title_html + summary_html + plot_html
1860
+
1861
+ def show(self, plot_format: str = "svg") -> None:
1862
+ """Display the report inline (in a Jupyter notebook)."""
1863
+ display(HTML(self.get_html_report(plot_format=plot_format)))
1864
+
1865
+ def _repr_html_(self) -> str:
1866
+ return self.get_html_report(plot_format="svg")
1867
+
1868
+ # -- internals ------------------------------------------------------------
1869
+
1870
+ def _build_title_html(self) -> str:
1871
+ est_param = getattr(self.olsmodel, "est_param", "C")
1872
+ # Apply the same line-break helper to all three rendered forms so
1873
+ # they wrap consistently. The helper has two passes: one for
1874
+ # parameter tokens (C__10, ...) used in the original and
1875
+ # normalized forms, and one for substituted numeric literals
1876
+ # ((0.5), ...) used in the expanded form.
1877
+ wrap = lambda eq: _add_linebreaks_before_long_params(eq, param=est_param)
1878
+ original = wrap(self.olsmodel.org_eq)
1879
+ expanded = wrap(self.olsmodel.org_eq_baked)
1880
+ normalized = wrap(self.normal.normalized)
1881
+ start, end = self.olsmodel.estimation_smpl
1882
+ endo = self.olsmodel.endo_var
1883
+ desc = self.omodel.var_description.get(endo, "")
1884
+ return (
1885
+ f"<h2>{self.olsmodel.caption}: {endo}: {desc}</h2>"
1886
+ f"<p><strong>Sample:</strong> {start} to {end}</p>"
1887
+ f"<p><strong>Original Equation:</strong><br>"
1888
+ f"<pre><code>{original}</code></pre></p>"
1889
+ f"<p><strong>Expanded Equation:</strong><br>"
1890
+ f"<pre><code>{expanded}</code></pre></p>"
1891
+ f"<p><strong>Normalized Equation:</strong><br>"
1892
+ f"<pre><code>{normalized}</code></pre></p>"
1893
+ )
1894
+
1895
+ def _build_plot_html(self, plot_format: str) -> str:
1896
+ buf = BytesIO()
1897
+ plot_actual_vs_fitted(
1898
+ self.olsmodel, save_to=buf, format=plot_format,
1899
+ figsize=(8, 5), dpi=150,
1900
+ )
1901
+ buf.seek(0)
1902
+ b64 = base64.b64encode(buf.read()).decode("utf-8")
1903
+ mime = "svg+xml" if plot_format == "svg" else "png"
1904
+ return (
1905
+ "<h3>Actual vs Fitted Plot</h3>"
1906
+ f'<img style="max-width:100%; height:auto;" '
1907
+ f'src="data:image/{mime};base64,{b64}" />'
1908
+ )
1909
+
1910
+
1911
+ def _add_linebreaks_before_long_params(
1912
+ equation: str, param: str = "C"
1913
+ ) -> str:
1914
+ """Insert a newline before each parameterized term so long equations
1915
+ render readably in the report.
1916
+
1917
+ Two passes:
1918
+
1919
+ 1. Before any ``{param}__<2-or-more-digits>`` token. Catches
1920
+ ``C__10``, ``C__11``, ... in the *original* and *normalized* forms,
1921
+ where placeholders are still present.
1922
+ 2. Before any parenthesized numeric literal that sits on a ``+`` /
1923
+ ``-`` boundary. Catches ``(0.5)``, ``(-0.3)``, ``(1.234567)``, ... in
1924
+ the *expanded* form, where placeholders have been substituted with
1925
+ their estimated values. The lookbehind for ``+``/``-`` is what
1926
+ distinguishes a substituted-parameter term (which always sits at
1927
+ an additive boundary) from a user-written parenthesized constant
1928
+ (which typically sits next to ``*`` or ``/`` instead).
1929
+
1930
+ A trailing word boundary on the parameter pattern keeps the match
1931
+ from extending into longer identifiers that happen to start with
1932
+ ``{param}__N``.
1933
+ """
1934
+ # Pass 1: before C__10, C__11, ... — the original / normalized form.
1935
+ pattern_param = rf"\s*{re.escape(param)}__\d{{2,}}\b"
1936
+ equation = re.sub(
1937
+ pattern_param, lambda m: "\n" + m.group(0).strip(), equation
1938
+ )
1939
+ # Pass 2: before (0.5), (-0.3), ... when they sit on a binary +/-
1940
+ # boundary — i.e. the substituted form. The leading capture group
1941
+ # ([A-Za-z0-9_)]) requires the operator to be preceded by an
1942
+ # operand-ending character, distinguishing a binary +/- from a unary
1943
+ # minus at expression start (where the preceding char is `=`, `(`, or
1944
+ # space). Whitespace on either side of the operator is optional —
1945
+ # both spaced ('A + (0.5)') and dense ('A+(0.5)') forms match.
1946
+ pattern_subst = r"([A-Za-z0-9_)])\s*([+\-])\s*(?=\(-?\d+(?:\.\d+)?\))"
1947
+ equation = re.sub(pattern_subst, r"\1\2\n", equation)
1948
+ return equation
1949
+
1950
+
1951
+ # =============================================================================
1952
+ # EqContainer — stitch equations into a ModelFlow model
1953
+ # =============================================================================
1954
+
1955
+ @dataclass
1956
+ class EqContainer:
1957
+ """Container of equations (estimators or :class:`Eq` identities).
1958
+
1959
+ Provides helpers to emit a normalized ModelFlow model, generate
1960
+ add-factor equations, and align historical data via add factors.
1961
+ """
1962
+ equations: List[Union[EstimatorBackend, Eq_parent, str]] = field(
1963
+ default_factory=list
1964
+ )
1965
+
1966
+ def test(self) -> None:
1967
+ """Print a one-line summary of each equation in the container."""
1968
+ for eq in self.equations:
1969
+ print(f"{eq.__class__.__name__:12} {eq.org_eq}")
1970
+
1971
+ @property
1972
+ def clean_normal(self) -> str:
1973
+ """Normalized equations (no add-factors), one per line."""
1974
+ return "\n".join(
1975
+ nz.normal(eq.org_eq_baked, add_add_factor=False).normalized
1976
+ for eq in self.equations
1977
+ )
1978
+
1979
+ @property
1980
+ def clean(self) -> str:
1981
+ """Original equations, one per line."""
1982
+ return "\n".join(eq.org_eq for eq in self.equations)
1983
+
1984
+ @property
1985
+ def model_clean(self):
1986
+ """A ModelFlow model from the no-add-factor normalized equations."""
1987
+ tmodel = model(self.clean_normal)
1988
+ tmodel.var_description = tmodel.enrich_var_description(
1989
+ self.var_description
1990
+ )
1991
+ return tmodel
1992
+
1993
+ @property
1994
+ def eqs_norm(self) -> str:
1995
+ """Normalized equations with FRML headers and per-equation flags."""
1996
+ parts = []
1997
+ for eq in self.equations:
1998
+ normed = nz.normal(
1999
+ eq.org_eq_baked,
2000
+ add_add_factor=eq.add_add_factor,
2001
+ make_fixable=eq.make_fixable,
2002
+ make_fitted=eq.make_fitted,
2003
+ ).normalized
2004
+ parts.append(f"{eq.frml_name} {normed}")
2005
+ return "\n".join(parts)
2006
+
2007
+ @property
2008
+ def model(self):
2009
+ """A full ModelFlow model including add-factor calculations."""
2010
+ tmodel = model(self.eqs_norm + "\n" + self.eqs_add_model)
2011
+ tmodel.var_description = tmodel.enrich_var_description(
2012
+ self.var_description
2013
+ )
2014
+ return tmodel
2015
+
2016
+ @property
2017
+ def var_description(self) -> dict:
2018
+ """Union of variable descriptions across member equations."""
2019
+ return reduce(
2020
+ lambda acc, eq: acc | eq.omodel.var_description,
2021
+ self.equations,
2022
+ {},
2023
+ )
2024
+
2025
+ @property
2026
+ def eqs_add_model(self) -> str:
2027
+ """Add-factor calculation equations (only for equations with
2028
+ ``add_add_factor=True``)."""
2029
+ parts = []
2030
+ for eq in self.equations:
2031
+ if not eq.add_add_factor:
2032
+ continue
2033
+ calc = nz.normal(
2034
+ eq.org_eq_baked,
2035
+ add_add_factor=eq.add_add_factor,
2036
+ make_fixable=eq.make_fixable,
2037
+ make_fitted=eq.make_fitted,
2038
+ ).calc_add_factor
2039
+ parts.append(f"<CALC_ADD_FACTOR> {calc}")
2040
+ return "\n".join(parts)
2041
+
2042
+ @property
2043
+ def add_model(self):
2044
+ """A ModelFlow model containing only the add-factor calculations."""
2045
+ return model(self.eqs_add_model.replace("<CALC_ADD_FACTOR>", "<CALC>"))
2046
+
2047
+ def init_addfactors(
2048
+ self,
2049
+ df: pd.DataFrame,
2050
+ start: Union[str, int] = "",
2051
+ end: Union[str, int] = "",
2052
+ show: bool = False,
2053
+ check: bool = False,
2054
+ silent: bool = True,
2055
+ multiplier: float = 1.0,
2056
+ ) -> pd.DataFrame:
2057
+ """Compute and apply add factors so the model reproduces ``df``.
2058
+
2059
+ Parameters
2060
+ ----------
2061
+ df : pandas.DataFrame
2062
+ Historical data to align with.
2063
+ start, end : str | int, optional
2064
+ Alignment window (defaults to the full range).
2065
+ show : bool, default False
2066
+ If True, print the calculated add factors.
2067
+ check : bool, default False
2068
+ If True, re-simulate using the aligned data and print the
2069
+ difference between actual and simulated outcomes.
2070
+ silent : bool, default True
2071
+ Silence ModelFlow runtime output.
2072
+ multiplier : float, default 1.0
2073
+ Scale factor for the residual check printout.
2074
+
2075
+ Returns
2076
+ -------
2077
+ pandas.DataFrame
2078
+ A modified copy of ``df`` with add factors applied.
2079
+ """
2080
+ am = self.add_model
2081
+ aligned = am(df, start=start, end=end, silent=silent)
2082
+ if show:
2083
+ print("\n\nAdd factors to align historic values and model results")
2084
+ print(am["*_A"].df)
2085
+ if check:
2086
+ full = self.model
2087
+ _ = full(aligned, start=start, end=end, silent=silent)
2088
+ print("\n\nDifference between historic values and model results")
2089
+ if multiplier != 1.0:
2090
+ print(f"Multiplied by {multiplier}")
2091
+ full.basedf = df
2092
+ display(full["#ENDO"].dif.df * multiplier)
2093
+ return aligned
2094
+
2095
+ def estimation_report(
2096
+ self,
2097
+ path: str = "html",
2098
+ filename: str = "estimation_report.html",
2099
+ plot_format: str = "svg",
2100
+ title: str = "Estimation Summary",
2101
+ open_file: bool = False,
2102
+ report_all: bool = False,
2103
+ ) -> None:
2104
+ """Export an interactive HTML report for the equations in this
2105
+ container.
2106
+
2107
+ By default, only estimated equations (subclasses of
2108
+ :class:`EstimatorBackend`) are included. Identity equations
2109
+ (:class:`Eq` instances) are silently filtered out. With
2110
+ ``report_all=True``, identities are also included as minimal
2111
+ panels showing just the equation text.
2112
+
2113
+ If the container holds nothing reportable (no estimations and
2114
+ ``report_all=False``, or completely empty), no file is written
2115
+ and a short message is printed instead.
2116
+
2117
+ Parameters
2118
+ ----------
2119
+ path, filename, plot_format, title, open_file, report_all
2120
+ Forwarded to :func:`export_estimation_reports_to_html`. See
2121
+ that function for details.
2122
+ """
2123
+ if report_all:
2124
+ to_report = list(self.equations)
2125
+ if not to_report:
2126
+ print(
2127
+ "[EqContainer.estimation_report] Container is empty. "
2128
+ "Nothing to report."
2129
+ )
2130
+ return
2131
+ else:
2132
+ to_report = [eq for eq in self.equations
2133
+ if isinstance(eq, EstimatorBackend)]
2134
+ skipped = len(self.equations) - len(to_report)
2135
+ if not to_report:
2136
+ print(
2137
+ f"[EqContainer.estimation_report] No estimated "
2138
+ f"equations in container (all {skipped} entries are "
2139
+ "identities). Pass report_all=True to include them. "
2140
+ "Nothing to report."
2141
+ )
2142
+ return
2143
+ if skipped:
2144
+ print(
2145
+ f"[EqContainer.estimation_report] Including "
2146
+ f"{len(to_report)} estimated equation(s); skipping "
2147
+ f"{skipped} identity equation(s). Pass report_all=True "
2148
+ "to include identities too."
2149
+ )
2150
+ export_estimation_reports_to_html(
2151
+ equations=to_report,
2152
+ path=path,
2153
+ filename=filename,
2154
+ plot_format=plot_format,
2155
+ title=title,
2156
+ open_file=open_file,
2157
+ report_all=report_all,
2158
+ )
2159
+
2160
+ # -- container arithmetic -------------------------------------------------
2161
+
2162
+ def __add__(self, other) -> "EqContainer":
2163
+ if isinstance(other, EqContainer):
2164
+ return EqContainer(self.equations + other.equations)
2165
+ if isinstance(other, Eq_parent):
2166
+ return EqContainer(self.equations + [other])
2167
+ if isinstance(other, str):
2168
+ return EqContainer(self.equations + process_string_eq(other))
2169
+ raise TypeError(
2170
+ f"Cannot add object of type {type(other).__name__} to EqContainer."
2171
+ )
2172
+
2173
+ def __iadd__(self, other) -> "EqContainer":
2174
+ if isinstance(other, EqContainer):
2175
+ self.equations.extend(other.equations)
2176
+ elif isinstance(other, Eq_parent):
2177
+ self.equations.append(other)
2178
+ elif isinstance(other, str):
2179
+ self.equations.extend(process_string_eq(other))
2180
+ else:
2181
+ raise TypeError(
2182
+ f"Cannot add object of type {type(other).__name__} "
2183
+ "to EqContainer."
2184
+ )
2185
+ return self
2186
+
2187
+ def __radd__(self, other) -> "EqContainer":
2188
+ if isinstance(other, str):
2189
+ return EqContainer(process_string_eq(other) + self.equations)
2190
+ raise TypeError(
2191
+ f"Cannot add object of type {type(other).__name__} to EqContainer."
2192
+ )
2193
+
2194
+ def __repr__(self) -> str:
2195
+ return f"EqContainer({self.equations})"
2196
+
2197
+ def __iter__(self):
2198
+ return iter(self.equations)
2199
+
2200
+
2201
+ # =============================================================================
2202
+ # Plot helper, HTML export, EViews output renderer
2203
+ # =============================================================================
2204
+
2205
+ def plot_actual_vs_fitted(
2206
+ emodel,
2207
+ save_to=None,
2208
+ format: str = "png",
2209
+ figsize: Tuple[float, float] = (7, 3),
2210
+ dpi: int = 150,
2211
+ ):
2212
+ """Plot Actual vs Fitted (top panel) and Residuals (bottom panel).
2213
+
2214
+ If ``save_to`` is a file-like object, the plot is saved there and the
2215
+ figure is closed; otherwise it is shown interactively.
2216
+ """
2217
+ fig = plt.figure(figsize=figsize, dpi=dpi)
2218
+ gs = GridSpec(3, 1, height_ratios=[2, 0.1, 1])
2219
+
2220
+ ax1 = fig.add_subplot(gs[0])
2221
+ ax2 = fig.add_subplot(gs[2], sharex=ax1)
2222
+
2223
+ ax1.plot(emodel.af_df.index, emodel.af_df["Actual"],
2224
+ label="Actual", linewidth=2)
2225
+ ax1.plot(emodel.af_df.index, emodel.af_df["Fitted"],
2226
+ label="Fitted", linestyle="--")
2227
+ ax1.set_title(f"Actual vs Fitted: {emodel.endo_var}")
2228
+ ax1.set_ylabel(emodel.endo_var)
2229
+ ax1.grid(True)
2230
+ ax1.legend()
2231
+
2232
+ ax2.plot(emodel.residuals_df.index, emodel.residuals_df["Residuals"],
2233
+ color="gray")
2234
+ ax2.axhline(0, color="red", linestyle="--", linewidth=1)
2235
+ ax2.set_title("Residuals")
2236
+ ax2.set_xlabel("Time")
2237
+ ax2.set_ylabel("Residual")
2238
+ ax2.grid(True)
2239
+
2240
+ plt.tight_layout()
2241
+
2242
+ if save_to is not None:
2243
+ fig.savefig(save_to, format=format, bbox_inches="tight")
2244
+ plt.close(fig)
2245
+ else:
2246
+ plt.show()
2247
+
2248
+
2249
+ def simple_html_escape(text: str) -> str:
2250
+ """Minimal HTML escape used by :func:`eviews_output_to_html`."""
2251
+ return (text.replace("&", "&amp;")
2252
+ .replace("<", "&lt;")
2253
+ .replace(">", "&gt;"))
2254
+
2255
+
2256
+ def eviews_output_to_html(eviews_text: str) -> str:
2257
+ """Convert an EViews spool-text dump into a small HTML fragment."""
2258
+ lines = eviews_text.strip().splitlines()
2259
+
2260
+ html: List[str] = ['<div style="font-family:monospace;">']
2261
+ table_rows: List[List[str]] = []
2262
+ stats_rows: List[List[str]] = []
2263
+ in_table = False
2264
+ in_stats = False
2265
+ collecting_equation = False
2266
+ equation_lines: List[str] = []
2267
+
2268
+ for raw in lines:
2269
+ line = raw.strip()
2270
+
2271
+ if line.startswith("===="):
2272
+ html.append("<hr>")
2273
+ if collecting_equation:
2274
+ html.append(
2275
+ "<pre>"
2276
+ + simple_html_escape("\n".join(equation_lines))
2277
+ + "</pre>"
2278
+ )
2279
+ equation_lines = []
2280
+ collecting_equation = False
2281
+ continue
2282
+
2283
+ if re.search(r"=\s*-?\s*C\(\d+\)", line):
2284
+ collecting_equation = True
2285
+ equation_lines.append(line)
2286
+ continue
2287
+ elif collecting_equation and not re.match(r"^[A-Z]", line):
2288
+ equation_lines.append(line)
2289
+ continue
2290
+ elif collecting_equation:
2291
+ html.append(
2292
+ "<pre>"
2293
+ + simple_html_escape("\n".join(equation_lines))
2294
+ + "</pre>"
2295
+ )
2296
+ equation_lines = []
2297
+ collecting_equation = False
2298
+
2299
+ if line.lower().startswith("coefficientc"):
2300
+ in_table = True
2301
+ table_rows.append(
2302
+ ["Name", "Coefficient", "Std. Error", "t-Statistic", "Prob."]
2303
+ )
2304
+ continue
2305
+ elif in_table and re.match(r"^C\(\d+\)", line):
2306
+ table_rows.append(re.split(r"\s+", line))
2307
+ continue
2308
+ elif in_table and not line:
2309
+ in_table = False
2310
+ continue
2311
+
2312
+ if re.match(r"^R-squared", line) or re.match(r"^S\.E\.", line):
2313
+ in_stats = True
2314
+
2315
+ if in_stats:
2316
+ if ":" in line or " " in line:
2317
+ stats_rows.append(re.split(r"\s{2,}", line))
2318
+ continue
2319
+
2320
+ if line:
2321
+ html.append(f"<div>{simple_html_escape(line)}</div>")
2322
+
2323
+ if table_rows:
2324
+ html.append('<table border="1" cellpadding="4" cellspacing="0">')
2325
+ for row in table_rows:
2326
+ html.append(
2327
+ "<tr>"
2328
+ + "".join(f"<td>{simple_html_escape(c)}</td>" for c in row)
2329
+ + "</tr>"
2330
+ )
2331
+ html.append("</table>")
2332
+
2333
+ if stats_rows:
2334
+ html.append('<table border="1" cellpadding="4" cellspacing="0">')
2335
+ for row in stats_rows:
2336
+ html.append(
2337
+ "<tr>"
2338
+ + "".join(f"<td>{simple_html_escape(c)}</td>" for c in row)
2339
+ + "</tr>"
2340
+ )
2341
+ html.append("</table>")
2342
+
2343
+ html.append("</div>")
2344
+ return "\n".join(html)
2345
+
2346
+
2347
+ def export_estimation_reports_to_html(
2348
+ equations: List[Any],
2349
+ path: str = "html",
2350
+ filename: str = "estimation_report.html",
2351
+ plot_format: str = "svg",
2352
+ title: str = "Estimation Summary",
2353
+ open_file: bool = False,
2354
+ report_all: bool = False,
2355
+ ) -> None:
2356
+ """Export estimated equations into one interactive HTML document.
2357
+
2358
+ The generated page has a collapsible Table of Contents, expandable
2359
+ per-equation sections, and Expand/Collapse/Print/Download buttons.
2360
+
2361
+ Parameters
2362
+ ----------
2363
+ equations : list
2364
+ Each element is one equation. Estimator instances
2365
+ (:class:`EstimatorBackend` subclasses) and :class:`LSResult`
2366
+ objects render with the full estimation report (regression
2367
+ summary, A/F plot, etc.). :class:`Eq` identity instances render
2368
+ with a minimal panel (just the equation text) when
2369
+ ``report_all=True``; they are silently ignored when
2370
+ ``report_all=False`` (the default).
2371
+ path : str, default "html"
2372
+ Output directory (created if missing).
2373
+ filename : str, default "estimation_report.html"
2374
+ File name (written inside ``path``).
2375
+ plot_format : {"svg", "png"}, default "svg"
2376
+ Format for embedded plots.
2377
+ title : str, default "Estimation Summary"
2378
+ Page title.
2379
+ open_file : bool, default False
2380
+ If True, opens the report in the system's default browser.
2381
+ report_all : bool, default False
2382
+ If True, also include identity equations in the TOC and as
2383
+ expandable panels (with a minimal "equation only" body). If
2384
+ False, identities are filtered out so the report shows estimated
2385
+ equations only.
2386
+ """
2387
+ out_dir = Path(path)
2388
+ out_dir.mkdir(parents=True, exist_ok=True)
2389
+ full_path = out_dir / filename
2390
+
2391
+ # Filter out identities unless report_all is set.
2392
+ if not report_all:
2393
+ equations = [eq for eq in equations
2394
+ if not _is_identity_only(eq)]
2395
+
2396
+ html_parts: List[str] = [_REPORT_HEADER.format(title=title, filename=filename)]
2397
+
2398
+ # Table of contents
2399
+ for i, eq in enumerate(equations):
2400
+ anchor = f"eq_{i}"
2401
+ var_name = eq.endo_var
2402
+ desc = eq.omodel.var_description.get(var_name, "")
2403
+ html_parts.append(
2404
+ f"<li><a href='#{anchor}'>{eq.caption}: {var_name}: {desc}</a></li>"
2405
+ )
2406
+ html_parts.append("</ul></div>")
2407
+
2408
+ # Per-equation panels
2409
+ for i, eq in enumerate(equations):
2410
+ anchor = f"eq_{i}"
2411
+ var_name = eq.endo_var
2412
+ desc = eq.omodel.var_description.get(var_name, "")
2413
+ # Three render paths:
2414
+ # 1. Estimator backend → full report via .mfresult
2415
+ # 2. LSResult → full report via .get_html_report
2416
+ # 3. Anything else (Eq identity) → minimal body, equation only
2417
+ if hasattr(eq, "mfresult"):
2418
+ body = eq.mfresult.get_html_report(plot_format=plot_format)
2419
+ elif hasattr(eq, "get_html_report"):
2420
+ body = eq.get_html_report(plot_format=plot_format)
2421
+ else:
2422
+ body = _identity_html_body(eq)
2423
+ html_parts.append(
2424
+ f'<button class="accordion" id="{anchor}">'
2425
+ f"{eq.caption}: {var_name}: {desc}</button>"
2426
+ f'<div class="panel">{body}'
2427
+ f'<br><a class="back-to-top" href="#top">⬆ Back to Top</a>'
2428
+ f"</div>"
2429
+ )
2430
+
2431
+ html_parts.append("</body></html>")
2432
+ full_path.write_text("\n".join(html_parts), encoding="utf-8")
2433
+ print(f"✔ Report saved to {full_path}")
2434
+
2435
+ if open_file:
2436
+ try:
2437
+ import google.colab # noqa: F401
2438
+ from IPython.display import HTML, display
2439
+ display(HTML(full_path.read_text(encoding="utf-8")))
2440
+ except ImportError:
2441
+ webbrowser.open(f"file://{full_path.resolve()}")
2442
+
2443
+
2444
+ def _is_identity_only(eq: Any) -> bool:
2445
+ """True if ``eq`` has no fit / regression result to render — i.e. a
2446
+ plain identity equation rather than an estimator or LSResult."""
2447
+ return (not hasattr(eq, "mfresult")
2448
+ and not hasattr(eq, "get_html_report"))
2449
+
2450
+
2451
+ def _identity_html_body(eq: Any) -> str:
2452
+ """Minimal panel body for an identity equation: just the equation
2453
+ text in a fixed-width block, no regression summary or plot."""
2454
+ return (
2455
+ f"<h3>Identity equation</h3>"
2456
+ f"<p><strong>Endogenous variable:</strong> {eq.endo_var}</p>"
2457
+ f"<p><strong>Equation:</strong></p>"
2458
+ f"<pre><code>{eq.org_eq}</code></pre>"
2459
+ )
2460
+
2461
+
2462
+ _REPORT_HEADER = """\
2463
+ <html>
2464
+ <head>
2465
+ <meta charset='utf-8'>
2466
+ <title>{title}</title>
2467
+ <style>
2468
+ body {{ font-family: Arial, sans-serif; margin: 40px; }}
2469
+ h1 {{ border-bottom: 2px solid #ccc; }}
2470
+ .toc {{ margin-bottom: 30px; border: 1px solid #ccc; background: #f9f9f9; padding: 10px; }}
2471
+ .toc h2 {{ margin-top: 0; }}
2472
+ .toc ul {{ list-style: none; padding-left: 0; }}
2473
+ .toc li {{ margin: 5px 0; }}
2474
+ .toggle-btn, .print-btn {{
2475
+ margin: 10px 10px 20px 0; background: #555; color: white;
2476
+ border: none; padding: 8px 12px; cursor: pointer; border-radius: 4px;
2477
+ }}
2478
+ .accordion {{
2479
+ background-color: #eee; color: #444; cursor: pointer;
2480
+ padding: 15px; width: 100%; border: none; text-align: left;
2481
+ outline: none; font-size: 16px; transition: 0.3s; margin-top: 20px;
2482
+ border-radius: 4px;
2483
+ }}
2484
+ .active, .accordion:hover {{ background-color: #ccc; }}
2485
+ .panel {{
2486
+ padding: 0 20px; display: none; background-color: white;
2487
+ overflow: hidden; border-left: 2px solid #ccc;
2488
+ border-right: 2px solid #ccc; border-bottom: 2px solid #ccc;
2489
+ border-radius: 0 0 6px 6px;
2490
+ }}
2491
+ .back-to-top {{
2492
+ margin-top: 20px; display: inline-block; background: #007BFF;
2493
+ color: white; padding: 6px 12px; border-radius: 4px;
2494
+ text-decoration: none;
2495
+ }}
2496
+ img {{ max-width: 100%; height: auto; }}
2497
+ @media print {{
2498
+ .toggle-btn, .print-btn, .back-to-top, .accordion {{ display: none !important; }}
2499
+ .panel {{ display: block !important; }}
2500
+ }}
2501
+ </style>
2502
+ <script>
2503
+ function toggleTOC() {{
2504
+ const toc = document.getElementById("toc-content");
2505
+ toc.style.display = (toc.style.display === "none") ? "block" : "none";
2506
+ }}
2507
+ function expandAll() {{
2508
+ const acc = document.getElementsByClassName("accordion");
2509
+ for (let i = 0; i < acc.length; i++) {{
2510
+ acc[i].classList.add("active");
2511
+ acc[i].nextElementSibling.style.display = "block";
2512
+ }}
2513
+ }}
2514
+ function collapseAll() {{
2515
+ const acc = document.getElementsByClassName("accordion");
2516
+ for (let i = 0; i < acc.length; i++) {{
2517
+ acc[i].classList.remove("active");
2518
+ acc[i].nextElementSibling.style.display = "none";
2519
+ }}
2520
+ }}
2521
+ function printAll() {{ expandAll(); setTimeout(() => window.print(), 200); }}
2522
+ function downloadHTML() {{
2523
+ const blob = new Blob([document.documentElement.outerHTML], {{ type: 'text/html' }});
2524
+ const url = URL.createObjectURL(blob);
2525
+ const a = document.createElement('a');
2526
+ a.href = url; a.download = '{filename}'; a.click();
2527
+ URL.revokeObjectURL(url);
2528
+ }}
2529
+ document.addEventListener("DOMContentLoaded", function() {{
2530
+ const acc = document.getElementsByClassName("accordion");
2531
+ for (let i = 0; i < acc.length; i++) {{
2532
+ acc[i].addEventListener("click", function() {{
2533
+ this.classList.toggle("active");
2534
+ const panel = this.nextElementSibling;
2535
+ panel.style.display = (panel.style.display === "block") ? "none" : "block";
2536
+ }});
2537
+ }}
2538
+
2539
+ // Force-open the panel for an anchor (e.g. #eq_3). Used when
2540
+ // the user clicks a TOC link or arrives via a deep link.
2541
+ // We FORCE OPEN rather than toggle, so a click on an already-
2542
+ // open panel's TOC entry doesn't accidentally collapse it.
2543
+ function openPanelForHash(hash) {{
2544
+ if (!hash || hash === "#top") return;
2545
+ const target = document.querySelector(hash);
2546
+ if (!target || !target.classList.contains("accordion")) return;
2547
+ target.classList.add("active");
2548
+ const panel = target.nextElementSibling;
2549
+ if (panel) panel.style.display = "block";
2550
+ // Re-scroll after expansion: the initial scroll happened
2551
+ // while the panel was still hidden, so the position is now
2552
+ // off. A small delay lets the layout reflow first.
2553
+ setTimeout(() => target.scrollIntoView({{behavior: "smooth", block: "start"}}), 50);
2554
+ }}
2555
+
2556
+ // Run once for the initial URL hash (deep links).
2557
+ openPanelForHash(window.location.hash);
2558
+ // And on every subsequent navigation (TOC clicks).
2559
+ window.addEventListener("hashchange",
2560
+ () => openPanelForHash(window.location.hash));
2561
+ }});
2562
+ </script>
2563
+ </head>
2564
+ <body id="top">
2565
+ <h1>{title}</h1>
2566
+ <button class="toggle-btn" onclick="toggleTOC()">Toggle Table of Contents</button>
2567
+ <button class="toggle-btn" onclick="expandAll()">Expand All</button>
2568
+ <button class="toggle-btn" onclick="collapseAll()">Collapse All</button>
2569
+ <button class="print-btn" onclick="printAll()">📄 Print All</button>
2570
+ <button class="print-btn" onclick="downloadHTML()">💾 Download</button>
2571
+ <div class="toc" id="toc-content">
2572
+ <h2>Table of Contents</h2>
2573
+ <ul>
2574
+ """
2575
+
2576
+
2577
+ # =============================================================================
2578
+ # Quick smoke test (only runs as __main__)
2579
+ # =============================================================================
2580
+
2581
+ if __name__ == "__main__":
2582
+ eviews_eq = """
2583
+ DLOG(BOLNECONPRVTKN) = - C(2) * (
2584
+ LOG(BOLNECONPRVTKN(-1)) - LOG((BOLNYYWBTOTLCN(-1) - BOLGGREVDRCTCN(-1) + BOLBXFSTREMTCD(-1) * BOLPANUSATLS(-1)) / BOLNECONPRVTXN(-1))
2585
+ ) + C(10) * DLOG((BOLNYYWBTOTLCN - BOLGGREVDRCTCN + BOLBXFSTREMTCD * BOLPANUSATLS) / BOLNECONPRVTXN)
2586
+ + C(11) * (BOLFMLBLLRLCFR / 100 - DLOG(BOLNECONPRVTKN))
2587
+ """
2588
+ ols_eq = "BOLNECONPRVTKN = C(1) + C(2) * BOLNYYWBTOTLCN + c__3 * BOLGGREVDRCTCN"
2589
+
2590
+ mbol, df = model.modelload("bol")
2591
+
2592
+ nls = Estimate_nls_lmfit(
2593
+ org_eq=eviews_eq, input_df=df, smpl=(2002, 2023),
2594
+ omodel=mbol, fit_kws={},
2595
+ )
2596
+ print("lmfit:", nls.coef_estimate_dict)
2597
+
2598
+ # Backwards-compatible call style still works:
2599
+ nls2 = Estimate_nls(
2600
+ eviews_eq, input_df=df, smpl=(2002, 2023),
2601
+ omodel=mbol, fit_kws={}, solver="eviews",
2602
+ )
2603
+ print("eviews:", nls2.coef_estimate_dict)
2604
+
2605
+ ols = Estimate_ols(org_eq=ols_eq, input_df=df)
2606
+ print("ols:", ols.coef_estimate_dict)
2607
+
2608
+ # Container arithmetic
2609
+ eq1 = Eq("a = b", input_df=df)
2610
+ eq1 + eq1
2611
+ multi = Eq("a = b\nc = d", input_df=df, frml_name="ib")
2612
+ container = multi + EqContainer([nls])
2613
+ container.test()