drtran 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- drtran/__init__.py +91 -0
- drtran/__main__.py +12 -0
- drtran/aggregate.py +169 -0
- drtran/calibrate.py +299 -0
- drtran/cast.py +377 -0
- drtran/cli.py +815 -0
- drtran/diagnose.py +203 -0
- drtran/embed.py +270 -0
- drtran/estimate.py +414 -0
- drtran/evaluate.py +204 -0
- drtran/forecast.py +581 -0
- drtran/identify.py +342 -0
- drtran/irf.py +209 -0
- drtran/mcp_server.py +815 -0
- drtran/netid.py +321 -0
- drtran/network.py +141 -0
- drtran/plots.py +210 -0
- drtran/pre.py +198 -0
- drtran/report.py +137 -0
- drtran/slots.py +439 -0
- drtran-0.1.0.dist-info/METADATA +229 -0
- drtran-0.1.0.dist-info/RECORD +25 -0
- drtran-0.1.0.dist-info/WHEEL +5 -0
- drtran-0.1.0.dist-info/entry_points.txt +3 -0
- drtran-0.1.0.dist-info/top_level.txt +1 -0
drtran/__init__.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""drtran — Box-Jenkins transfer function models by exact maximum likelihood.
|
|
2
|
+
|
|
3
|
+
The bridge between two programs that already work:
|
|
4
|
+
|
|
5
|
+
- **fue** identifies and estimates **univariate** models and leaves one `.pre`
|
|
6
|
+
per series.
|
|
7
|
+
- **drvarma** evaluates Mauricio's **exact VARMA likelihood** (`elf`) and
|
|
8
|
+
maximises it with factored BFGS.
|
|
9
|
+
|
|
10
|
+
drtran reads the `.pre` files, builds the parametric *cast* (parameters -> VARMA
|
|
11
|
+
structure) and lets drvarma estimate **every parameter at once**: the transfer,
|
|
12
|
+
both ARMA, the deterministics, the means and the variances.
|
|
13
|
+
|
|
14
|
+
Y_t = SUM_j omega_j(B)/delta_j(B) * B^b_j * X_j,t + N_t
|
|
15
|
+
|
|
16
|
+
Design principle, non-negotiable
|
|
17
|
+
--------------------------------
|
|
18
|
+
drvarma's `elf` is used **as it is**: not modified, not patched, not
|
|
19
|
+
special-cased. It is the reference implementation of the exact likelihood. Any
|
|
20
|
+
discrepancy with fue is a bug of drtran's cast, **never** of `elf`.
|
|
21
|
+
|
|
22
|
+
Validation criterion (the gate to everything else)
|
|
23
|
+
--------------------------------------------------
|
|
24
|
+
**Diagonal joint estimation == fue run separately.** With a diagonal structure
|
|
25
|
+
(diagonal AR/MA and covariance, no transfer) the exact likelihood factorises, so
|
|
26
|
+
the joint fit must reproduce the **sum** of the univariate ones. If it does not
|
|
27
|
+
match, the cast is wrong.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from .aggregate import (Aggregate, forecast_aggregates, read_aggregates,
|
|
31
|
+
report_aggregates)
|
|
32
|
+
from .cast import (Link, build_cast_spec, build_sigma, cast_diagonal,
|
|
33
|
+
compute_irf, x0_from_pre)
|
|
34
|
+
from .diagnose import (Adequacy, chi_test, report_adequacy,
|
|
35
|
+
transfer_adequacy)
|
|
36
|
+
from .embed import cast_embedded, loglik_embedded, normalize_phi0
|
|
37
|
+
from .estimate import Fit, StdErrors, fit, loglik, standard_errors, unpack, x0_full
|
|
38
|
+
from .evaluate import (RollingEval, fixed_window_fit, report_rolling,
|
|
39
|
+
rolling_evaluation, truncate, write_rolling_csv)
|
|
40
|
+
from .forecast import (Forecast, error_variance, forecast,
|
|
41
|
+
forecast_mean, integrated_weights, psi_weights,
|
|
42
|
+
level_band, report_forecast, to_level,
|
|
43
|
+
variance_decomposition)
|
|
44
|
+
from .identify import (Identification, identify, prewhiten,
|
|
45
|
+
report_identification)
|
|
46
|
+
from .irf import ImpulseResponse, impulse_response, report_irf
|
|
47
|
+
from .netid import (Candidate, IdentifiedNetwork, identify_network,
|
|
48
|
+
report_network, residuals, write_guided)
|
|
49
|
+
from .network import check_acyclic, find_cycle, read_dag, write_dag
|
|
50
|
+
from .plots import plot_ccf, plot_forecast, plot_irf, prewhitened_pair, save
|
|
51
|
+
from .pre import PreSpec, check_scale, load_pre, next_pre_path, write_pre
|
|
52
|
+
from .report import build_forecast_result, write_forecast_report
|
|
53
|
+
from .slots import Slot, SlotTable, build_slots, read_cns
|
|
54
|
+
|
|
55
|
+
# Read from the installed metadata rather than repeated here: this line said
|
|
56
|
+
# "0.1.0b1" while pyproject.toml said "0.1.0", so `drtran.__version__` reported a
|
|
57
|
+
# version that had not existed for some time. Two places to change is one place
|
|
58
|
+
# too many, and the one that drifts is always the one nobody builds from.
|
|
59
|
+
try:
|
|
60
|
+
from importlib.metadata import PackageNotFoundError, version as _pkg_version
|
|
61
|
+
try:
|
|
62
|
+
__version__ = _pkg_version("drtran")
|
|
63
|
+
except PackageNotFoundError: # running from a source tree
|
|
64
|
+
__version__ = "0.0.0.dev0"
|
|
65
|
+
except ImportError: # pragma: no cover
|
|
66
|
+
__version__ = "0.0.0.dev0"
|
|
67
|
+
|
|
68
|
+
__all__ = [
|
|
69
|
+
"PreSpec", "load_pre", "check_scale", "write_pre", "next_pre_path",
|
|
70
|
+
"plot_ccf", "plot_irf", "plot_forecast", "prewhitened_pair", "save",
|
|
71
|
+
"Aggregate", "read_aggregates", "forecast_aggregates", "report_aggregates",
|
|
72
|
+
"write_forecast_report", "build_forecast_result",
|
|
73
|
+
"Link", "build_cast_spec", "build_sigma", "cast_diagonal", "compute_irf",
|
|
74
|
+
"x0_from_pre",
|
|
75
|
+
"cast_embedded", "loglik_embedded", "normalize_phi0",
|
|
76
|
+
"Adequacy", "transfer_adequacy", "report_adequacy", "chi_test",
|
|
77
|
+
"Fit", "fit", "loglik", "unpack", "x0_full",
|
|
78
|
+
"StdErrors", "standard_errors",
|
|
79
|
+
"RollingEval", "rolling_evaluation", "fixed_window_fit", "truncate",
|
|
80
|
+
"report_rolling", "write_rolling_csv",
|
|
81
|
+
"Identification", "identify", "prewhiten", "report_identification",
|
|
82
|
+
"ImpulseResponse", "impulse_response", "report_irf",
|
|
83
|
+
"Forecast", "forecast", "report_forecast", "level_band", "psi_weights",
|
|
84
|
+
"error_variance", "integrated_weights", "forecast_mean", "to_level",
|
|
85
|
+
"variance_decomposition",
|
|
86
|
+
"Candidate", "IdentifiedNetwork", "identify_network", "report_network",
|
|
87
|
+
"residuals", "write_guided",
|
|
88
|
+
"read_dag", "write_dag", "find_cycle", "check_acyclic",
|
|
89
|
+
"Slot", "SlotTable", "build_slots", "read_cns",
|
|
90
|
+
"__version__",
|
|
91
|
+
]
|
drtran/__main__.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""`python -m drtran` — the same entry point as the `drtran-py` script.
|
|
2
|
+
|
|
3
|
+
Deliberately NOT installed as `drtran`: that name belongs to the C binary on a
|
|
4
|
+
machine that has both, and shadowing it silently is how a battery starts
|
|
5
|
+
comparing a program against itself.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from .cli import main
|
|
11
|
+
|
|
12
|
+
sys.exit(main())
|
drtran/aggregate.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Accounting identities, computed AFTER forecasting — and their band.
|
|
2
|
+
|
|
3
|
+
Port of the `-a` block of `drtran.c`. An identity like
|
|
4
|
+
|
|
5
|
+
OCUPADOS = + EA + EP + EI + EU + EC
|
|
6
|
+
PARADOS = + ACTIVOS - EA - EP - EI - EU - EC
|
|
7
|
+
|
|
8
|
+
does **not** belong in the model: it is arithmetic on the answer, not a
|
|
9
|
+
restriction on the parameters, and the legacy computed it after forecasting for
|
|
10
|
+
exactly that reason. Putting it in would add a series that is a linear
|
|
11
|
+
combination of the others, and the likelihood would be singular.
|
|
12
|
+
|
|
13
|
+
What is not trivial is the **band**. The series' forecast errors are CORRELATED —
|
|
14
|
+
they share innovations through the network — so the variance of the aggregate is
|
|
15
|
+
NOT the sum of the variances. It is `c' V c` with V the full forecast-error
|
|
16
|
+
covariance. On a system where the sectors move together, treating them as
|
|
17
|
+
independent understates the band badly, and in the direction that flatters the
|
|
18
|
+
model.
|
|
19
|
+
|
|
20
|
+
There is a second wrinkle. The series are modelled TRANSFORMED (log, Box-Cox)
|
|
21
|
+
and the identity lives in LEVELS, so the covariance has to be carried across by
|
|
22
|
+
the delta method::
|
|
23
|
+
|
|
24
|
+
dz/db = level / refactor (lambda = 0)
|
|
25
|
+
dz/db = (lambda*b/refactor + 1)^(1/lambda - 1) / refactor (otherwise)
|
|
26
|
+
|
|
27
|
+
and the variance is `sum_i sum_j c_i c_j J_i J_j V_ij(l)`.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import math
|
|
33
|
+
from dataclasses import dataclass, field
|
|
34
|
+
|
|
35
|
+
import numpy as np
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class Aggregate:
|
|
40
|
+
"""A named linear combination of the series."""
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
coef: np.ndarray # (m,) the c vector, mostly zeros
|
|
44
|
+
|
|
45
|
+
def terms(self, names):
|
|
46
|
+
out = []
|
|
47
|
+
for i, c in enumerate(self.coef):
|
|
48
|
+
if c:
|
|
49
|
+
out.append(f"{'+' if c > 0 else '-'} {names[i]}")
|
|
50
|
+
return " ".join(out)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class AggregateForecast:
|
|
55
|
+
"""The aggregate's path with its band."""
|
|
56
|
+
|
|
57
|
+
name: str
|
|
58
|
+
level: np.ndarray = None
|
|
59
|
+
sd: np.ndarray = None
|
|
60
|
+
lower: np.ndarray = None
|
|
61
|
+
upper: np.ndarray = None
|
|
62
|
+
terms: str = ""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def read_aggregates(path, names):
|
|
66
|
+
"""Read an aggregates file: `NAME = +A -B`, one per line, `#` for comments.
|
|
67
|
+
|
|
68
|
+
Signs may be attached (`+EA`) or separate (`+ EA`), as in the C. An unknown
|
|
69
|
+
series name is an error rather than a silently dropped term — a dropped term
|
|
70
|
+
turns an identity into a different identity that still adds up.
|
|
71
|
+
"""
|
|
72
|
+
names = list(names)
|
|
73
|
+
out = []
|
|
74
|
+
with open(path) as f:
|
|
75
|
+
for nline, line in enumerate(f, 1):
|
|
76
|
+
line = line.split("#", 1)[0].strip()
|
|
77
|
+
if not line:
|
|
78
|
+
continue
|
|
79
|
+
if "=" not in line:
|
|
80
|
+
raise ValueError(
|
|
81
|
+
f"{path}:{nline}: expected NAME = +A -B: {line!r}")
|
|
82
|
+
lhs, rhs = line.split("=", 1)
|
|
83
|
+
name = lhs.strip()
|
|
84
|
+
if not name:
|
|
85
|
+
raise ValueError(f"{path}:{nline}: aggregate with no name")
|
|
86
|
+
c = np.zeros(len(names))
|
|
87
|
+
sign = 1.0
|
|
88
|
+
for tok in rhs.split():
|
|
89
|
+
if tok == "+":
|
|
90
|
+
sign = 1.0
|
|
91
|
+
continue
|
|
92
|
+
if tok == "-":
|
|
93
|
+
sign = -1.0
|
|
94
|
+
continue
|
|
95
|
+
if tok[0] == "+":
|
|
96
|
+
sign, tok = 1.0, tok[1:]
|
|
97
|
+
elif tok[0] == "-":
|
|
98
|
+
sign, tok = -1.0, tok[1:]
|
|
99
|
+
if not tok:
|
|
100
|
+
continue
|
|
101
|
+
if tok not in names:
|
|
102
|
+
raise ValueError(f"{path}:{nline}: unknown series {tok!r}; "
|
|
103
|
+
f"the ones loaded are {names}")
|
|
104
|
+
c[names.index(tok)] += sign
|
|
105
|
+
sign = 1.0
|
|
106
|
+
if not np.any(c):
|
|
107
|
+
raise ValueError(f"{path}:{nline}: {name!r} has no terms")
|
|
108
|
+
out.append(Aggregate(name=name, coef=c))
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _jacobian(level, model):
|
|
113
|
+
"""`dz/db` at the forecast point: the delta-method factor to the level."""
|
|
114
|
+
lam = float(getattr(model, "boxlam", 0.0) or 0.0)
|
|
115
|
+
refc = float(getattr(model, "refactor", 1.0) or 1.0)
|
|
116
|
+
if abs(lam) < 1e-8:
|
|
117
|
+
return np.asarray(level, float) / refc
|
|
118
|
+
# level = base^(1/lam), so dlevel/db = base^(1/lam - 1)/refactor
|
|
119
|
+
return np.power(np.asarray(level, float), 1.0 - lam) / refc
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def forecast_aggregates(fc, cast_spec, aggregates, origin=None, z=1.96):
|
|
123
|
+
"""The aggregates' paths and bands, `c' J V J c` per horizon."""
|
|
124
|
+
from .forecast import to_level
|
|
125
|
+
|
|
126
|
+
m = cast_spec.m
|
|
127
|
+
L = fc.L
|
|
128
|
+
levels = np.zeros((L, m))
|
|
129
|
+
jac = np.zeros((L, m))
|
|
130
|
+
for i in range(m):
|
|
131
|
+
lv = to_level(fc, cast_spec, series=i, origin=origin)
|
|
132
|
+
levels[:, i] = lv
|
|
133
|
+
jac[:, i] = _jacobian(lv, cast_spec.series[i].spec.model)
|
|
134
|
+
|
|
135
|
+
names = list(cast_spec.names)
|
|
136
|
+
out = []
|
|
137
|
+
for ag in aggregates:
|
|
138
|
+
c = np.asarray(ag.coef, float)
|
|
139
|
+
pnt = levels @ c
|
|
140
|
+
sd = np.zeros(L)
|
|
141
|
+
for l in range(1, L + 1):
|
|
142
|
+
V = np.asarray(fc.var_level[l], float)
|
|
143
|
+
g = c * jac[l - 1] # the delta-method vector
|
|
144
|
+
var = float(g @ V @ g)
|
|
145
|
+
sd[l - 1] = math.sqrt(var) if var > 0 else 0.0
|
|
146
|
+
out.append(AggregateForecast(name=ag.name, level=pnt, sd=sd,
|
|
147
|
+
lower=pnt - z * sd, upper=pnt + z * sd,
|
|
148
|
+
terms=ag.terms(names)))
|
|
149
|
+
return out
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def report_aggregates(aggs):
|
|
153
|
+
"""The aggregates' tables, in the C's layout."""
|
|
154
|
+
L = ["=" * 62,
|
|
155
|
+
" AGGREGATES (accounting identities)",
|
|
156
|
+
"=" * 62]
|
|
157
|
+
for a in aggs:
|
|
158
|
+
L += ["",
|
|
159
|
+
f" Aggregate: {a.name} = {a.terms}",
|
|
160
|
+
" Computed AFTER forecasting; its band is c'Vc, so it accounts",
|
|
161
|
+
" for the correlation between the series' forecast errors.",
|
|
162
|
+
"",
|
|
163
|
+
" l LEVEL sd lower upper",
|
|
164
|
+
" " + "-" * 56]
|
|
165
|
+
for l in range(len(a.level)):
|
|
166
|
+
L.append(f" {l + 1:3d} {a.level[l]:11.4f} {a.sd[l]:8.4f} "
|
|
167
|
+
f"{a.lower[l]:11.4f} {a.upper[l]:11.4f}")
|
|
168
|
+
L.append("=" * 62)
|
|
169
|
+
return "\n".join(L)
|
drtran/calibrate.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""Which observations are bending the instruments.
|
|
2
|
+
|
|
3
|
+
ART's `diagnose_interventions` does more than flag extreme residuals: it measures
|
|
4
|
+
**how much each one distorts the ACF/PACF, the JB and the Q**. That is the idea
|
|
5
|
+
worth carrying over, because a transfer model is identified and judged with
|
|
6
|
+
instruments too — the CCF and two portmanteaus — and they bend the same way.
|
|
7
|
+
|
|
8
|
+
The instruments differ, so the questions do:
|
|
9
|
+
|
|
10
|
+
=================== =========================================================
|
|
11
|
+
ART which observation distorts the **ACF/PACF** and the Q?
|
|
12
|
+
mtram which observation distorts the **CCF** I read (b, r, s)
|
|
13
|
+
from, and the **adequacy portmanteau** I judge with?
|
|
14
|
+
=================== =========================================================
|
|
15
|
+
|
|
16
|
+
**Why this is not ART's scan run again.** ART calibrates on the *univariate*
|
|
17
|
+
rung, and its interventions arrive here already in the `.pre` as deterministics.
|
|
18
|
+
Re-running it would be worse than redundant: an anomaly in the output's
|
|
19
|
+
univariate residuals may be **explained by the input** once the transfer is in
|
|
20
|
+
the model, so carrying it in as an intervention attributes to a dummy what the
|
|
21
|
+
transfer explains — double counting, and it biases omega toward zero. What
|
|
22
|
+
survives the JOINT fit is the genuine intervention.
|
|
23
|
+
|
|
24
|
+
**The method is leave-one-out, not a contribution heuristic.** Recomputing the
|
|
25
|
+
CCF and the portmanteau without observation *t* answers the question the analyst
|
|
26
|
+
actually has — *would my conclusion change?* — instead of a proxy for it. On a
|
|
27
|
+
215-observation sample it costs nothing.
|
|
28
|
+
|
|
29
|
+
Which matters most at node **N6**. When adequacy fails there are two causes and
|
|
30
|
+
they need opposite responses:
|
|
31
|
+
|
|
32
|
+
* the **shape** is wrong → re-identify (b, r, s);
|
|
33
|
+
* **one observation** is → an intervention, and re-specifying the shape around
|
|
34
|
+
it is how a model acquires a lag nobody can interpret.
|
|
35
|
+
|
|
36
|
+
Telling them apart is what this module is for.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import math
|
|
42
|
+
from dataclasses import dataclass, field
|
|
43
|
+
|
|
44
|
+
import numpy as np
|
|
45
|
+
|
|
46
|
+
from .diagnose import _p_chi, chi_test
|
|
47
|
+
from .identify import ccf
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class Anomaly:
|
|
52
|
+
"""One observation, and what it does to the instruments."""
|
|
53
|
+
|
|
54
|
+
index: int # 0-based, into the residual array
|
|
55
|
+
obs: int # 1-based observation number in the series
|
|
56
|
+
date: str = ""
|
|
57
|
+
z: float = 0.0 # standardised structural residual
|
|
58
|
+
variance_fraction: float = 0.0 # z^2 / sum z^2
|
|
59
|
+
p_transfer_without: float = float("nan") # adequacy p, leaving it out
|
|
60
|
+
ccf_lags_affected: list = field(default_factory=list)
|
|
61
|
+
# THE GLOBAL EFFECT, which is the primary one: by inflating the variance an
|
|
62
|
+
# anomaly divides EVERY coefficient, not only the lags it touches.
|
|
63
|
+
compression: float = 1.0 # mean|r(k)| without it / with it
|
|
64
|
+
ccf_max: float = 0.0 # max|r(k)| with it
|
|
65
|
+
ccf_max_without: float = 0.0 # and without
|
|
66
|
+
ccf_without: object = None # the whole CCF without it, to plot
|
|
67
|
+
|
|
68
|
+
def __repr__(self): # pragma: no cover
|
|
69
|
+
return (f"Anomaly({self.date or self.obs}, z={self.z:+.2f}, "
|
|
70
|
+
f"var={100*self.variance_fraction:.1f}%)")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class Calibration:
|
|
75
|
+
"""What the leave-one-out sweep found."""
|
|
76
|
+
|
|
77
|
+
p_transfer: float = float("nan") # with every observation in
|
|
78
|
+
p_exog: float = float("nan")
|
|
79
|
+
n: int = 0
|
|
80
|
+
threshold: float = 3.5
|
|
81
|
+
anomalies: list = field(default_factory=list)
|
|
82
|
+
decisive: list = field(default_factory=list) # those that flip the verdict
|
|
83
|
+
names: tuple = ("X", "Y")
|
|
84
|
+
ccf: object = None # the CCF with every observation in
|
|
85
|
+
|
|
86
|
+
@property
|
|
87
|
+
def verdict(self):
|
|
88
|
+
"""`"shape"`, `"observation"` or `"adequate"` — the N6 branch to take."""
|
|
89
|
+
if self.p_transfer >= 0.05:
|
|
90
|
+
return "adequate"
|
|
91
|
+
return "observation" if self.decisive else "shape"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _date_of(model, obs):
|
|
95
|
+
try:
|
|
96
|
+
y, p = model.series._obs_to_date(obs)
|
|
97
|
+
except Exception: # pragma: no cover
|
|
98
|
+
return str(obs)
|
|
99
|
+
freq = int(getattr(model.series, "freq", 1) or 1)
|
|
100
|
+
return f"{p:02d}/{y}" if freq > 1 else str(y)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def calibrate(fit, link_index=0, threshold=3.5, nlags=None,
|
|
104
|
+
contrib_threshold=0.05):
|
|
105
|
+
"""Leave-one-out calibration of the CCF and the adequacy portmanteau.
|
|
106
|
+
|
|
107
|
+
Returns a `Calibration`. An observation is **decisive** when dropping it
|
|
108
|
+
moves the adequacy p-value across 0.05 — i.e. when the verdict on the
|
|
109
|
+
transfer rests on that single point.
|
|
110
|
+
"""
|
|
111
|
+
from .netid import residuals
|
|
112
|
+
|
|
113
|
+
cs = fit.cast_spec
|
|
114
|
+
link = cs.links[link_index]
|
|
115
|
+
a, ifa = residuals(fit.x, cs, embed=fit.embed, structural=True)
|
|
116
|
+
if ifa:
|
|
117
|
+
raise RuntimeError(f"cannot obtain the residuals: ifault={ifa}")
|
|
118
|
+
|
|
119
|
+
n = a.shape[0]
|
|
120
|
+
if nlags is None:
|
|
121
|
+
nlags = max(10, min(n // 4, 24))
|
|
122
|
+
npar_tr = (link.s + 1) + link.r
|
|
123
|
+
|
|
124
|
+
aX = np.asarray(a[:, link.inp], float)
|
|
125
|
+
aN = np.asarray(a[:, link.out], float)
|
|
126
|
+
|
|
127
|
+
def _p_adequacy(x_, n_):
|
|
128
|
+
cpos = ccf(x_, n_, nlags)
|
|
129
|
+
Qt, ncorr = chi_test(cpos, len(x_), first=0)
|
|
130
|
+
return _p_chi(Qt, max(1, ncorr - npar_tr)), cpos
|
|
131
|
+
|
|
132
|
+
p_full, ccf_full = _p_adequacy(aX, aN)
|
|
133
|
+
cneg = ccf(aN, aX, nlags)
|
|
134
|
+
Qe, dfe = chi_test(cneg, n, first=1)
|
|
135
|
+
p_exog = _p_chi(Qe, dfe)
|
|
136
|
+
|
|
137
|
+
# standardise the OUTPUT's innovation: that is where an intervention shows
|
|
138
|
+
z = aN - aN.mean()
|
|
139
|
+
sd = z.std()
|
|
140
|
+
z = z / sd if sd > 1e-15 else z
|
|
141
|
+
zz = float(np.sum(z * z)) or 1.0
|
|
142
|
+
|
|
143
|
+
model = cs.series[link.out].spec.model
|
|
144
|
+
lost = int(getattr(model.series, "nobs", n)) - n
|
|
145
|
+
rng = float(np.max(np.abs(ccf_full))) or 1.0
|
|
146
|
+
|
|
147
|
+
out, decisive = [], []
|
|
148
|
+
for t in range(n):
|
|
149
|
+
if abs(z[t]) < threshold:
|
|
150
|
+
continue
|
|
151
|
+
keep = np.ones(n, bool)
|
|
152
|
+
keep[t] = False
|
|
153
|
+
p_wo, _c = _p_adequacy(aX[keep], aN[keep])
|
|
154
|
+
|
|
155
|
+
lags = []
|
|
156
|
+
for k in range(nlags + 1):
|
|
157
|
+
if t + k < n:
|
|
158
|
+
c = abs(aX[t] * aN[t + k]) / (n * aX.std() * aN.std() + 1e-30)
|
|
159
|
+
if c > contrib_threshold * rng:
|
|
160
|
+
lags.append(k)
|
|
161
|
+
|
|
162
|
+
# The GLOBAL effect. An anomaly inflates the residual variance, which is
|
|
163
|
+
# the DIVISOR of every correlation, so it shrinks all of them at once --
|
|
164
|
+
# not just the lags it happens to touch. In the school's teaching this is
|
|
165
|
+
# the thing to VERIFY, and it is easy to see: take the point out and the
|
|
166
|
+
# coefficients come back.
|
|
167
|
+
m_with = float(np.mean(np.abs(ccf_full)))
|
|
168
|
+
m_wo = float(np.mean(np.abs(_c)))
|
|
169
|
+
an = Anomaly(index=t, obs=lost + t + 1,
|
|
170
|
+
date=_date_of(model, lost + t + 1),
|
|
171
|
+
z=float(z[t]), variance_fraction=float(z[t] ** 2 / zz),
|
|
172
|
+
p_transfer_without=float(p_wo), ccf_lags_affected=lags,
|
|
173
|
+
compression=(m_wo / m_with if m_with > 1e-30 else 1.0),
|
|
174
|
+
ccf_max=float(np.max(np.abs(ccf_full))),
|
|
175
|
+
ccf_max_without=float(np.max(np.abs(_c))),
|
|
176
|
+
ccf_without=np.asarray(_c, float))
|
|
177
|
+
out.append(an)
|
|
178
|
+
if p_full < 0.05 <= p_wo:
|
|
179
|
+
decisive.append(an)
|
|
180
|
+
|
|
181
|
+
out.sort(key=lambda a_: -abs(a_.z))
|
|
182
|
+
return Calibration(p_transfer=float(p_full), p_exog=float(p_exog), n=n,
|
|
183
|
+
threshold=threshold, anomalies=out, decisive=decisive,
|
|
184
|
+
names=(cs.names[link.inp], cs.names[link.out]),
|
|
185
|
+
ccf=np.asarray(ccf_full, float))
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def report_calibration(cal):
|
|
189
|
+
"""The calibration table, and the N6 branch it implies."""
|
|
190
|
+
xn, yn = cal.names
|
|
191
|
+
L = ["=" * 66,
|
|
192
|
+
f" ANOMALY CALIBRATION — {xn} -> {yn}",
|
|
193
|
+
"=" * 66,
|
|
194
|
+
" Which observations are bending the instruments this model is",
|
|
195
|
+
" identified and judged with: the CCF, and the adequacy portmanteau.",
|
|
196
|
+
"",
|
|
197
|
+
f" adequacy p = {cal.p_transfer:.4f} exogeneity p = {cal.p_exog:.4f}"
|
|
198
|
+
f" (|z| > {cal.threshold:g}, n = {cal.n})",
|
|
199
|
+
""]
|
|
200
|
+
if not cal.anomalies:
|
|
201
|
+
L += [" No residual exceeds the threshold. Whatever the portmanteau says,",
|
|
202
|
+
" it is not one observation saying it.", "=" * 66]
|
|
203
|
+
return "\n".join(L)
|
|
204
|
+
|
|
205
|
+
L += [" date z var% CCF x max|r| -> sin él p sin él",
|
|
206
|
+
" " + "-" * 64]
|
|
207
|
+
for a in cal.anomalies:
|
|
208
|
+
L.append(f" {a.date:>8s} {a.z:+7.2f} {100*a.variance_fraction:5.1f}%"
|
|
209
|
+
f" {a.compression:5.2f}x {a.ccf_max:6.3f} -> {a.ccf_max_without:6.3f}"
|
|
210
|
+
f" {a.p_transfer_without:6.4f}")
|
|
211
|
+
L += ["",
|
|
212
|
+
" «CCF x» es lo importante: en cuánto CRECEN todos los coeficientes al",
|
|
213
|
+
" quitar esa observación. Un anómalo infla la varianza residual, que es",
|
|
214
|
+
" el DIVISOR de toda correlación, así que aplasta TODOS los retardos a",
|
|
215
|
+
" la vez — no sólo el que toca. Es un hecho a VERIFICAR, y se verifica",
|
|
216
|
+
" mirando: quítalo y los coeficientes resucitan (plot_calibration).",
|
|
217
|
+
""]
|
|
218
|
+
|
|
219
|
+
v = cal.verdict
|
|
220
|
+
if v == "adequate":
|
|
221
|
+
big = [a for a in cal.anomalies if a.variance_fraction > 0.15]
|
|
222
|
+
if big:
|
|
223
|
+
a = big[0]
|
|
224
|
+
L += [f" *** THE TRANSFER PASSES, AND {a.date} IS WHY IT PASSES SO WELL.",
|
|
225
|
+
f" Taking it out multiplies every CCF coefficient by "
|
|
226
|
+
f"{a.compression:.2f} —",
|
|
227
|
+
f" max|r| goes {a.ccf_max:.3f} -> {a.ccf_max_without:.3f}.",
|
|
228
|
+
f" It carries {100*a.variance_fraction:.0f} % of the residual "
|
|
229
|
+
f"variance on its own.",
|
|
230
|
+
"",
|
|
231
|
+
" A large anomaly INFLATES the residual variance, which",
|
|
232
|
+
" shrinks every correlation, which makes the portmanteau",
|
|
233
|
+
f" comfortable: p = {cal.p_transfer:.4f} here against "
|
|
234
|
+
f"{a.p_transfer_without:.4f} without it.",
|
|
235
|
+
" An adequacy bought that way is not evidence of a good",
|
|
236
|
+
" transfer — it is evidence of an uncalibrated",
|
|
237
|
+
" intervention. It belongs in `art`, on the univariate",
|
|
238
|
+
" rung, and travels here in the .pre."]
|
|
239
|
+
else:
|
|
240
|
+
L += [" The transfer is adequate, and no single observation is",
|
|
241
|
+
" carrying the verdict."]
|
|
242
|
+
elif v == "observation":
|
|
243
|
+
d = cal.decisive[0]
|
|
244
|
+
L += [" *** THE VERDICT RESTS ON ONE OBSERVATION.",
|
|
245
|
+
f" Adequacy is {cal.p_transfer:.4f} with {d.date} in and "
|
|
246
|
+
f"{d.p_transfer_without:.4f} without it.",
|
|
247
|
+
"",
|
|
248
|
+
" This is an INTERVENTION, not a misspecified shape.",
|
|
249
|
+
" Re-identifying (b, r, s) around it is how a model acquires",
|
|
250
|
+
" a lag nobody can interpret. Calibrate the anomaly instead —",
|
|
251
|
+
" in `art`, on the univariate rung, where interventions live",
|
|
252
|
+
" and travel here in the .pre."]
|
|
253
|
+
else:
|
|
254
|
+
L += [" Adequacy fails and NO single observation explains it: dropping",
|
|
255
|
+
" any one of them leaves the verdict where it was. The SHAPE is",
|
|
256
|
+
" wrong — go back and re-identify (b, r, s)."]
|
|
257
|
+
L.append("=" * 66)
|
|
258
|
+
return "\n".join(L)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def plot_calibration(cal, anomaly=None, lags=None):
|
|
262
|
+
"""The verification: the CCF **with and without** the anomaly, overlaid.
|
|
263
|
+
|
|
264
|
+
This is the "easy to see" part of the school's rule. The claim — that one
|
|
265
|
+
observation is flattening every coefficient by inflating the variance — is
|
|
266
|
+
not something to take on trust from a number. Remove the point and the
|
|
267
|
+
correlations come back; the picture either shows that or it does not.
|
|
268
|
+
"""
|
|
269
|
+
try:
|
|
270
|
+
import matplotlib
|
|
271
|
+
matplotlib.use("Agg")
|
|
272
|
+
import matplotlib.pyplot as plt
|
|
273
|
+
except ImportError: # pragma: no cover
|
|
274
|
+
raise ImportError("plotting needs matplotlib: pip install 'drtran[plots]'")
|
|
275
|
+
|
|
276
|
+
a = anomaly or (cal.anomalies[0] if cal.anomalies else None)
|
|
277
|
+
if a is None or a.ccf_without is None:
|
|
278
|
+
raise ValueError("nothing to verify: no anomaly above the threshold")
|
|
279
|
+
|
|
280
|
+
wo = np.asarray(a.ccf_without, float)
|
|
281
|
+
wi = np.asarray(cal.ccf, float)
|
|
282
|
+
k = np.arange(len(wo))
|
|
283
|
+
band = 2.0 / math.sqrt(cal.n)
|
|
284
|
+
|
|
285
|
+
fig, ax = plt.subplots(figsize=(9.5, 3.8))
|
|
286
|
+
ax.bar(k - 0.19, wi, width=0.36, color="#9ca3af", label="con la anomalía")
|
|
287
|
+
ax.bar(k + 0.19, wo, width=0.36, color="#1d4ed8", label=f"sin {a.date}")
|
|
288
|
+
ax.axhline(band, ls="--", lw=0.9, color="#b91c1c")
|
|
289
|
+
ax.axhline(-band, ls="--", lw=0.9, color="#b91c1c")
|
|
290
|
+
ax.axhline(0, lw=0.8, color="black")
|
|
291
|
+
ax.set_xlabel("lag k (k >= 0: the transfer)")
|
|
292
|
+
ax.set_ylabel("r(k)")
|
|
293
|
+
ax.set_title(f"Verificación — quitando {a.date} cada coeficiente x"
|
|
294
|
+
f"{a.compression:.2f} (media |r|)")
|
|
295
|
+
ax.legend(frameon=False, fontsize=9)
|
|
296
|
+
for sp in ("top", "right"):
|
|
297
|
+
ax.spines[sp].set_visible(False)
|
|
298
|
+
fig.tight_layout()
|
|
299
|
+
return fig
|