drvec 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- drvec/__init__.py +22 -0
- drvec/_gsl.py +83 -0
- drvec/api.py +942 -0
- drvec/c_engine.py +391 -0
- drvec/cast.py +196 -0
- drvec/cli.py +592 -0
- drvec/data.py +295 -0
- drvec/diagnose.py +439 -0
- drvec/fdhess.py +130 -0
- drvec/fit.py +528 -0
- drvec/forecast.py +314 -0
- drvec/irfboot.py +216 -0
- drvec/levels.py +158 -0
- drvec/loglik.py +45 -0
- drvec/nla.py +266 -0
- drvec/parallel.py +86 -0
- drvec/params.py +311 -0
- drvec/rank.py +401 -0
- drvec/report.py +1712 -0
- drvec/results.py +167 -0
- drvec/rng.py +51 -0
- drvec/rungtable.py +82 -0
- drvec/start.py +317 -0
- drvec/stderr.py +33 -0
- drvec/xlink.py +226 -0
- drvec-0.1.0.dist-info/METADATA +205 -0
- drvec-0.1.0.dist-info/RECORD +31 -0
- drvec-0.1.0.dist-info/WHEEL +5 -0
- drvec-0.1.0.dist-info/entry_points.txt +2 -0
- drvec-0.1.0.dist-info/licenses/COPYING +339 -0
- drvec-0.1.0.dist-info/top_level.txt +1 -0
drvec/forecast.py
ADDED
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""-f H: forecasts in levels, with their theoretical bands; -estwin E: the
|
|
2
|
+
rolling-origin evaluation that holds the parameters fitted on 1..E.
|
|
3
|
+
|
|
4
|
+
C: `forecast_core` (drvec.c:5935), `forecast_vec` (6456), `rolling_eval`
|
|
5
|
+
(5990) and the back-transform `bt_level`/`bt_jac` (1517-1541), loop for loop.
|
|
6
|
+
|
|
7
|
+
The recursion runs on Ybar = [nabla Y2 ; W] with the fitted stationary VARMA,
|
|
8
|
+
the shocks after the origin at zero; the levels are rebuilt from the origin's
|
|
9
|
+
Y2 level (the Y2 block cumulated, then Y1 = W - B2'Y2). The bands come from
|
|
10
|
+
the psi weights through the level map G_m (level_error_map), Var(h) = sum of
|
|
11
|
+
G_m Sigma G_m'. Levels are handed back in each series' own units, z =
|
|
12
|
+
BoxCox^-1((w + det) / refactor), det the series' deterministic component at
|
|
13
|
+
that raw observation (bt_det, data.Data.det_path: known over the sample and
|
|
14
|
+
the horizon, a step or a trend goes on), and the s.e. by the delta method.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import math
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def bt_level(lam, refac, w, det=0.0):
|
|
22
|
+
"""w (the system's units) -> z (the series' own units) at a raw
|
|
23
|
+
observation whose deterministic component is det."""
|
|
24
|
+
c = (w + det) / refac
|
|
25
|
+
if abs(lam) < 1.0e-8:
|
|
26
|
+
return math.exp(c)
|
|
27
|
+
if abs(lam - 1.0) < 1.0e-12:
|
|
28
|
+
return c
|
|
29
|
+
if lam * c + 1.0 <= 0.0:
|
|
30
|
+
return float("nan")
|
|
31
|
+
return pow(lam * c + 1.0, 1.0 / lam)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def bt_jac(lam, refac, w, det=0.0):
|
|
35
|
+
"""dz/dw at w: the delta-method factor."""
|
|
36
|
+
c = (w + det) / refac
|
|
37
|
+
if abs(lam) < 1.0e-8:
|
|
38
|
+
return math.exp(c) / refac
|
|
39
|
+
if abs(lam - 1.0) < 1.0e-12:
|
|
40
|
+
return 1.0 / refac
|
|
41
|
+
if lam * c + 1.0 <= 0.0:
|
|
42
|
+
return float("nan")
|
|
43
|
+
return pow(lam * c + 1.0, 1.0 / lam - 1.0) / refac
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def forecast_core(M, r, p, q, phi, theta, mu, w, a, y2, B2, o, H):
|
|
47
|
+
"""drvec.c:5935, from origin o (1-based: the last observation used).
|
|
48
|
+
w, a: (n x M) lists, 0-based rows; y2: the Y2 levels (0-based rows).
|
|
49
|
+
-> (Yb, lev), H x M lists, .inp column order."""
|
|
50
|
+
s = M - r
|
|
51
|
+
Yb = [[0.0] * M for _ in range(H)]
|
|
52
|
+
for h in range(1, H + 1):
|
|
53
|
+
for i in range(M):
|
|
54
|
+
acc = 0.0
|
|
55
|
+
for k in range(1, p + 1):
|
|
56
|
+
t = h - k
|
|
57
|
+
ph = phi[k - 1][i]
|
|
58
|
+
for l in range(M):
|
|
59
|
+
acc += ph[l] * ((Yb[t - 1][l] if t >= 1 else w[o + t - 1][l]) - mu[l])
|
|
60
|
+
for k in range(1, q + 1):
|
|
61
|
+
t = h - k
|
|
62
|
+
if t >= 1:
|
|
63
|
+
continue
|
|
64
|
+
th = theta[k - 1][i]
|
|
65
|
+
for l in range(M):
|
|
66
|
+
acc -= th[l] * a[o + t - 1][l]
|
|
67
|
+
Yb[h - 1][i] = mu[i] + acc
|
|
68
|
+
lev = [[0.0] * M for _ in range(H)]
|
|
69
|
+
for h in range(H):
|
|
70
|
+
for i in range(s):
|
|
71
|
+
lev[h][i] = (y2[o - 1][i] if h == 0 else lev[h - 1][i]) + Yb[h][i]
|
|
72
|
+
for j in range(r):
|
|
73
|
+
acc = Yb[h][s + j]
|
|
74
|
+
for i in range(s):
|
|
75
|
+
acc -= B2[i][j] * lev[h][i]
|
|
76
|
+
lev[h][s + j] = acc
|
|
77
|
+
return Yb, lev
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _det_at(data, H):
|
|
81
|
+
"""bt_det over t = 1..nobs_raw + H (bt_alloc's horizon, drvec.c:6852):
|
|
82
|
+
-> f(i, t) for a 1-based raw observation t, 0.0 outside it."""
|
|
83
|
+
T = data.nobs_raw + H
|
|
84
|
+
paths = [data.det_path(i, T) for i in range(data.M)]
|
|
85
|
+
|
|
86
|
+
def at(i, t):
|
|
87
|
+
return paths[i][t - 1] if 1 <= t <= T else 0.0
|
|
88
|
+
return at
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _date(start, freq, k):
|
|
92
|
+
"""ObsToDate: the date of raw observation k (1-based)."""
|
|
93
|
+
from .diagnose import obs_to_date
|
|
94
|
+
return obs_to_date(start[0], start[1], k, freq)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def z_of(conf):
|
|
98
|
+
"""The band's multiplier, the C's three constants (forecast_vec)."""
|
|
99
|
+
return 2.575829 if conf >= 0.99 else 1.959964 if conf >= 0.95 else 1.644854
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def forecast_arrays(res, fv, H, conf=0.95):
|
|
103
|
+
"""THE forecast computation (drvec.c:6456's arithmetic, loop for loop):
|
|
104
|
+
the recursion, the psi-weight variance through the level map, and the
|
|
105
|
+
back-transform. forecast_vec prints the .out section and <base>.forecast
|
|
106
|
+
from these numbers, and FitResult.forecasts hands them over as arrays, so
|
|
107
|
+
there is one code and the two cannot disagree.
|
|
108
|
+
-> dict with dates [(period, sub)] (H), and (H, M) lists in each series'
|
|
109
|
+
own units: level, low, high, se (the delta-method s.e. the file prints),
|
|
110
|
+
plus model_level and model_se (the level and its theoretical s.e. in the
|
|
111
|
+
model's units, before the deterministic path and the back-transform), z."""
|
|
112
|
+
from .report import compute_psi_weights, level_error_map
|
|
113
|
+
spec, data = res.spec, res.data
|
|
114
|
+
M, r, p, q = spec.M, spec.r, spec.p, spec.q
|
|
115
|
+
n = fv.rd.n
|
|
116
|
+
phi = [fv.phi[k].tolist() for k in range(p)]
|
|
117
|
+
theta = [fv.theta[k].tolist() for k in range(q)]
|
|
118
|
+
mu = [float(v) for v in fv.mu]
|
|
119
|
+
qq = fv.qq.tolist()
|
|
120
|
+
w = fv.w.tolist()
|
|
121
|
+
raw_origin = n + 1
|
|
122
|
+
lam = [float(v) for v in data.lambdas]
|
|
123
|
+
ref = [float(v) for v in data.refactors]
|
|
124
|
+
det = _det_at(data, H)
|
|
125
|
+
Psi = compute_psi_weights(M, p, q, phi, theta, H)
|
|
126
|
+
Sig = [[res.sigma2 * qq[i][j] for j in range(M)] for i in range(M)]
|
|
127
|
+
_Yb, lev = forecast_core(M, r, p, q, phi, theta, mu, w, fv.a, fv.rd.y2, fv.B2, n, H)
|
|
128
|
+
z = z_of(conf)
|
|
129
|
+
Var = [[0.0] * M for _ in range(M)]
|
|
130
|
+
Csum = [[0.0] * M for _ in range(M)]
|
|
131
|
+
SE = [[0.0] * M for _ in range(H)]
|
|
132
|
+
for h in range(1, H + 1):
|
|
133
|
+
m = h - 1
|
|
134
|
+
for i in range(M):
|
|
135
|
+
for j in range(M):
|
|
136
|
+
Csum[i][j] += Psi[m][i][j]
|
|
137
|
+
G = level_error_map(Csum, Psi[m], fv.B2, M, r)
|
|
138
|
+
for i in range(M):
|
|
139
|
+
for j in range(M):
|
|
140
|
+
acc = 0.0
|
|
141
|
+
for k in range(M):
|
|
142
|
+
for l in range(M):
|
|
143
|
+
acc += G[i][k] * Sig[k][l] * G[j][l]
|
|
144
|
+
Var[i][j] += acc
|
|
145
|
+
for i in range(M):
|
|
146
|
+
SE[h - 1][i] = math.sqrt(Var[i][i]) if Var[i][i] > 0.0 else 0.0
|
|
147
|
+
out = {"dates": [], "level": [], "low": [], "high": [], "se": [],
|
|
148
|
+
"model_level": [list(row) for row in lev[:H]], "model_se": SE, "z": z,
|
|
149
|
+
"conf": conf}
|
|
150
|
+
for h in range(1, H + 1):
|
|
151
|
+
out["dates"].append(tuple(_date(data.start, data.freq, raw_origin + h)))
|
|
152
|
+
rl, rlo, rhi, rse = [], [], [], []
|
|
153
|
+
for i in range(M):
|
|
154
|
+
L = lev[h - 1][i]
|
|
155
|
+
e = SE[h - 1][i]
|
|
156
|
+
dt = det(i, raw_origin + h)
|
|
157
|
+
rl.append(bt_level(lam[i], ref[i], L, dt))
|
|
158
|
+
rlo.append(bt_level(lam[i], ref[i], L - z * e, dt))
|
|
159
|
+
rhi.append(bt_level(lam[i], ref[i], L + z * e, dt))
|
|
160
|
+
rse.append(bt_jac(lam[i], ref[i], L, dt) * e)
|
|
161
|
+
out["level"].append(rl)
|
|
162
|
+
out["low"].append(rlo)
|
|
163
|
+
out["high"].append(rhi)
|
|
164
|
+
out["se"].append(rse)
|
|
165
|
+
return out
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def forecast_vec(res, fv, H, base, conf=0.95):
|
|
169
|
+
"""-> (the .out's Forecast section, <base>.forecast's text), printed from
|
|
170
|
+
forecast_arrays' numbers."""
|
|
171
|
+
from .report import banner
|
|
172
|
+
from . import __version__
|
|
173
|
+
spec, data = res.spec, res.data
|
|
174
|
+
M, r, p, q = spec.M, spec.r, spec.p, spec.q
|
|
175
|
+
s = M - r
|
|
176
|
+
n = fv.rd.n
|
|
177
|
+
phi = [fv.phi[k].tolist() for k in range(p)]
|
|
178
|
+
theta = [fv.theta[k].tolist() for k in range(q)]
|
|
179
|
+
mu = [float(v) for v in fv.mu]
|
|
180
|
+
w = fv.w.tolist()
|
|
181
|
+
a = fv.a
|
|
182
|
+
names = data.names
|
|
183
|
+
fc = forecast_arrays(res, fv, H, conf)
|
|
184
|
+
z = fc["z"]
|
|
185
|
+
|
|
186
|
+
o = banner("Forecast")
|
|
187
|
+
worst = 0.0
|
|
188
|
+
t0 = n // 2 + 1
|
|
189
|
+
if t0 < p + q + 1:
|
|
190
|
+
t0 = p + q + 1
|
|
191
|
+
for t in range(t0, n + 1):
|
|
192
|
+
for i in range(M):
|
|
193
|
+
pred = mu[i]
|
|
194
|
+
for k in range(1, p + 1):
|
|
195
|
+
for l in range(M):
|
|
196
|
+
pred += phi[k - 1][i][l] * (w[t - k - 1][l] - mu[l])
|
|
197
|
+
for k in range(1, q + 1):
|
|
198
|
+
for l in range(M):
|
|
199
|
+
pred -= theta[k - 1][i][l] * a[t - k - 1][l]
|
|
200
|
+
d = abs(w[t - 1][i] - a[t - 1][i] - pred)
|
|
201
|
+
if d > worst:
|
|
202
|
+
worst = d
|
|
203
|
+
o += ("\n one-step self-check: max |Ybar_t - a_t - pred(t|t-1)| = %.3e over %d obs\n"
|
|
204
|
+
" (the residue is the xi truncation, of order xitol = %.0e)\n"
|
|
205
|
+
% (worst, n - t0 + 1, 1.0e-3))
|
|
206
|
+
o += ("\n%d step%s ahead, in levels. Columns are the .inp's: Y2 block (1..%d), then Y1.\n"
|
|
207
|
+
"s.e. are THEORETICAL; a %.0f%% band is +/- %.4f s.e.\n"
|
|
208
|
+
% (H, "" if H == 1 else "s", s, 100.0 * conf, z))
|
|
209
|
+
o += ("Levels are in each series' OWN units: deterministic path added\n"
|
|
210
|
+
"back, refactor divided out, Box-Cox inverted (the median under\n"
|
|
211
|
+
"a log); s.e. by the delta method (BUG-36).\n")
|
|
212
|
+
o += "\n h" + "".join(" %14s %10s" % (nm, "s.e.") for nm in names) + "\n"
|
|
213
|
+
for h in range(1, H + 1):
|
|
214
|
+
o += "%4d" % h
|
|
215
|
+
for i in range(M):
|
|
216
|
+
o += " %14.6f %10.6f" % (fc["level"][h - 1][i], fc["se"][h - 1][i])
|
|
217
|
+
o += "\n"
|
|
218
|
+
o += ("\n ! the Y1 s.e. inherits the CUMULATED error of the nabla Y2 block,\n"
|
|
219
|
+
" because the two are tied by the cointegrating relation\n")
|
|
220
|
+
|
|
221
|
+
f = ["DRVEC %s (python) -- forecasts from a VEC(%d) model" % (__version__, r),
|
|
222
|
+
"input=%s.inp p=%d q=%d r=%d case=%d freq=%d horizon=%d bands=%.0f%%"
|
|
223
|
+
% (base, p, q, r, spec.case, data.freq, H, 100.0 * conf),
|
|
224
|
+
"Levels in each series' own units (deterministic path added back, refactor "
|
|
225
|
+
"divided out, Box-Cox inverted: the median under a log). Columns 1..%d are "
|
|
226
|
+
"the Y2 block, %d..%d the Y1 block." % (s, s + 1, M),
|
|
227
|
+
"Low/High are +/- %.4f standard errors in the model's units, transformed at "
|
|
228
|
+
"both ends; s.e. by the delta method and are THEORETICAL: they assume the "
|
|
229
|
+
"specification is right." % z, ""]
|
|
230
|
+
for i in range(M):
|
|
231
|
+
f.append("Series %d (%s):" % (i + 1, names[i]))
|
|
232
|
+
f.append(" %-9s %14s %14s %14s %12s" % ("date", "Level", "Low", "High", "s.e."))
|
|
233
|
+
for h in range(1, H + 1):
|
|
234
|
+
per, sub = fc["dates"][h - 1]
|
|
235
|
+
d = (" %3d/%-5d" % (sub, per)) if data.freq > 1 else (" %-9d" % per)
|
|
236
|
+
f.append(d + " %14.6f %14.6f %14.6f %12.6f" % (
|
|
237
|
+
fc["level"][h - 1][i], fc["low"][h - 1][i], fc["high"][h - 1][i],
|
|
238
|
+
fc["se"][h - 1][i]))
|
|
239
|
+
f.append("")
|
|
240
|
+
return o, "\n".join(f) + "\n"
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def rolling_eval(res, E, H):
|
|
244
|
+
"""drvec.c:5990. The parameters of the fit on 1..E held fixed, the origin
|
|
245
|
+
rolled over E..n-H on the WHOLE sample (Ybar rebuilt and filtered once with
|
|
246
|
+
those parameters). -> (the .out's section, <base>.recursive's text), or None
|
|
247
|
+
when E + H passes the end."""
|
|
248
|
+
from . import __version__
|
|
249
|
+
from .cast import shootx, unpack
|
|
250
|
+
from .loglik import elf
|
|
251
|
+
spec, data = res.spec, res.data
|
|
252
|
+
M, r, p, q = spec.M, spec.r, spec.p, spec.q
|
|
253
|
+
s = M - r
|
|
254
|
+
nfull = data.nobs_raw - 1
|
|
255
|
+
if E + H > nfull:
|
|
256
|
+
return None
|
|
257
|
+
rd = data.at(r, full=True)
|
|
258
|
+
mu, phi, theta, _qq, w, ifc = shootx(spec, rd, res.x)
|
|
259
|
+
if ifc:
|
|
260
|
+
return None
|
|
261
|
+
_f1, _f2, a, ifa = elf(spec, rd, res.x, atf=True)
|
|
262
|
+
if ifa > 0:
|
|
263
|
+
return None
|
|
264
|
+
a = [[float(a[t, i]) for i in range(1, M + 1)] for t in range(1, rd.n + 1)]
|
|
265
|
+
B2 = unpack(spec, res.x)[5]
|
|
266
|
+
phi = [phi[k].tolist() for k in range(p)]
|
|
267
|
+
theta = [theta[k].tolist() for k in range(q)]
|
|
268
|
+
mu = [float(v) for v in mu]
|
|
269
|
+
w = w.tolist()
|
|
270
|
+
lam = [float(v) for v in data.lambdas]
|
|
271
|
+
ref = [float(v) for v in data.refactors]
|
|
272
|
+
names = data.names
|
|
273
|
+
det = _det_at(data, H)
|
|
274
|
+
sae = [[0.0] * M for _ in range(H)]
|
|
275
|
+
sse = [[0.0] * M for _ in range(H)]
|
|
276
|
+
spe = [[0.0] * M for _ in range(H)]
|
|
277
|
+
cnt = [[0] * M for _ in range(H)]
|
|
278
|
+
csv = ["# DRVEC %s (python) rolling-origin errors, one row per (origin, horizon, "
|
|
279
|
+
"series)" % __version__,
|
|
280
|
+
"# estwin=%d horizon=%d p=%d q=%d r=%d case=%d" % (E, H, p, spec.q, r, spec.case),
|
|
281
|
+
"origin,h,series,actual,forecast,error"]
|
|
282
|
+
nor = 0
|
|
283
|
+
o = E
|
|
284
|
+
while o + H <= nfull:
|
|
285
|
+
_Yb, lev = forecast_core(M, r, p, q, phi, theta, mu, w, a, rd.y2, B2, o, H)
|
|
286
|
+
nor += 1
|
|
287
|
+
for h in range(1, H + 1):
|
|
288
|
+
for i in range(M):
|
|
289
|
+
act = rd.y2[o + h - 1][i] if i < s else rd.dm[o + h - 1][i]
|
|
290
|
+
tr = o + h + 1 # the raw observation
|
|
291
|
+
act = bt_level(lam[i], ref[i], act, det(i, tr))
|
|
292
|
+
fc = bt_level(lam[i], ref[i], lev[h - 1][i], det(i, tr))
|
|
293
|
+
e = act - fc
|
|
294
|
+
sae[h - 1][i] += abs(e)
|
|
295
|
+
sse[h - 1][i] += e * e
|
|
296
|
+
if abs(act) > 1.0e-12:
|
|
297
|
+
spe[h - 1][i] += abs(e / act)
|
|
298
|
+
cnt[h - 1][i] += 1
|
|
299
|
+
csv.append("%d,%d,%s,%.10f,%.10f,%.10f" % (o, h, names[i], act, fc, e))
|
|
300
|
+
o += 1
|
|
301
|
+
out = ("\n=== Rolling-origin evaluation (out of sample) ===\n"
|
|
302
|
+
" Estimated ONCE on observations 1..%d; parameters held FIXED.\n"
|
|
303
|
+
" %d origins, %d..%d, each compared with what actually happened.\n"
|
|
304
|
+
" The parameters have not seen the data they are scored against.\n\n"
|
|
305
|
+
% (E, nor, E, E + nor - 1))
|
|
306
|
+
out += " h series MAE RMSE MAPE%\n"
|
|
307
|
+
for h in range(H):
|
|
308
|
+
for i in range(M):
|
|
309
|
+
out += "%4d %-12s %13.6f %13.6f %13.4f\n" % (
|
|
310
|
+
h + 1, names[i], sae[h][i] / cnt[h][i], math.sqrt(sse[h][i] / cnt[h][i]),
|
|
311
|
+
100.0 * spe[h][i] / cnt[h][i])
|
|
312
|
+
out += ("\n These are the numbers that decide whether the model earns its\n"
|
|
313
|
+
" parameters. A likelihood, an AIC and a theoretical band do not.\n")
|
|
314
|
+
return out, "\n".join(csv) + "\n"
|
drvec/irfboot.py
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""-irfboot N: percentile bands for the level responses and the decomposition
|
|
2
|
+
by the parametric bootstrap (irf_bootstrap, drvec.c:7540-7760).
|
|
3
|
+
|
|
4
|
+
N samples are simulated from the fitted model (simulate_h0, the generator of
|
|
5
|
+
the rank bootstrap, from its seed), each is refitted as the data were (same
|
|
6
|
+
rank, same restrictions, the ladder start and the search), and the responses
|
|
7
|
+
of every refit are collected. The bands are Efron's percentiles,
|
|
8
|
+
col[ceil(p nok)] of the sorted replications."""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import math
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
|
|
14
|
+
from .rank import fit_rank, simulate_h0
|
|
15
|
+
from .report import banner, ldl_sigma, level_irf_R
|
|
16
|
+
from .rng import Rng
|
|
17
|
+
|
|
18
|
+
PICK = (0, 1, 2, 4, 8, 12, 16, 20, 30, 40)
|
|
19
|
+
QP = (0.025, 0.05, 0.50, 0.95, 0.975)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _irf_at(spec, data, x, K):
|
|
23
|
+
"""model_at + level_irf_R: the responses R[k][i][j] of the model x, or None."""
|
|
24
|
+
from .cast import shootx, unpack
|
|
25
|
+
from .loglik import elf
|
|
26
|
+
rd = data.at(spec.r)
|
|
27
|
+
mu, phi, theta, qq, w, ifs = shootx(spec, rd, x)
|
|
28
|
+
if ifs:
|
|
29
|
+
return None
|
|
30
|
+
f1, _f2, _a, ifa = elf(spec, rd, x, atf=True)
|
|
31
|
+
if ifa:
|
|
32
|
+
return None
|
|
33
|
+
sigma2 = f1 / (rd.n * spec.M)
|
|
34
|
+
ld = ldl_sigma(sigma2, qq.tolist(), spec.M)
|
|
35
|
+
if ld is None:
|
|
36
|
+
return None
|
|
37
|
+
P, D = ld
|
|
38
|
+
B2 = unpack(spec, x)[5]
|
|
39
|
+
return level_irf_R(spec.M, spec.r, spec.p, spec.q, phi.tolist(), theta.tolist(), B2, K, P, D)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class IrfBoot:
|
|
44
|
+
N: int
|
|
45
|
+
K: int
|
|
46
|
+
R0: list
|
|
47
|
+
irf: list # per good replication: R[k][i][j]
|
|
48
|
+
fev: list # per good replication: fevd[k][i][j] in %
|
|
49
|
+
nfail: int
|
|
50
|
+
# the C engine (drvec.c_engine): the replications are not in its files, so
|
|
51
|
+
# the bands come read back from <base>.irfboot, and the .out's band section
|
|
52
|
+
# and the file are the C's own text
|
|
53
|
+
c_nok: int = None
|
|
54
|
+
c_bands: object = None # results.IrfBands, or None (fewer than 20)
|
|
55
|
+
c_out: str = None
|
|
56
|
+
c_file: str = None
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def nok(self):
|
|
60
|
+
return self.c_nok if self.c_nok is not None else len(self.irf)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _refit_irf(spec, data, sim, K):
|
|
64
|
+
"""One replication's refit and its responses (None when it fails). It
|
|
65
|
+
consumes no random numbers: the sample is drawn before."""
|
|
66
|
+
db = data.with_raw(sim)
|
|
67
|
+
res, _ = fit_rank(spec, db)
|
|
68
|
+
if res.ifault == 0:
|
|
69
|
+
return _irf_at(spec, db, res.x, K)
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def irf_bootstrap(spec, data, x, N, K, rng=None, progress=None, workers=1):
|
|
74
|
+
"""irf_bootstrap. progress(b, ok) is called after replication b (1-based)
|
|
75
|
+
with ok False when it failed. workers > 1 draws the N samples first, in the
|
|
76
|
+
generator's order, and refits them in a process pool: the same numbers."""
|
|
77
|
+
rng = rng or Rng()
|
|
78
|
+
R0 = _irf_at(spec, data, x, K)
|
|
79
|
+
if R0 is None:
|
|
80
|
+
raise RuntimeError("-irfboot: the fitted model cannot be simulated")
|
|
81
|
+
M = spec.M
|
|
82
|
+
irf, fev, nfail = [], [], 0
|
|
83
|
+
if workers > 1:
|
|
84
|
+
from .parallel import map_refits
|
|
85
|
+
sims = [simulate_h0(spec, data, x, rng) for _b in range(N)]
|
|
86
|
+
goods = map_refits(_refit_irf, data, [(spec, sim, K) if sim is not None else None
|
|
87
|
+
for sim in sims], workers)
|
|
88
|
+
else:
|
|
89
|
+
goods = None
|
|
90
|
+
for _b in range(N):
|
|
91
|
+
if goods is not None:
|
|
92
|
+
good = next(goods)
|
|
93
|
+
else:
|
|
94
|
+
good = None
|
|
95
|
+
sim = simulate_h0(spec, data, x, rng)
|
|
96
|
+
if sim is not None:
|
|
97
|
+
good = _refit_irf(spec, data, sim, K)
|
|
98
|
+
if progress:
|
|
99
|
+
progress(_b + 1, good is not None)
|
|
100
|
+
if good is None:
|
|
101
|
+
nfail += 1
|
|
102
|
+
continue
|
|
103
|
+
F = [[[0.0] * M for _ in range(M)] for _ in range(K + 1)]
|
|
104
|
+
for i in range(M):
|
|
105
|
+
acc = [0.0] * M
|
|
106
|
+
for k in range(K + 1):
|
|
107
|
+
tot = 0.0
|
|
108
|
+
for j in range(M):
|
|
109
|
+
acc[j] += good[k][i][j] * good[k][i][j]
|
|
110
|
+
tot += acc[j]
|
|
111
|
+
for j in range(M):
|
|
112
|
+
F[k][i][j] = 100.0 * acc[j] / tot if tot > 0.0 else 0.0
|
|
113
|
+
irf.append(good)
|
|
114
|
+
fev.append(F)
|
|
115
|
+
if goods is not None:
|
|
116
|
+
goods.close()
|
|
117
|
+
return IrfBoot(N, K, R0, irf, fev, nfail)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _pct(col, p):
|
|
121
|
+
n = len(col)
|
|
122
|
+
ix = int(math.ceil(p * n))
|
|
123
|
+
ix = min(max(ix, 1), n)
|
|
124
|
+
return col[ix - 1]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _point_fev(R0, k, i, j, M):
|
|
128
|
+
a = tot = 0.0
|
|
129
|
+
for kk in range(k + 1):
|
|
130
|
+
for jj in range(M):
|
|
131
|
+
tot += R0[kk][i][jj] * R0[kk][i][jj]
|
|
132
|
+
if jj == j:
|
|
133
|
+
a += R0[kk][i][jj] * R0[kk][i][jj]
|
|
134
|
+
return 100.0 * a / tot if tot > 0.0 else 0.0
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def band_arrays(bt, M):
|
|
138
|
+
"""-> results.IrfBands, the numbers of <base>.irfboot (None when nok < 20,
|
|
139
|
+
where the C writes no bands)."""
|
|
140
|
+
import numpy as np
|
|
141
|
+
from .results import IrfBands
|
|
142
|
+
if bt.c_nok is not None:
|
|
143
|
+
return bt.c_bands
|
|
144
|
+
K, nok = bt.K, bt.nok
|
|
145
|
+
if nok < 20:
|
|
146
|
+
return None
|
|
147
|
+
R0 = bt.R0
|
|
148
|
+
irf = np.array(R0, float)
|
|
149
|
+
fv = np.array([[[_point_fev(R0, k, i, j, M) for j in range(M)] for i in range(M)]
|
|
150
|
+
for k in range(K + 1)], float)
|
|
151
|
+
ip = {pp * 100.0: np.zeros((K + 1, M, M)) for pp in QP}
|
|
152
|
+
fp = {pp * 100.0: np.zeros((K + 1, M, M)) for pp in QP}
|
|
153
|
+
for j in range(M):
|
|
154
|
+
for i in range(M):
|
|
155
|
+
for k in range(K + 1):
|
|
156
|
+
for w, dst in ((0, ip), (1, fp)):
|
|
157
|
+
src = bt.fev if w else bt.irf
|
|
158
|
+
col = sorted(src[t][k][i][j] for t in range(nok))
|
|
159
|
+
for pp in QP:
|
|
160
|
+
dst[pp * 100.0][k, i, j] = _pct(col, pp)
|
|
161
|
+
return IrfBands(K, nok, bt.nfail, irf, ip, fv, fp)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def bands_text(bt, spec, names, base):
|
|
165
|
+
"""The .out's band section and the <base>.irfboot file -> (out, file or None)."""
|
|
166
|
+
if bt.c_out is not None:
|
|
167
|
+
# the C's own text; the one name in it is the file's, <base>.irfboot
|
|
168
|
+
cb = getattr(bt, "c_base", None)
|
|
169
|
+
o = bt.c_out
|
|
170
|
+
if cb and cb != base:
|
|
171
|
+
o = o.replace(" k, is in %s.irfboot.\n" % cb, " k, is in %s.irfboot.\n" % base, 1)
|
|
172
|
+
return o, bt.c_file
|
|
173
|
+
M, K, nok = spec.M, bt.K, bt.nok
|
|
174
|
+
o = banner("Bands for the Level Responses, Parametric Bootstrap")
|
|
175
|
+
o += ("\n %d replications simulated from the fitted model and REFITTED as the\n"
|
|
176
|
+
" data were (same rank, restrictions and ladder start); %d failed and\n"
|
|
177
|
+
" are left out. Percentile intervals (Efron), conditional on r = %d.\n"
|
|
178
|
+
" Same orthogonalisation as the responses above. The whole table, every\n"
|
|
179
|
+
" k, is in %s.irfboot.\n" % (nok, bt.nfail, spec.r, base))
|
|
180
|
+
if nok < 20:
|
|
181
|
+
return o + "\n Too few replications for bands (%d).\n" % nok, None
|
|
182
|
+
fb = ("# drvec -irfboot %d: %d replications, percentile bands\n"
|
|
183
|
+
"what,shock,response,k,point,p2.5,p5,p50,p95,p97.5\n" % (bt.N, nok))
|
|
184
|
+
R0 = bt.R0
|
|
185
|
+
for j in range(M):
|
|
186
|
+
for i in range(M):
|
|
187
|
+
o += ("\nResponse of %s to a shock to %s:\n"
|
|
188
|
+
" k point 5%% 95%% 2.5%% 97.5%%\n" % (names[i], names[j]))
|
|
189
|
+
for k in range(K + 1):
|
|
190
|
+
for w in (0, 1):
|
|
191
|
+
src = bt.fev if w else bt.irf
|
|
192
|
+
col = sorted(src[t][k][i][j] for t in range(nok))
|
|
193
|
+
q = [_pct(col, pp) for pp in QP]
|
|
194
|
+
pt = R0[k][i][j] if not w else _point_fev(R0, k, i, j, M)
|
|
195
|
+
fb += "%s,%s,%s,%d,%.8g,%.8g,%.8g,%.8g,%.8g,%.8g\n" % (
|
|
196
|
+
"fevd" if w else "irf", names[j], names[i], k + 1 if w else k,
|
|
197
|
+
pt, q[0], q[1], q[2], q[3], q[4])
|
|
198
|
+
if not w and (k in PICK or k == K):
|
|
199
|
+
o += "%5d %12.6f %8.4f %8.4f %8.4f %8.4f\n" % (
|
|
200
|
+
k, R0[k][i][j], q[1], q[3], q[0], q[4])
|
|
201
|
+
for i in range(M):
|
|
202
|
+
o += "\nFEVD of %s, %% due to each shock: point [5%%, 95%%]\n h" % names[i]
|
|
203
|
+
o += "".join(" %26s" % nm for nm in names) + "\n"
|
|
204
|
+
for k in range(K + 1):
|
|
205
|
+
hh = k + 1
|
|
206
|
+
if not (hh in PICK or hh == K + 1):
|
|
207
|
+
continue
|
|
208
|
+
o += "%5d" % hh
|
|
209
|
+
for j in range(M):
|
|
210
|
+
col = sorted(bt.fev[t][k][i][j] for t in range(nok))
|
|
211
|
+
i5 = min(max(int(math.ceil(0.05 * nok)), 1), nok)
|
|
212
|
+
i95 = min(int(math.ceil(0.95 * nok)), nok)
|
|
213
|
+
o += " %6.1f%% [%5.1f%%, %5.1f%%]" % (_point_fev(R0, k, i, j, M),
|
|
214
|
+
col[i5 - 1], col[i95 - 1])
|
|
215
|
+
o += "\n"
|
|
216
|
+
return o, fb
|
drvec/levels.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""The fitted VEC written as a VARMA in levels, for drvarma's forecaster.
|
|
2
|
+
|
|
3
|
+
The decision (sima's PLAN-vec, decision 5 and A4): a VEC is a VARMA with an
|
|
4
|
+
error term, so the suite forecasts it with the VARMA system's code, one
|
|
5
|
+
forecaster and not two. This module only WRITES the model; the forecast is
|
|
6
|
+
drvarma's (`drvarma.forecast.forecast_given`). drvec's own `-f`
|
|
7
|
+
(forecast.forecast_vec) stays: it is the C's, under byte identity, and the
|
|
8
|
+
test (tests/test_levels.py) holds the two together.
|
|
9
|
+
|
|
10
|
+
The VEC is fitted as a stationary VARMA on Ybar = [nabla Y2 ; W], W = Y1 +
|
|
11
|
+
B2' Y2 (cast.shootx):
|
|
12
|
+
|
|
13
|
+
PhiBar(B) (Ybar_t - mu) = ThetaBar(B) a_t, a_t ~ N(0, sigma2 Q)
|
|
14
|
+
|
|
15
|
+
and Ybar_t = T0 Y_t + T1 Y_{t-1}, Y = [Y2 ; Y1] the levels in the .inp's
|
|
16
|
+
column order, in the model's units (fue's cast: refactor * BoxCox(y) - det),
|
|
17
|
+
|
|
18
|
+
T0 = [ I_s 0 ; B2' I_r ], T1 = [ -I_s 0 ; 0 0 ].
|
|
19
|
+
|
|
20
|
+
Multiplied out and premultiplied by T0^-1 = [ I_s 0 ; -B2' I_r ] so that
|
|
21
|
+
the polynomial starts at I, the levels VARMA of order (p + 1, q) is
|
|
22
|
+
|
|
23
|
+
Phi(B) Y_t = c + Theta(B) e_t, e_t = T0^-1 a_t,
|
|
24
|
+
Phi_1 = T0^-1 (PhiBar_1 T0 - T1)
|
|
25
|
+
Phi_k = T0^-1 (PhiBar_k T0 + PhiBar_{k-1} T1) (k = 2..p)
|
|
26
|
+
Phi_{p+1} = T0^-1 PhiBar_p T1
|
|
27
|
+
Theta_k = T0^-1 ThetaBar_k T0
|
|
28
|
+
c = T0^-1 PhiBar(1) mu
|
|
29
|
+
Sigma_e = T0^-1 (sigma2 Q) T0^-T
|
|
30
|
+
|
|
31
|
+
Phi(1) = T0^-1 PhiBar(1) T(1) has rank r: the unit roots are in Phi(B)
|
|
32
|
+
itself, across the series, which is why the ladder's forecast (a
|
|
33
|
+
univariate operator per series, after the VARMA) cannot take it and the
|
|
34
|
+
forecast goes through `forecast_given`. The drift of case 3 is not of the
|
|
35
|
+
form Phi(1) mu (it lies outside Phi(1)'s columns), so it travels as the
|
|
36
|
+
constant c. e_t is the levels' one-step error (e_Y2 = a_nablaY2, e_Y1 = a_W -
|
|
37
|
+
B2' a_Y2): the psi weights of this Phi and Theta are drvec's level error map
|
|
38
|
+
(report.level_error_map) times T0, and Sigma_e undoes the T0, so the band is
|
|
39
|
+
the same quantity.
|
|
40
|
+
|
|
41
|
+
The deterministic part:
|
|
42
|
+
- cases 1, 2, 3, -case pre and -nodrift: all in mu, hence in c (a held drift
|
|
43
|
+
is a zero in mu);
|
|
44
|
+
- each file's own deterministic variables (steps, trends, impulses, Easter,
|
|
45
|
+
seasonal dummies of fue): kept out of Y by fue's cast and added back over
|
|
46
|
+
the horizon before the Box-Cox is inverted, as drvec's -f does (data.det_path,
|
|
47
|
+
fue's `_build_xi`);
|
|
48
|
+
- -trend (case 4) and -xlink are refused, as drvec's -f refuses them: the C
|
|
49
|
+
does not forecast them, so there is nothing to hold the conversion to.
|
|
50
|
+
|
|
51
|
+
The Box-Cox (a log at lambda = 0) is inverted by fue's `_inv_boxcox`, inside
|
|
52
|
+
drvarma, at each series' lambda and refactor.
|
|
53
|
+
|
|
54
|
+
The fit is read through the public accessors (FitResult.stationary,
|
|
55
|
+
.residuals, .beta2) and the deterministic paths through Data.det_path; the
|
|
56
|
+
band multiplier is drvec's own (forecast.z_of).
|
|
57
|
+
"""
|
|
58
|
+
from __future__ import annotations
|
|
59
|
+
|
|
60
|
+
from dataclasses import dataclass
|
|
61
|
+
|
|
62
|
+
import numpy as np
|
|
63
|
+
|
|
64
|
+
from .forecast import z_of # drvec's band multiplier (the C's three constants)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class LevelsVarma:
|
|
69
|
+
"""A fitted VEC as a levels VARMA, Phi(B) Y_t = c + Theta(B) e_t, ready
|
|
70
|
+
for `drvarma.forecast.forecast_given`. Arrays in the .inp's column order
|
|
71
|
+
(the Y2 block, then Y1), the levels in the model's units."""
|
|
72
|
+
phi: np.ndarray # (p + 1, M, M)
|
|
73
|
+
theta: np.ndarray # (q, M, M)
|
|
74
|
+
sigma: np.ndarray # (M, M), the covariance of e_t
|
|
75
|
+
const: np.ndarray # (M,), c
|
|
76
|
+
w: np.ndarray # (n + 1, M), the levels up to the origin
|
|
77
|
+
a: np.ndarray # (n + 1, M), e_t (row 0, the first level, has none)
|
|
78
|
+
names: list
|
|
79
|
+
lambdas: list
|
|
80
|
+
refactors: list
|
|
81
|
+
r: int
|
|
82
|
+
origin: int # the raw observation of the origin (1-based)
|
|
83
|
+
data: object = None # drvec's Data: the deterministic paths (det_path)
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def M(self):
|
|
87
|
+
return self.w.shape[1]
|
|
88
|
+
|
|
89
|
+
def det(self, H):
|
|
90
|
+
"""(M, H): each series' deterministic component at the H dates
|
|
91
|
+
(drvec's bt_det, built over the sample and H steps beyond it)."""
|
|
92
|
+
T = self.data.nobs_raw + H # bt_alloc's horizon, as -f builds it
|
|
93
|
+
o = self.origin
|
|
94
|
+
return np.array([self.data.det_path(i, T)[o:o + H] for i in range(self.M)],
|
|
95
|
+
float).reshape(self.M, H)
|
|
96
|
+
|
|
97
|
+
def forecast(self, H, conf=0.95, z=None):
|
|
98
|
+
"""H steps through drvarma's forecaster. z: the band's multiplier,
|
|
99
|
+
drvec's for `conf` unless given (drvarma's ladder uses 1.96).
|
|
100
|
+
-> drvarma's list of dicts, one per series (w, sd, level, low, high)."""
|
|
101
|
+
from drvarma.forecast import forecast_given
|
|
102
|
+
return forecast_given(self.phi, self.theta, self.sigma, self.w, self.a, H,
|
|
103
|
+
const=self.const, det=self.det(H), lam=self.lambdas,
|
|
104
|
+
refactor=self.refactors, z=z_of(conf) if z is None else z)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def to_levels_varma(res):
|
|
108
|
+
"""FitResult -> LevelsVarma, at the fit's estimates, from the fit's origin
|
|
109
|
+
(the last observation it saw: with -estwin E, observation E + 1)."""
|
|
110
|
+
spec, data = res.spec, res.data
|
|
111
|
+
if spec.trend:
|
|
112
|
+
raise ValueError("-trend (case 4): drvec does not forecast it (-f refuses it), "
|
|
113
|
+
"and neither does the levels VARMA")
|
|
114
|
+
if spec.xlink:
|
|
115
|
+
raise ValueError("-xlink: drvec does not forecast it (-f refuses it), "
|
|
116
|
+
"and neither does the levels VARMA")
|
|
117
|
+
st = res.stationary
|
|
118
|
+
B2 = res.beta2
|
|
119
|
+
resid = res.residuals
|
|
120
|
+
M, r, p, q = spec.M, spec.r, spec.p, spec.q
|
|
121
|
+
s = M - r
|
|
122
|
+
n = resid.shape[0]
|
|
123
|
+
T0 = np.eye(M)
|
|
124
|
+
T0i = np.eye(M)
|
|
125
|
+
for j in range(r):
|
|
126
|
+
for i in range(s):
|
|
127
|
+
T0[s + j, i] = B2[i, j]
|
|
128
|
+
T0i[s + j, i] = -B2[i, j]
|
|
129
|
+
T1 = np.zeros((M, M))
|
|
130
|
+
T1[:s, :s] = -np.eye(s)
|
|
131
|
+
pb = np.asarray(st.phi, float).reshape(p, M, M)
|
|
132
|
+
tb = np.asarray(st.theta, float).reshape(q, M, M)
|
|
133
|
+
phi = np.zeros((p + 1, M, M))
|
|
134
|
+
for k in range(1, p + 2):
|
|
135
|
+
acc = np.zeros((M, M))
|
|
136
|
+
if k <= p:
|
|
137
|
+
acc += pb[k - 1] @ T0
|
|
138
|
+
if k == 1:
|
|
139
|
+
acc -= T1
|
|
140
|
+
else:
|
|
141
|
+
acc += pb[k - 2] @ T1
|
|
142
|
+
phi[k - 1] = T0i @ acc
|
|
143
|
+
theta = np.array([T0i @ tb[k] @ T0 for k in range(q)]).reshape(q, M, M)
|
|
144
|
+
mu = np.asarray(st.mu, float)
|
|
145
|
+
phi1 = np.eye(M) - pb.sum(axis=0)
|
|
146
|
+
const = T0i @ (phi1 @ mu)
|
|
147
|
+
sigma = T0i @ st.Sigma @ T0i.T
|
|
148
|
+
w = np.asarray(data.raw[:n + 1], float)
|
|
149
|
+
a = np.zeros((n + 1, M))
|
|
150
|
+
a[1:] = resid @ T0i.T
|
|
151
|
+
return LevelsVarma(phi, theta, sigma, const, w, a, list(data.names),
|
|
152
|
+
[float(v) for v in data.lambdas], [float(v) for v in data.refactors],
|
|
153
|
+
r, n + 1, data)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def forecast_levels(res, H, conf=0.95, z=None):
|
|
157
|
+
"""The fitted VEC forecast H steps by drvarma (to_levels_varma + forecast)."""
|
|
158
|
+
return to_levels_varma(res).forecast(H, conf, z)
|