drvec 0.1.0__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
drvec/api.py ADDED
@@ -0,0 +1,942 @@
1
+ """The public entry points: `fit` (one rank) and `rank_test` (r = 0..M-1)."""
2
+ from __future__ import annotations
3
+
4
+ import math
5
+ import os
6
+ from dataclasses import dataclass, field
7
+
8
+ import numpy as np
9
+
10
+ from .data import Data, read_pre
11
+ from .params import Spec, slot_labels
12
+ from .rank import fit_rank, ma_min_modulus
13
+ from .rank import rank_test as _rank_test
14
+ from .cast import shootx
15
+
16
+
17
+ @dataclass
18
+ class Param:
19
+ label: str # the C's .out wording, in the .inp's names
20
+ kind: str # mean, alpha, gamma, theta, q, beta
21
+ value: float # in the reader's notation (alpha = -Lambda)
22
+ se: float
23
+
24
+
25
+ @dataclass
26
+ class FitResult:
27
+ spec: Spec
28
+ names: list
29
+ nobs: int
30
+ x: list # the parameter vector, in drvec's internal layout
31
+ loglik: float
32
+ sigma2: float
33
+ termcode: int
34
+ iters: int
35
+ fk: float
36
+ se: list
37
+ search: object # fit.SearchResult
38
+ seed_info: object # fit.Profile (r >= 1) or fit.Ladder (r = 0)
39
+ ma_min_modulus: float
40
+ data: object = None # the data as read (the .out's header and diagnostics)
41
+ cov: object = None # the BFGS covariance of the winner (npar x npar)
42
+ drift_how: str = None # "pre" (-case pre) or "nodrift" when drifts are held
43
+ lr_free: float = None # -fixb2row, -xsys: logL of the free model
44
+ xs_df: int = None # -xsys: the parameters the embedding removes
45
+ gz_df: int = None # -gzero: the parameters the restriction removes (alpha's too)
46
+ xl_base_ll: float = None # -xlink: logL of the VEC without the inputs
47
+ xl_base_info: object = None # its ladder and profile (written to the .out)
48
+ xl_bridge_ll: float = None # logL at that optimum with omega = 0
49
+ irfboot: object = None # -irfboot: irfboot.IrfBoot
50
+ alpha_free: object = None # -alpha/-weakex: the est() of H(r) (fit.EstResult)
51
+ fdhess: object = None # -fdhess: fdhess.FdHess (se and cov are its when .ok)
52
+ forecast_h: int = 0 # -f H
53
+ estwin: int = 0 # -estwin E
54
+ gate: object = None # r = 0, all diagonal: fit.Gate with the certificate
55
+ provenance: object = None # results.Provenance: who computed the bootstrap
56
+ _fv: object = field(default=None, repr=False, compare=False)
57
+
58
+ # ------------------------------------------------------------------ #
59
+ # The accessors (STUDY-vec §4.2): the numbers the .out prints, as #
60
+ # arrays, from the routines that print them. See drvec.results. #
61
+ # ------------------------------------------------------------------ #
62
+
63
+ def _final(self):
64
+ if self._fv is None:
65
+ from .report import _Fit
66
+ self._fv = _Fit(self)
67
+ return self._fv
68
+
69
+ def _ro(self):
70
+ from .params import RowOrder
71
+ return RowOrder(self.spec.M, self.spec.r)
72
+
73
+ @property
74
+ def alpha(self):
75
+ """alpha (M x r), the adjustment, in Johansen's sign (negative is
76
+ error-correcting), rows in the .inp's order."""
77
+ fv, i2l = self._final(), self._ro().inp2lam
78
+ M, r = self.spec.M, self.spec.r
79
+ return np.array([[-fv.Lam[i2l(a)][b] for b in range(r)] for a in range(M)],
80
+ float).reshape(M, r)
81
+
82
+ @property
83
+ def beta2(self):
84
+ """B2 (s x r): beta = [B2 ; I_r], normalised on the Y1 block (held rows
85
+ and -fixb2's values included)."""
86
+ fv = self._final()
87
+ s, r = self.spec.s, self.spec.r
88
+ return np.array([[fv.B2[a][b] for b in range(r)] for a in range(s)],
89
+ float).reshape(s, r)
90
+
91
+ @property
92
+ def beta(self):
93
+ """beta (M x r), rows in the .inp's order: B2 on the Y2 block, I_r on Y1."""
94
+ fv = self._final()
95
+ M, s, r = self.spec.M, self.spec.s, self.spec.r
96
+ return np.array([[fv.B2[row][c] if row < s else (1.0 if row - s == c else 0.0)
97
+ for c in range(r)] for row in range(M)], float).reshape(M, r)
98
+
99
+ @property
100
+ def Pi(self):
101
+ """Pi = alpha beta' (M x M), the long run, as the .out computes it."""
102
+ fv, i2l = self._final(), self._ro().inp2lam
103
+ M, s, r = self.spec.M, self.spec.s, self.spec.r
104
+ Pi = np.zeros((M, M))
105
+ B2 = fv.B2
106
+ for a in range(M):
107
+ for b in range(M):
108
+ acc = 0.0
109
+ for j in range(r):
110
+ Bbj = B2[b][j] if b < s else (1.0 if b - s == j else 0.0)
111
+ acc -= fv.Lam[i2l(a)][j] * Bbj
112
+ Pi[a, b] = acc
113
+ return Pi
114
+
115
+ @property
116
+ def gamma(self):
117
+ """Gamma(1..p-1) (p - 1, M, M), on nabla Y_{t-k}, in the .inp's order."""
118
+ fv, i2l = self._final(), self._ro().inp2lam
119
+ M = self.spec.M
120
+ return np.array([[[fv.F[k][i2l(a)][i2l(b)] for b in range(M)] for a in range(M)]
121
+ for k in range(self.spec.nf)], float).reshape(self.spec.nf, M, M)
122
+
123
+ @property
124
+ def theta(self):
125
+ """Theta(1..q) (q, M, M), the MA on the innovations, in the .inp's order
126
+ (Theta(L) = I - Theta_1 L - ...)."""
127
+ fv, i2l = self._final(), self._ro().inp2lam
128
+ M, q = self.spec.M, self.spec.q
129
+ return np.array([[[fv.Th[k][i2l(a)][i2l(b)] for b in range(M)] for a in range(M)]
130
+ for k in range(q)], float).reshape(q, M, M)
131
+
132
+ def _qin(self, a, b):
133
+ Q = self._final().Q
134
+ return Q[a][b] if a >= b else Q[b][a]
135
+
136
+ @property
137
+ def Q(self):
138
+ """Q (M x M, symmetric), the innovation covariance up to scale
139
+ (Sigma = sigma2 Q), in the .inp's order."""
140
+ i2l = self._ro().inp2lam
141
+ M = self.spec.M
142
+ return np.array([[self._qin(i2l(a), i2l(b)) for b in range(M)] for a in range(M)],
143
+ float)
144
+
145
+ @property
146
+ def Sigma(self):
147
+ """Sigma = sigma2 Q (M x M), the innovation covariance, in the .inp's order."""
148
+ i2l = self._ro().inp2lam
149
+ M = self.spec.M
150
+ return np.array([[self.sigma2 * self._qin(i2l(a), i2l(b)) for b in range(M)]
151
+ for a in range(M)], float)
152
+
153
+ @property
154
+ def means(self):
155
+ """The .out's mean vector: E[W] (case 2), or E[nabla Y2] then E[W]
156
+ (case 3; a held drift is 0); empty in case 1."""
157
+ return np.array(self._final().mu_m, float)
158
+
159
+ @property
160
+ def delta(self):
161
+ """delta = E[nabla Y] (M), case 3 only (None otherwise), as printed."""
162
+ spec, fv = self.spec, self._final()
163
+ M, s = spec.M, spec.s
164
+ mu_m = fv.mu_m
165
+ if not (spec.case == 3 and len(mu_m) >= s):
166
+ return None
167
+ out = []
168
+ for a in range(M):
169
+ if a < s:
170
+ d = mu_m[a]
171
+ else:
172
+ d = 0.0
173
+ for i3 in range(s):
174
+ d -= fv.B2[i3][a - s] * mu_m[i3]
175
+ if spec.trend:
176
+ d += fv.dtr[a - s]
177
+ out.append(d)
178
+ return np.array(out, float)
179
+
180
+ @property
181
+ def residual_names(self):
182
+ """The names of the residual columns: Ybar's, the Y2 names then ec1..ecr."""
183
+ s = self.spec.s
184
+ return list(self.names[:s]) + ["ec%d" % (j + 1) for j in range(self.spec.r)]
185
+
186
+ @property
187
+ def residuals(self):
188
+ """The innovations A_t of the stationary VARMA (n x M), Ybar's columns
189
+ (residual_names): what the .out's residual diagnostics read."""
190
+ return np.array(self._final().a, float)
191
+
192
+ @property
193
+ def stationary(self):
194
+ """The stationary representation on Ybar (drvec.results.Stationary)."""
195
+ from .results import Stationary
196
+ fv = self._final()
197
+ return Stationary(self.residual_names, np.array(fv.mu, float),
198
+ np.array(fv.phi, float), np.array(fv.theta, float),
199
+ np.array(fv.qq, float), float(self.sigma2), np.array(fv.w, float))
200
+
201
+ def _irf_horizon(self, K):
202
+ return (10 if self._final().rd.n < 40 else 20) if K is None else int(K)
203
+
204
+ def _level_numbers(self, K=None):
205
+ from .report import level_irf_numbers
206
+ nb = level_irf_numbers(self, self._final(), self._irf_horizon(K))
207
+ if nb is not None and "error" in nb:
208
+ raise ValueError("the level responses are not defined: " + nb["error"])
209
+ return nb
210
+
211
+ def level_irf(self, K=None):
212
+ """The level responses (K + 1, M, M): [k, i, j] = the response of series i
213
+ at lag k to a one-s.d. orthogonalised shock to series j (Sigma = P D P'
214
+ in the .inp's order). K defaults to the .out's (10 when n < 40, else
215
+ 20). None when s = 0."""
216
+ nb = self._level_numbers(K)
217
+ return None if nb is None else np.array(nb["R"], float)
218
+
219
+ def fevd(self, K=None):
220
+ """The level FEVD in %% (K + 1, M, M): [k, i, j] = share of shock j in the
221
+ (k + 1)-step forecast error variance of series i. None when s = 0."""
222
+ nb = self._level_numbers(K)
223
+ return None if nb is None else np.array(nb["fevd"], float)
224
+
225
+ def long_run(self):
226
+ """The long-run gain and mean lag in levels (drvec.results.LongRun)."""
227
+ from .results import LongRun
228
+ nb = self._level_numbers(None)
229
+ if nb is None:
230
+ return None
231
+ ml = np.array([[np.nan if v is None else v for v in row] for row in nb["mean_lag"]],
232
+ float)
233
+ return LongRun(np.array(nb["gain"], float), ml, np.array(nb["shock_sd"], float),
234
+ nb["converged"], nb["k_stop"], nb["tail"], nb["beta_gain"])
235
+
236
+ def irf_bands(self):
237
+ """The -irfboot bands as arrays (drvec.results.IrfBands), or None without
238
+ -irfboot or with fewer than 20 good replications (the .out prints no
239
+ bands then)."""
240
+ if self.irfboot is None:
241
+ return None
242
+ from .irfboot import band_arrays
243
+ return band_arrays(self.irfboot, self.spec.M)
244
+
245
+ def diagnostics(self):
246
+ """Hosking's portmanteau and the multivariate Jarque-Bera of the
247
+ residuals, as the .out prints them (drvec.results.Diagnostics); None when
248
+ n < 20 (the .out has no such section)."""
249
+ from .report import diag_k, multivariate_numbers
250
+ from .results import Diagnostics
251
+ fv = self._final()
252
+ n, M = fv.rd.n, self.spec.M
253
+ if n < 20 or M < 1 or min(n // 4, 12) < 1:
254
+ return None
255
+ d = multivariate_numbers(fv.a, n, M, self.data.freq, diag_k(self.spec))
256
+ return Diagnostics(d["lags"], d["Q"], d["df"], d["pQ"], d["JB"], d["JB_df"],
257
+ d["pJB"], d["skewness"], d["kurtosis"], d["part_df"])
258
+
259
+ def forecasts(self, H=None, conf=0.95):
260
+ """The forecasts in levels as arrays (drvec.results.Forecasts): the
261
+ numbers of <base>.forecast. H defaults to fit(forecast=H)'s; a fit with
262
+ -estwin forecasts from the end of its window, as the C does."""
263
+ from .forecast import forecast_arrays
264
+ from .results import Forecasts
265
+ H = self.forecast_h if H is None else int(H)
266
+ if H < 1:
267
+ raise ValueError("forecasts need a horizon: H >= 1 (or fit(forecast=H))")
268
+ d = forecast_arrays(self, self._final(), H, conf)
269
+ return Forecasts(d["dates"], list(self.names), np.array(d["level"], float),
270
+ np.array(d["low"], float), np.array(d["high"], float),
271
+ np.array(d["se"], float), d["conf"], d["z"],
272
+ np.array(d["model_level"], float), np.array(d["model_se"], float))
273
+
274
+ @property
275
+ def restriction_lr(self):
276
+ """The restriction's LR test as the .out prints it
277
+ (drvec.results.RestrictionLR), or None when the fit carries none (a
278
+ free fit, and -fixb2 alone, for which the C prints no LR)."""
279
+ from .report import restriction_lr
280
+ from .results import RestrictionLR
281
+ d = restriction_lr(self)
282
+ return None if d is None else RestrictionLR(**d)
283
+
284
+ def forecast_text(self, base):
285
+ """<base>.forecast, as the C writes it (None without -f)."""
286
+ if not self.forecast_h:
287
+ return None
288
+ from .forecast import forecast_vec
289
+ return forecast_vec(self, self._final(), self.forecast_h, base)[1]
290
+
291
+ def recursive_text(self):
292
+ """<base>.recursive, the per-origin errors of -estwin (None without it,
293
+ or when the window leaves no room for the horizon)."""
294
+ if not (self.estwin and self.forecast_h):
295
+ return None
296
+ from .forecast import rolling_eval
297
+ rv = rolling_eval(self, self.estwin, self.forecast_h)
298
+ return rv[1] if rv else None
299
+
300
+ def irfboot_text(self, base):
301
+ """<base>.irfboot, as the C writes it (None without -irfboot)."""
302
+ if self.irfboot is None:
303
+ return None
304
+ from .irfboot import bands_text
305
+ return bands_text(self.irfboot, self.spec, self.names, base)[1]
306
+
307
+ @property
308
+ def npar(self):
309
+ return len(self.x)
310
+
311
+ @property
312
+ def aic(self):
313
+ return (-2.0 * self.loglik + 2.0 * self.npar) / self.nobs
314
+
315
+ @property
316
+ def bic(self):
317
+ return (-2.0 * self.loglik + self.npar * math.log(self.nobs)) / self.nobs
318
+
319
+ @property
320
+ def ma_on_boundary(self):
321
+ """BUG-49: an MA root on the unit circle; the .out prints no s.e. for Theta."""
322
+ return self.ma_min_modulus < 1.0001
323
+
324
+ @property
325
+ def ladder(self):
326
+ si = self.seed_info
327
+ return getattr(si, "ladder", si)
328
+
329
+ @property
330
+ def params(self):
331
+ labs = slot_labels(self.spec, self.names,
332
+ self.data.xl.names if (self.data is not None and self.data.xl)
333
+ else None)
334
+ return [Param(lab, kind, sg * v, se)
335
+ for (lab, kind, sg), v, se in zip(labs, self.x, self.se)]
336
+
337
+ def out_text(self, base):
338
+ """<base>.out, as the C writes it (report.fit_out)."""
339
+ from .report import fit_out
340
+ return fit_out(self, base)
341
+
342
+ def param(self, label):
343
+ for p in self.params:
344
+ if p.label == label:
345
+ return p
346
+ raise KeyError(label)
347
+
348
+
349
+ def _data(pre_files):
350
+ return pre_files if isinstance(pre_files, Data) else read_pre(pre_files)
351
+
352
+
353
+ ENGINES = ("auto", "python", "c")
354
+
355
+
356
+ def _choose_engine(engine, seed, files, what):
357
+ """-> (CBinary or None, note). engine None reads DRVEC_ENGINE (default
358
+ "auto"). "auto": the C when a matching binary is found and the call can
359
+ go to it, else Python, with the reason in the note. "c": the C or an error."""
360
+ from . import c_engine
361
+ from .rng import SEED
362
+ if engine is None:
363
+ engine = os.environ.get("DRVEC_ENGINE", "").strip() or "auto"
364
+ if engine not in ENGINES:
365
+ raise ValueError(f"engine must be one of {', '.join(ENGINES)}, got {engine!r}")
366
+ if engine == "python":
367
+ return None, "engine='python'"
368
+ why = None
369
+ if seed is not None and int(seed) != SEED:
370
+ why = (f"the C's bootstrap generator is fixed at its seed {SEED}; seed={seed} "
371
+ "runs in Python")
372
+ elif not files:
373
+ why = "the data were given as a Data object, not as .pre files the C can read"
374
+ if why is None:
375
+ try:
376
+ return c_engine.find(), f"engine={engine!r}"
377
+ except c_engine.CEngineUnavailable as e:
378
+ why = str(e)
379
+ if engine == "c":
380
+ raise c_engine.CEngineUnavailable(f"{what} cannot run on drvec C: {why}")
381
+ return None, "engine='auto', Python: " + why
382
+
383
+
384
+ def _files_of(pre_files):
385
+ if isinstance(pre_files, Data):
386
+ return None
387
+ if isinstance(pre_files, (str, bytes)) or hasattr(pre_files, "__fspath__"):
388
+ return None
389
+ return [os.fsdecode(f) for f in pre_files]
390
+
391
+
392
+ def load_alpha_A(path, M):
393
+ """load_alpha_A (drvec.c:2665): a first line `M sa`, then M rows of sa
394
+ numbers; lines beginning with '*' or '#' (or empty) are comments. A'A must
395
+ pass choldcp (psi identified). -> A as a tuple of rows."""
396
+ with open(path) as f:
397
+ lines = f.read().splitlines(keepends=True)
398
+ it = iter(lines)
399
+ mm = sa = 0
400
+ got = False
401
+ for line in it:
402
+ if line[:1] in ("*", "#", "\n") or line == "":
403
+ continue
404
+ tok = line.split()
405
+ try:
406
+ mm, sa = int(tok[0]), int(tok[1])
407
+ got = True
408
+ break
409
+ except (ValueError, IndexError):
410
+ continue
411
+ if not got or mm != M or sa < 1 or sa > M:
412
+ raise ValueError(f"{path} must start with 'M sa', with M = {M} and "
413
+ f"1 <= sa <= {M} (leido {mm} {sa})")
414
+ A = []
415
+ for i in range(M):
416
+ while True:
417
+ line = next(it, None)
418
+ if line is None:
419
+ raise ValueError(f"{path} ends at row {i + 1}")
420
+ if line[:1] not in ("*", "#", "\n"):
421
+ break
422
+ tok = line.split()
423
+ if len(tok) < sa:
424
+ raise ValueError(f"{path}, fila {i + 1}: faltan columnas")
425
+ A.append(tuple(_atof(t) for t in tok[:sa]))
426
+ return tuple(A)
427
+
428
+
429
+ def _atof(t):
430
+ """C's atof: the longest numeric prefix, 0.0 when there is none."""
431
+ import re
432
+ m = re.match(r"\s*[-+]?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?", t)
433
+ return float(m.group(0)) if m else 0.0
434
+
435
+
436
+ def _alpha_matrix(M, alpha, weakex):
437
+ """-> (A, weakex): -weakex i (build_weakex_A, drvec.c:2727) or -alpha (a
438
+ path, or the matrix), with A's rank checked through A'A (choldcp)."""
439
+ from .nla import choldcp
440
+ if weakex:
441
+ if not 1 <= weakex <= M:
442
+ raise ValueError(f"-weakex {weakex} is outside 1..{M}")
443
+ A = tuple(tuple(1.0 if (i != weakex - 1 and c == (i if i < weakex - 1 else i - 1))
444
+ else 0.0 for c in range(M - 1)) for i in range(M))
445
+ return A, int(weakex)
446
+ if isinstance(alpha, (str, bytes)) or hasattr(alpha, "__fspath__"):
447
+ A = load_alpha_A(alpha, M)
448
+ else:
449
+ A = tuple(tuple(float(v) for v in row) for row in alpha)
450
+ if len(A) != M or not A[0] or any(len(row) != len(A[0]) for row in A) \
451
+ or len(A[0]) > M:
452
+ raise ValueError(f"A must be M x sa with M = {M} and 1 <= sa <= {M}")
453
+ sa = len(A[0])
454
+ G = [[sum(A[k][i] * A[k][j] for k in range(M)) for j in range(sa)] for i in range(sa)]
455
+ # The C's own check is choldcp's verdict, which only faults on a NEGATIVE
456
+ # pivot: an exactly singular A'A passes, and the fit runs on NaN (psi's
457
+ # seed divides by zero) and exits 0 -- a C defect (bugs/drvec-c-alpha-rank).
458
+ # The port refuses it with the C's message, as the C means to.
459
+ import numpy as np
460
+ if choldcp(G)[1] > 0 or np.linalg.matrix_rank(np.array(A, float)) < sa:
461
+ raise ValueError(f"A does not have rank {sa} (A'A is singular), so psi\n"
462
+ " is not identified")
463
+ return A, 0
464
+
465
+
466
+ def _drift_mask(data, case, nodrift):
467
+ """-> (case, drift mask, how): -case pre reads each .pre's mean flag
468
+ (drvec.c:6905-6925); -nodrift i holds series i's drift at 0."""
469
+ M = data.M
470
+ if case == "pre":
471
+ mask = []
472
+ for i in range(M):
473
+ held = not data.pre_mean_free[i]
474
+ if held and abs(data.pre_mu[i]) > 1.0e-12:
475
+ held = False # a mean held at a non-zero value goes free
476
+ mask.append(not held)
477
+ return 3, (tuple(mask) if not all(mask) else None), "pre"
478
+ if nodrift:
479
+ if case != 3:
480
+ raise ValueError("-nodrift holds a drift of case 3 (-case 3 or -case pre); in "
481
+ "cases 1 and 2 no series has one.")
482
+ mask = [True] * M
483
+ for i in nodrift:
484
+ if not 1 <= i <= M:
485
+ raise ValueError(f"-nodrift {i}: the system has {M} series")
486
+ mask[i - 1] = False
487
+ return 3, tuple(mask), "nodrift"
488
+ return case, None, None
489
+
490
+
491
+ MSG_GZERO = ("-gzero holds entries of Gamma(k) for a single fit with q = 0\n"
492
+ " and p >= 2 (at p = 1 there is no Gamma). It combines with\n"
493
+ " -alpha/-weakex, not with -xsys, -fixb2row, -fixb2, -xlink,\n"
494
+ " -diagar, -warma, a moving average, or the modes -lrtest,\n"
495
+ " -specs, -rungs, -matest, -artest and -eval.")
496
+
497
+
498
+ def fit(pre_files, p, q, r, case=2, ma="free", diag_ar=False, diag_cov=False,
499
+ fixb2row=None, nodrift=None, trend=False, irfboot=0, xsys=None,
500
+ xpre=None, xlink=None, forecast=0, fdhess=False, alpha=None, weakex=None,
501
+ fixb2=None, estwin=0, multistart=0, log=None, seed=None, workers=None,
502
+ engine=None, folder=None, progress=None, gzero=None):
503
+ """Estimate the VARMA-VEC of rank r on the `.pre` files (the C's
504
+ `drvec f1.pre ... p q r -case <case>`), with the C's ladder and search.
505
+
506
+ case: 1, 2, 3 or "pre" (case 3 with each drift free or held as its .pre
507
+ says). fixb2row: {row: value} (1-based rows of B2, the nabla Y2 block),
508
+ the C's -fixb2row i v; the free model is fitted too, for the LR. nodrift:
509
+ 1-based series whose drift is held at 0 (-nodrift). trend: -trend (case 4).
510
+ irfboot: -irfboot N, bands for the level responses and the decomposition by
511
+ the parametric bootstrap. fdhess: -fdhess, the standard errors from the
512
+ finite-difference Hessian at the optimum. alpha: -alpha FILE (a path, or the
513
+ M x sa matrix A itself), weakex: -weakex i (1-based): alpha = A psi, with the
514
+ LR against the free model. fixb2: -fixb2 v (a number, an a-priori value for
515
+ every entry of B2) or True (-fixb2 alone: B2 held at its static-OLS value).
516
+ forecast: -f H, H steps in levels with their theoretical bands (the .out's
517
+ section and FitResult.forecast_text). estwin: -estwin E, the fit on rows
518
+ 1..E and, with -f H, the rolling-origin evaluation (recursive_text).
519
+ multistart: -multistart n, n - 1 jittered starts added to the search.
520
+ gzero: -gzero i j, pairs (i, j) (1-based, the input's order): Gamma_k[i][j]
521
+ = 0 for every k, series j out of equation i's short run; with alpha/weakex
522
+ the conditional model's joint restriction. The free VEC (no mask, alpha
523
+ free) is fitted too, for the LR.
524
+
525
+ The -irfboot bootstrap: seed (None: the C's 987654321), workers (the
526
+ process pool for the refits; None: DRVEC_WORKERS or the cores less one;
527
+ 1: in this process; the numbers do not depend on it), engine ("auto",
528
+ "python" or "c"; None reads DRVEC_ENGINE, default "auto": drvec C when a
529
+ matching binary is found, see drvec.c_engine), folder (where the C runs and
530
+ leaves its files; None: a temporary folder), progress(b, ok) after each
531
+ replication (Python engine; the C reports at the end). The engine used is
532
+ in FitResult.provenance."""
533
+ if (xpre or xlink) and not (xpre and xlink):
534
+ raise ValueError("-xpre and -xlink go together: the inputs and the\n"
535
+ " network of links that says where they enter.")
536
+ if xlink and (forecast or estwin):
537
+ raise ValueError(
538
+ "-xlink needs the .pre route in the levels layout, and a\n"
539
+ " single fit: it cannot be combined with -differenced, -warma,\n"
540
+ " -lrtest, -specs, -rungs, -matest, -artest, -eval, -f, -estwin,\n"
541
+ " -writeinp or -writeres.")
542
+ if trend and (forecast or estwin):
543
+ raise ValueError(
544
+ "-trend (case 4) goes on case 3's drifts (-case 3 or -case pre)\n"
545
+ " for a single fit: it cannot be combined with -warma, -f,\n"
546
+ " -estwin, -writeinp, -writeres or the modes -lrtest, -specs,\n"
547
+ " -rungs, -matest, -artest and -eval (not yet).")
548
+ if irfboot and estwin:
549
+ raise ValueError(
550
+ "-irfboot bootstraps the responses of a single fit in the levels\n"
551
+ " layout: it cannot be combined with -differenced, -xlink, -trend,\n"
552
+ " -warma, -estwin, -writeinp, -writeres or the modes -lrtest,\n"
553
+ " -specs, -rungs, -matest, -artest and -eval (not yet).")
554
+ if xsys and (alpha is not None or weakex):
555
+ raise ValueError(
556
+ "-xsys embeds an input in the system for a single fit with\n"
557
+ " q = 0. It cannot be combined with -alpha/-weakex (it sets\n"
558
+ " alpha itself), -xlink, -warma, a moving average (q > 0, not\n"
559
+ " yet), or the modes -lrtest, -specs, -rungs, -matest, -artest\n"
560
+ " and -eval.")
561
+ if fixb2row and fixb2 is not None and fixb2 is not False:
562
+ raise ValueError(
563
+ "-fixb2row holds rows of B2 at a single fit. It cannot be\n"
564
+ " combined with -fixb2 (which holds every row), -warma, or the\n"
565
+ " modes -lrtest, -specs, -rungs, -matest, -artest and -eval.")
566
+ if xlink and case == 1:
567
+ raise ValueError("-xlink needs a free mean of W (-case 2 or 3): the input\n"
568
+ " keeps its mean, and nu(1) times it goes to E[W].")
569
+ if xlink and (xsys or irfboot):
570
+ raise ValueError(
571
+ "-irfboot bootstraps the responses of a single fit in the levels\n"
572
+ " layout: it cannot be combined with -differenced, -xlink, -trend,\n"
573
+ " -warma, -estwin, -writeinp, -writeres or the modes -lrtest,\n"
574
+ " -specs, -rungs, -matest, -artest and -eval (not yet)." if irfboot else
575
+ "-xsys embeds an input in the system for a single fit with\n"
576
+ " q = 0. It cannot be combined with -alpha/-weakex (it sets\n"
577
+ " alpha itself), -xlink, -warma, a moving average (q > 0, not\n"
578
+ " yet), or the modes -lrtest, -specs, -rungs, -matest, -artest\n"
579
+ " and -eval.")
580
+ if irfboot and trend:
581
+ raise ValueError(
582
+ "-irfboot bootstraps the responses of a single fit in the levels\n"
583
+ " layout: it cannot be combined with -differenced, -xlink, -trend,\n"
584
+ " -warma, -estwin, -writeinp, -writeres or the modes -lrtest,\n"
585
+ " -specs, -rungs, -matest, -artest and -eval (not yet).")
586
+ if gzero and (xsys or xlink or fixb2row or diag_ar or q > 0 or p < 2
587
+ or (fixb2 is not None and fixb2 is not False)):
588
+ raise ValueError(MSG_GZERO)
589
+ case0 = case
590
+ data = _data(pre_files)
591
+ gz = ()
592
+ if gzero:
593
+ for i, j in gzero:
594
+ if int(i) == int(j):
595
+ raise ValueError(f"-gzero {i} {j}: the diagonal of Gamma is the series' own "
596
+ "model\n (the ladder's rung), and is never held.")
597
+ if not (1 <= int(i) <= data.M and 1 <= int(j) <= data.M):
598
+ raise ValueError(f"-gzero {i} {j}: the system has {data.M} series")
599
+ gz = tuple(sorted(set((int(i) - 1, int(j) - 1) for i, j in gzero)))
600
+ if estwin:
601
+ # P5.2: the fit on rows 1..E of nabla Y, the rest kept for the
602
+ # rolling evaluation (drvec.c:740-747)
603
+ nfull = data.nobs_raw - 1
604
+ if estwin < 10 or estwin >= nfull:
605
+ raise ValueError(f"-estwin {estwin} is not inside 10..{nfull - 1}")
606
+ data = data.with_raw(data.raw)
607
+ data.estwin = int(estwin)
608
+ links = ()
609
+ if xlink:
610
+ from .xlink import load
611
+ data = data.with_raw(data.raw)
612
+ links, data.xl = load(data, list(xpre) if not isinstance(xpre, str) else [xpre],
613
+ xlink)
614
+ xs = ()
615
+ if xsys:
616
+ if q > 0 or fixb2row:
617
+ raise ValueError(
618
+ "-xsys embeds an input in the system for a single fit with\n"
619
+ " q = 0. It cannot be combined with -alpha/-weakex (it sets\n"
620
+ " alpha itself), -xlink, -warma, a moving average (q > 0, not\n"
621
+ " yet), or the modes -lrtest, -specs, -rungs, -matest, -artest\n"
622
+ " and -eval.")
623
+ M = data.M
624
+ if len(set(xsys)) >= M:
625
+ raise ValueError("-xsys declares every series an input: there is nothing "
626
+ "left to explain.")
627
+ for i in xsys:
628
+ if not 1 <= i <= M:
629
+ raise ValueError(f"-xsys {i}: the system has {M} series")
630
+ if i > M - r:
631
+ raise ValueError(
632
+ f"-xsys {i}: series {i} is in the Y1 block (the last {r} of\n"
633
+ " the input), on which beta is normalised. An input does\n"
634
+ " not adjust to the equilibrium, so it cannot be the\n"
635
+ " series a relation is solved for: put it in the nabla Y2\n"
636
+ f" block (among the first {M - r}).")
637
+ xs = tuple(sorted(set(i - 1 for i in xsys)))
638
+ case, drift, how = _drift_mask(data, case, nodrift)
639
+ if trend and case != 3:
640
+ raise ValueError("-trend (case 4) goes on case 3's drifts (-case 3 or -case pre)")
641
+ hold = ()
642
+ if fixb2row:
643
+ if r < 1:
644
+ raise ValueError("-fixb2row needs r >= 1: at r = 0 there is no beta to restrict.")
645
+ s = data.M - r
646
+ for i in fixb2row:
647
+ if not 1 <= i <= s:
648
+ raise ValueError(f"-fixb2row {i}: B2 has {s} row{'' if s == 1 else 's'} (the "
649
+ f"nabla Y2 block, the first {s} series in the input's order).")
650
+ hold = tuple(sorted((i - 1, float(v)) for i, v in fixb2row.items()))
651
+ A, wx = (), 0
652
+ if alpha is not None or weakex:
653
+ A, wx = _alpha_matrix(data.M, alpha, weakex)
654
+ if r < 1:
655
+ raise ValueError("alpha = A*psi means nothing with r = 0 (there is no "
656
+ "error-correction term)")
657
+ spec = Spec(data.M, p, q, r, case, ma, diag_ar, diag_cov, drift, trend, hold, xsys=xs,
658
+ alpha=A, weakex=wx, gzero=gz)
659
+ if fixb2 is not None and fixb2 is not False and r > 0:
660
+ if fixb2 is True:
661
+ # -fixb2 alone: B2 at the static-OLS estimate init_guess computes
662
+ # (drvec.c:4253-4262), a data-chosen restriction
663
+ from .start import prelim_b2
664
+ B = prelim_b2(spec, data.at(r))
665
+ bf = tuple(tuple(float(B[i][j]) for j in range(r)) for i in range(spec.s))
666
+ spec = spec.but(b2_fix=bf, fixb2_given=False)
667
+ else:
668
+ v = float(fixb2)
669
+ spec = spec.but(b2_fix=tuple((v,) * r for _ in range(spec.s)), fixb2_given=True)
670
+ if spec.npar() >= data.nobs * data.M:
671
+ raise ValueError(f"the model has {spec.npar()} parameters and the sample "
672
+ f"{data.nobs * data.M} data points")
673
+ lr_free = None
674
+ alpha_free = None
675
+ if hold:
676
+ # the free model, as the C fits it for the LR: init_guess and the
677
+ # search, no ladder (drvec.c:11351-11380); with -alpha/-weakex the
678
+ # restriction on alpha is off too (the joint test)
679
+ from .fit import fit_search
680
+ from .start import init_guess
681
+ sf = spec.but(b2_hold=(), alpha=(), weakex=0)
682
+ rdf = data.at(r)
683
+ rf = fit_search(sf, rdf, init_guess(sf, rdf))
684
+ lr_free = rf.ll if rf.ifault == 0 else None
685
+ elif spec.alpha and not gz:
686
+ # H(r) for the LR of alpha = A psi: ONE est() from init_guess, no
687
+ # search and no ladder, its banner written to the .out
688
+ # (drvec.c:11415-11439)
689
+ from .fit import est
690
+ from .start import init_guess
691
+ sf = spec.but(alpha=(), weakex=0)
692
+ rdf = data.at(r)
693
+ alpha_free = est(sf, rdf, init_guess(sf, rdf))
694
+ lr_free = alpha_free.ll if alpha_free.ifault == 0 else None
695
+ xs_df = None
696
+ xl_base = None
697
+ if links:
698
+ spec = spec.but(xlink=links)
699
+ if links:
700
+ # the VEC rung below, without the inputs: its ladder and profile are
701
+ # written to the .out, its search is not (drvec.c:11262-11296); its
702
+ # optimum with omega = 0 is the start, and the bridge is evaluated
703
+ from .loglik import loglik
704
+ sb = spec.but(xlink=())
705
+ rb, infob = fit_rank(sb, data)
706
+ xl_base = (rb.ll if rb.ifault == 0 else None, infob)
707
+ x0 = list(rb.x) + [0.0] * (spec.npar() - sb.npar())
708
+ llb, _s2, ifb = loglik(spec, data.at(r), x0)
709
+ xl_bridge = (llb if ifb == 0 else None)
710
+ from .fit import fit_search
711
+ res = fit_search(spec, data.at(r), x0, njitter=multistart if multistart > 1 else 1,
712
+ log=log)
713
+ info = None
714
+ if xs:
715
+ # the free VEC, as the C fits it for the LR of strong exogeneity:
716
+ # the ladder start and the search (drvec.c:11383-11414)
717
+ sf = spec.but(xsys=())
718
+ rf, _ = fit_rank(sf, data)
719
+ lr_free = rf.ll if rf.ifault == 0 else None
720
+ xs_df = sf.npar() - spec.npar()
721
+ gz_df = None
722
+ if gz:
723
+ # -gzero (with -alpha/-weakex if given): the free VEC, no mask and alpha
724
+ # free, with the ladder start and the search, as -xsys's
725
+ sf = spec.but(gzero=(), alpha=(), weakex=0)
726
+ rf, _ = fit_rank(sf, data)
727
+ lr_free = rf.ll if rf.ifault == 0 else None
728
+ gz_df = sf.npar() - spec.npar()
729
+ if not links:
730
+ res, info = fit_rank(spec, data, log, njitter=multistart if multistart > 1 else 1)
731
+ if res.ifault != 0:
732
+ raise RuntimeError(f"ESTIMATION FAILED: ifault = {res.ifault}")
733
+ rd = data.at(r)
734
+ _mu, _phi, theta, _qq, _w, _ = shootx(spec, rd, res.x)
735
+ e = res.best.est
736
+ out = FitResult(spec, list(data.names), rd.n, list(res.x), res.ll, res.s2,
737
+ e.termcode, e.iters, e.fk, list(e.dev), res, info,
738
+ ma_min_modulus(theta), data, e.cov)
739
+ out.drift_how = how
740
+ if r == 0 and spec.diag_ar and spec.ma == "diag" and spec.diag_cov and not links:
741
+ # the diagonal rung as a plain fit: the entry gate with the optimality
742
+ # certificate, the logL AT the start brought in (drvec.c:11571-11595)
743
+ from .fit import gate_contract
744
+ from .loglik import XITOL, concentrated, elf
745
+ out.gate = gate_contract(spec, rd, res.x, res.ll)
746
+ f1, f2, _a, ifs = elf(spec, rd, res.seed, XITOL, False)
747
+ if ifs == 0:
748
+ out.gate.ll_start = concentrated(spec.M, rd.n, f1, f2)
749
+ out.forecast_h = int(forecast or 0)
750
+ out.estwin = int(estwin or 0)
751
+ out.alpha_free = alpha_free
752
+ out.lr_free = lr_free
753
+ out.xs_df = xs_df
754
+ out.gz_df = gz_df
755
+ if links:
756
+ out.xl_base_ll, out.xl_base_info = xl_base
757
+ out.xl_bridge_ll = xl_bridge
758
+ if fdhess:
759
+ # report_fit's first step (drvec.c:7805): the s.e. and the covariance
760
+ # every Wald test reads are replaced when the Hessian is usable
761
+ from .fdhess import exact_hessian_se
762
+ out.fdhess = exact_hessian_se(spec, rd, out.x)
763
+ if out.fdhess.ok:
764
+ out.se = list(out.fdhess.dev)
765
+ out.cov = out.fdhess.cov
766
+ if irfboot > 0:
767
+ K = 10 if rd.n < 40 else 20
768
+ files = _files_of(pre_files)
769
+ binary, note = _choose_engine(engine, seed, files, "-irfboot")
770
+ if binary is not None:
771
+ out.irfboot, out.provenance = _irfboot_c(
772
+ binary, note, files, spec, data, out.x, irfboot, K, folder, progress,
773
+ dict(p=p, q=q, r=r, case=case0, ma=ma, diag_ar=diag_ar, diag_cov=diag_cov, fixb2row=fixb2row,
774
+ nodrift=nodrift, xsys=xsys, alpha=alpha, weakex=weakex, fixb2=fixb2,
775
+ multistart=multistart, gzero=gzero))
776
+ else:
777
+ from .irfboot import irf_bootstrap
778
+ from .parallel import resolve_workers
779
+ from .results import python_provenance
780
+ from .rng import Rng
781
+ nw = resolve_workers(workers, irfboot)
782
+ out.irfboot = irf_bootstrap(spec, data, out.x, irfboot, K,
783
+ rng=Rng() if seed is None else Rng(seed),
784
+ progress=progress, workers=nw)
785
+ out.provenance = python_provenance(nw, seed, note)
786
+ if out.provenance is None:
787
+ from .results import python_provenance
788
+ out.provenance = python_provenance(1, None, "no bootstrap")
789
+ return out
790
+
791
+
792
+ def _irfboot_c(binary, note, files, spec, data, x, N, K, folder, progress, kw):
793
+ """fit(irfboot=N) on drvec C: the bands from its files (drvec.c_engine)."""
794
+ from . import c_engine
795
+ from .irfboot import IrfBoot, _irf_at
796
+ base = "drvecjob"
797
+ with c_engine.Folder(folder) as d:
798
+ argv = c_engine.fit_argv(files, kw.pop("p"), kw.pop("q"), kw.pop("r"), d,
799
+ irfboot=N, base=base, **kw)
800
+ job = c_engine.Job(binary, argv, d, base)
801
+ try:
802
+ job.wait()
803
+ finally:
804
+ if job.proc.poll() is None:
805
+ job.proc.kill()
806
+ with open(job.path(".out")) as f:
807
+ otext = f.read()
808
+ ftext = None
809
+ if os.path.exists(job.path(".irfboot")):
810
+ with open(job.path(".irfboot")) as f:
811
+ ftext = f.read()
812
+ nok, nfail, sect, bands = c_engine.read_irfboot(otext, ftext, data.names, K, base)
813
+ prov = c_engine.provenance(binary, d if folder else None, argv, job.elapsed, note)
814
+ if progress:
815
+ progress(N, True)
816
+ bt = IrfBoot(N, K, _irf_at(spec, data, x, K), [], [], nfail, c_nok=nok, c_bands=bands,
817
+ c_out=sect, c_file=ftext)
818
+ bt.c_base = base
819
+ return bt, prov
820
+
821
+
822
+ def rank_test(pre_files, p, q, case=2, ma="free", bootstrap=0, boot_path=None,
823
+ progress=None, trend=False, xsys=None, xpre=None, xlink=None, multistart=0,
824
+ nodrift=None, seed=None, workers=None, engine=None, folder=None):
825
+ """The sequential LR test for the rank (`-lrtest [-bootstrap N]`).
826
+
827
+ case="pre" and nodrift= are the mask on case 3's drifts, as in fit(): every
828
+ rank, observed or simulated, is fitted with it (the C's global mask); the
829
+ asymptotic tables are case 3's and do not apply to the mixed case.
830
+
831
+ The C declares the rank test with exogenous inputs (-xsys, -xlink) and with
832
+ -trend as not done yet; so does the port, with its messages.
833
+
834
+ The bootstrap: progress(b, replication) after each replication (its .h0 is
835
+ the comparison); seed (None: the C's 987654321); workers (the process pool
836
+ for the refits; None: DRVEC_WORKERS or the cores less one; 1: in this
837
+ process; the numbers do not depend on it); engine ("auto", "python" or "c";
838
+ None reads DRVEC_ENGINE, default "auto": drvec C when a matching binary is
839
+ found, see drvec.c_engine); folder (where the C runs and leaves its files;
840
+ None: a temporary folder). The engine used is in RankTest.provenance."""
841
+ if xlink or xpre:
842
+ raise ValueError(
843
+ "-xlink needs the .pre route in the levels layout, and a\n"
844
+ " single fit: it cannot be combined with -differenced, -warma,\n"
845
+ " -lrtest, -specs, -rungs, -matest, -artest, -eval, -f, -estwin,\n"
846
+ " -writeinp or -writeres.")
847
+ if xsys:
848
+ raise ValueError(
849
+ "-xsys embeds an input in the system for a single fit with\n"
850
+ " q = 0. It cannot be combined with -alpha/-weakex (it sets\n"
851
+ " alpha itself), -xlink, -warma, a moving average (q > 0, not\n"
852
+ " yet), or the modes -lrtest, -specs, -rungs, -matest, -artest\n"
853
+ " and -eval.")
854
+ if trend:
855
+ raise ValueError(
856
+ "-trend (case 4) goes on case 3's drifts (-case 3 or -case pre)\n"
857
+ " for a single fit: it cannot be combined with -warma, -f,\n"
858
+ " -estwin, -writeinp, -writeres or the modes -lrtest, -specs,\n"
859
+ " -rungs, -matest, -artest and -eval (not yet).")
860
+ data = _data(pre_files)
861
+ case0 = case
862
+ case, drift, how = _drift_mask(data, case, nodrift)
863
+ spec = Spec(data.M, p, q, 0, case, ma, drift=drift)
864
+ nj = multistart if multistart > 1 else 1
865
+ from .results import python_provenance
866
+ if bootstrap <= 0:
867
+ out = _rank_test(data, spec, 0, boot_path, progress, njitter=nj)
868
+ out.drift_how = how
869
+ out.provenance = python_provenance(1, None, "no bootstrap")
870
+ return out
871
+ files = _files_of(pre_files)
872
+ binary, note = _choose_engine(engine, seed, files, "the rank test's bootstrap")
873
+ if binary is None:
874
+ from .parallel import resolve_workers
875
+ nw = resolve_workers(workers, bootstrap)
876
+ out = _rank_test(data, spec, bootstrap, boot_path, progress, njitter=nj,
877
+ seed=seed, workers=nw)
878
+ out.drift_how = how
879
+ out.provenance = python_provenance(nw, seed, note)
880
+ return out
881
+ from . import c_engine
882
+ from .rank import Bootstrap, not_i1r, summarise
883
+ base = "drvecjob"
884
+ with c_engine.Folder(folder) as d:
885
+ argv = c_engine.rank_argv(files, p, q, case=case0, ma=ma, bootstrap=bootstrap,
886
+ nodrift=nodrift, multistart=multistart, base=base)
887
+ job = c_engine.Job(binary, argv, d, base)
888
+
889
+ def hook(t, ds):
890
+ # the port's observed fits are done (they ran while the C did)
891
+ def line(s):
892
+ r = c_engine.parse_boot_line(s)
893
+ if r is not None and progress:
894
+ progress(r.rep, r)
895
+ try:
896
+ job.wait(on_boot_line=line)
897
+ finally:
898
+ if job.proc.poll() is None:
899
+ job.proc.kill()
900
+ reps = c_engine.read_boot(job.path(".boot"))
901
+ pairs_ok = [rr for rr in range(t.spec.M - 1)
902
+ if not (t.fits[rr].ifault or t.fits[rr + 1].ifault)]
903
+ # the C draws nothing for a model that is not I(1, r): no lines
904
+ noti1 = not_i1r(t.spec, ds, t.fits, pairs_ok)
905
+ want = [rr for rr in pairs_ok if rr not in noti1]
906
+ if sorted(reps) != want:
907
+ raise c_engine.CEngineError(
908
+ "drvec C bootstrapped the comparisons %s and the port's fits allow %s: "
909
+ "the two fits disagree (%s)" % (sorted(reps), want, d))
910
+ bs = []
911
+ for rr in pairs_ok:
912
+ if rr in noti1:
913
+ bs.append(Bootstrap(rr, t.lr(rr), [], i1r=noti1[rr]))
914
+ continue
915
+ if len(reps[rr]) != bootstrap:
916
+ raise c_engine.CEngineError(
917
+ f"drvec C wrote {len(reps[rr])} replications for H0: r = {rr}, "
918
+ f"not {bootstrap} ({d})")
919
+ bs.append(summarise(rr, t.lr(rr), reps[rr]))
920
+ return bs
921
+ try:
922
+ out = _rank_test(data, spec, bootstrap, boot_path, None, njitter=nj,
923
+ boot_hook=hook)
924
+ finally:
925
+ if job.proc.poll() is None:
926
+ job.proc.kill()
927
+ job.proc.wait()
928
+ out.provenance = c_engine.provenance(binary, d if folder else None, argv,
929
+ getattr(job, "elapsed", None), note)
930
+ out.drift_how = how
931
+ return out
932
+
933
+
934
+ def rungs(pre_files, p, q, r=0, case=2, ma="free"):
935
+ """-rungs: the ladder below the rank at r = 0 -- rungs 0, 1 and 2 with
936
+ rung 0's gate and the nested chi2 LRs (rungtable.run_rungs). r only names the
937
+ header's rank, as the C's does."""
938
+ from .rungtable import run_rungs
939
+ data = _data(pre_files)
940
+ if not 0 <= r < data.M:
941
+ raise ValueError(f"r={r} must be < M={data.M}")
942
+ return run_rungs(data, Spec(data.M, p, q, r, case, ma))