drvec 0.1.0__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
drvec/rank.py ADDED
@@ -0,0 +1,401 @@
1
+ """The sequential LR test for the cointegration rank, with its parametric
2
+ bootstrap.
3
+
4
+ C: `run_lrtest` (drvec.c:9516-9960), `bootstrap_rank` (2356-2455),
5
+ `simulate_h0` (2226), `fit_ll` (2288), the critical values (161-172).
6
+
7
+ LR(r -> r+1) = 2 [L(r+1) - L(r)] (exact likelihood; may be < 0, BUG-26)
8
+
9
+ Every rank, observed or simulated, is fitted as the C fits it: the cold start,
10
+ the ladder (route (B) for r >= 1), the search. The bootstrap simulates from the
11
+ fitted rank-r model in Ybar coordinates and inverts the transformation to levels.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import math
16
+ from dataclasses import dataclass, field
17
+
18
+ import numpy as np
19
+ from drvarma._engine import elf_c
20
+
21
+ from .cast import shootx
22
+ from .fit import fit_search, gate_profile_seed, ladder_seed_r0
23
+ from .loglik import XITOL
24
+ from .nla import choldcp
25
+ from .rng import Rng
26
+ from .start import init_guess
27
+
28
+ # lambda-max critical values, 10% 5% 1%, by M - r = 1..11 (drvec.c:161-172)
29
+ CV_NONE = [(2.98, 4.13, 6.94), (9.47, 11.22, 15.09), (15.72, 17.80, 22.25),
30
+ (21.84, 24.16, 29.06), (27.92, 30.44, 35.74), (33.93, 36.63, 42.23),
31
+ (39.91, 42.77, 48.66), (45.89, 48.88, 55.03), (51.85, 54.96, 61.34),
32
+ (57.80, 61.04, 67.64), (63.72, 67.08, 73.89)]
33
+ CV_CONST = [(7.52, 9.24, 12.97), (13.75, 15.67, 20.20), (19.77, 22.00, 26.81),
34
+ (25.56, 28.14, 33.24), (31.66, 34.40, 39.79), (37.45, 40.30, 46.82),
35
+ (43.25, 46.45, 51.91), (48.91, 52.00, 57.95), (54.35, 57.42, 63.71),
36
+ (60.25, 63.57, 69.94), (66.02, 69.74, 76.63)]
37
+
38
+
39
+ def fit_rank(spec, data, log=None, njitter=1):
40
+ """One rank as the C fits it (run_lrtest's loop body, fit_ll): init_guess,
41
+ the ladder seed, the search (with -multistart's jitters when njitter > 1).
42
+ -> (SearchResult, Profile or Ladder)."""
43
+ rd = data.at(spec.r)
44
+ x = init_guess(spec, rd)
45
+ if spec.r > 0:
46
+ x, info = gate_profile_seed(spec, data, x, log)
47
+ else:
48
+ x, info = ladder_seed_r0(spec, data, x, log)
49
+ return fit_search(spec, rd, x, njitter=njitter, log=log), info
50
+
51
+
52
+ def ma_min_modulus(theta):
53
+ """Smallest modulus of the roots of |Theta*(B)| = 0 (1/|eigenvalue| of the
54
+ companion matrix, as chekma builds it); inf when there is no MA."""
55
+ theta = np.asarray(theta, float)
56
+ if theta.shape[0] == 0 or not np.any(theta):
57
+ return math.inf
58
+ q, m, _ = theta.shape
59
+ A = np.zeros((m * q, m * q))
60
+ for k in range(q):
61
+ A[:m, k * m:(k + 1) * m] = theta[k]
62
+ for k in range(q - 1):
63
+ A[(k + 1) * m:(k + 2) * m, k * m:(k + 1) * m] = np.eye(m)
64
+ ev = np.abs(np.linalg.eigvals(A))
65
+ ev = ev[ev > 0]
66
+ return float(1.0 / ev.max()) if ev.size else math.inf
67
+
68
+
69
+ def simulate_h0(spec, data, x, rng):
70
+ """drvec.c:2226. One sample of LEVELS (nobs_raw, M) under the rank-r model x,
71
+ or None when sigma2 * qq has no Cholesky factor (`simfail`)."""
72
+ M, r, s, p, q = spec.M, spec.r, spec.s, spec.p, spec.q
73
+ rd = data.at(r)
74
+ n = rd.n
75
+ mu, phi, theta, qq, w, _ = shootx(spec, rd, x)
76
+ _lg, f1, f2, _a, ifa = elf_c(M, n, p, q, mu, phi, theta, qq, w, 1.0, XITOL, True)
77
+ sigma2 = f1 / (n * M)
78
+ from .cast import unpack
79
+ B2 = unpack(spec, x)[5] # held rows (-fixb2row) included
80
+ burn = 50 + 10 * p
81
+ T = n + burn
82
+ L, ifc = choldcp([[sigma2 * float(qq[i, j]) for j in range(M)] for i in range(M)])
83
+ if ifc:
84
+ return None
85
+ ph = phi.tolist()
86
+ th = theta.tolist()
87
+ mu = mu.tolist()
88
+ wb = [[0.0] * M for _ in range(T)]
89
+ ab = [[0.0] * M for _ in range(T)]
90
+ for t in range(T):
91
+ z = [rng.normal() for _ in range(M)]
92
+ for i in range(M):
93
+ acc = 0.0
94
+ for k in range(i + 1):
95
+ acc += L[i][k] * z[k]
96
+ ab[t][i] = acc
97
+ for i in range(M):
98
+ acc = ab[t][i]
99
+ for j in range(1, p + 1):
100
+ if t - j >= 0:
101
+ for k in range(M):
102
+ acc += ph[j - 1][i][k] * (wb[t - j][k] - mu[k])
103
+ for j in range(1, q + 1):
104
+ if t - j >= 0:
105
+ for k in range(M):
106
+ acc -= th[j - 1][i][k] * ab[t - j][k]
107
+ wb[t][i] = mu[i] + acc
108
+ out = [[0.0] * M for _ in range(n + 1)]
109
+ out[0] = [float(v) for v in data.raw[0]]
110
+ for t in range(n):
111
+ tb = t + burn
112
+ for i in range(s):
113
+ out[t + 1][i] = out[t][i] + wb[tb][i]
114
+ for j in range(r):
115
+ v = wb[tb][s + j]
116
+ for i in range(s):
117
+ v -= B2[i][j] * out[t + 1][i]
118
+ out[t + 1][s + j] = v
119
+ return np.array(out, float)
120
+
121
+
122
+ GRANGER_ZERO = 1.0e-6 # report.GRANGER_ZERO, the rank condition's threshold
123
+
124
+
125
+ def i1r_check(spec, data, x):
126
+ """drvec.c:i1r_check -- is the model under H0 I(1) with rank r? Cavaliere,
127
+ Rahbek and Taylor (2012, Algorithm 1, step ii) check it before simulating.
128
+ The model is x at rank spec.r on the observed data. -> (fails, ar_min, lr_sv)."""
129
+ M, p, q = spec.M, spec.p, spec.q
130
+ rd = data.at(spec.r)
131
+ mu, phi, theta, qq, w, _ = shootx(spec, rd, x)
132
+ _lg, f1, _f2, _a, _ifa = elf_c(M, rd.n, p, q, mu, phi, theta, qq, w, 1.0, XITOL, True)
133
+ return i1r_measure(phi, theta, (f1 / (rd.n * M)) * np.asarray(qq, float), spec.r)
134
+
135
+
136
+ def i1r_measure(phi, theta, sigma, r):
137
+ """The two conditions of i1r_check on Ybar = [nabla Y2 ; W], Phi(B) Ybar =
138
+ Theta(B) a with Var(a) = sigma (elf's signs; phi, theta (k, M, M)):
139
+ every AR root outside the circle, and nabla Y2's long-run covariance (the
140
+ first s = M - r rows of K = Phi(1)^-1 Theta(1), times sigma) non-singular,
141
+ measured scale-free as the smallest singular value of D^-1/2 K_s L (1 for
142
+ unrelated trends, 0 for a lost one). -> (fails, ar_min, lr_sv)."""
143
+ from .report import report_operator_roots
144
+ phi = np.asarray(phi, float)
145
+ theta = np.asarray(theta, float)
146
+ M = sigma.shape[0]
147
+ s = M - r
148
+ ar_min = 1.0e12
149
+ if phi.shape[0] > 0:
150
+ ar_min = report_operator_roots("", phi, M, phi.shape[0])[1]
151
+ if ar_min <= 1.0 + 1.0e-6:
152
+ return True, ar_min, 1.0
153
+ if s <= 0:
154
+ return False, ar_min, 1.0
155
+ P1 = np.eye(M) - phi.sum(axis=0) if phi.shape[0] else np.eye(M)
156
+ T1 = np.eye(M) - theta.sum(axis=0) if theta.shape[0] else np.eye(M)
157
+ K = np.linalg.solve(P1, T1)
158
+ L, ifc = choldcp([[float(sigma[i, j]) for j in range(M)] for i in range(M)])
159
+ if ifc:
160
+ return True, ar_min, 0.0
161
+ G = K[:s, :] @ np.asarray(L, float)
162
+ nrm = np.sqrt((G * G).sum(axis=1))
163
+ G = np.divide(G, nrm[:, None], out=np.zeros_like(G), where=nrm[:, None] > 0.0)
164
+ lr_sv = float(np.linalg.svd(G, compute_uv=False)[-1])
165
+ return lr_sv < GRANGER_ZERO, ar_min, lr_sv
166
+
167
+
168
+ def not_i1r(spec, data, fits, ranks):
169
+ """{r: (ar_min, lr_sv)} for the H0 ranks whose fitted model fails i1r_check."""
170
+ out = {}
171
+ for rr in ranks:
172
+ fails, ar_min, lr_sv = i1r_check(spec.but(r=rr), data, fits[rr].x)
173
+ if fails:
174
+ out[rr] = (ar_min, lr_sv)
175
+ return out
176
+
177
+
178
+ @dataclass
179
+ class Replication:
180
+ h0: int
181
+ rep: int
182
+ status: str # ok, simfail, fitfail
183
+ lr: float = float("nan")
184
+
185
+ def line(self):
186
+ return (f"{self.h0},{self.rep},{self.status},%.17g" % self.lr if self.status == "ok"
187
+ else f"{self.h0},{self.rep},{self.status},")
188
+
189
+
190
+ @dataclass
191
+ class Bootstrap:
192
+ h0: int
193
+ lr: float
194
+ reps: list
195
+ cv: tuple = None
196
+ pvalue: float = None
197
+ i1r: tuple = None # (ar_min, lr_sv) when the H0 model is not I(1, r): no samples
198
+
199
+ @property
200
+ def nok(self):
201
+ return sum(r.status == "ok" for r in self.reps)
202
+
203
+
204
+ def _refit_pair(spec_r, data, sim):
205
+ """One replication: the sample refitted at ranks r and r + 1 (the C's two
206
+ fit_ll calls). -> (ifault_r, ll_r, ifault_r1, ll_r1). It consumes no
207
+ random numbers: the sample is drawn before (drvec.parallel)."""
208
+ db = data.with_raw(sim)
209
+ r0, _ = fit_rank(spec_r, db)
210
+ r1, _ = fit_rank(spec_r.but(r=spec_r.r + 1), db)
211
+ return r0.ifault, r0.ll, r1.ifault, r1.ll
212
+
213
+
214
+ def bootstrap_rank(spec_r, data, x, N, lr_obs, rng, progress=None, workers=1, pairs=None):
215
+ """bootstrap_rank (2356). spec_r is the H0 rank's spec, x its fitted vector.
216
+ progress(b, replication) after each replication, a failed one included.
217
+ workers > 1: the N samples drawn first, in the generator's order, and
218
+ refitted in a process pool -- the same numbers (drvec.parallel). pairs:
219
+ the refits already under way (an iterator of _refit_pair's results, in
220
+ order), when the caller drew the samples itself."""
221
+ rr = spec_r.r
222
+ reps = []
223
+ own = pairs is None and workers > 1
224
+ if own:
225
+ from .parallel import map_refits
226
+ sims = [simulate_h0(spec_r, data, x, rng) for _b in range(N)]
227
+ pairs = map_refits(_refit_pair, data,
228
+ [(spec_r, sim) if sim is not None else None for sim in sims],
229
+ workers)
230
+ for b in range(1, N + 1):
231
+ if pairs is not None:
232
+ pr = next(pairs)
233
+ else:
234
+ sim = simulate_h0(spec_r, data, x, rng)
235
+ pr = None if sim is None else _refit_pair(spec_r, data, sim)
236
+ if pr is None:
237
+ reps.append(Replication(rr, b, "simfail"))
238
+ elif pr[0] == 0 and pr[2] == 0:
239
+ reps.append(Replication(rr, b, "ok", 2.0 * (pr[3] - pr[1])))
240
+ else:
241
+ reps.append(Replication(rr, b, "fitfail"))
242
+ if progress:
243
+ progress(b, reps[-1])
244
+ if own:
245
+ pairs.close()
246
+ return summarise(rr, lr_obs, reps)
247
+
248
+
249
+ def summarise(rr, lr_obs, reps):
250
+ """The bootstrap's critical values and p-value from its replications
251
+ (bootstrap_rank's tail): every ok replication counts, negative LR*
252
+ included (BUG-26)."""
253
+ out = Bootstrap(rr, lr_obs, reps)
254
+ stat = [r.lr for r in reps if r.status == "ok"]
255
+ nok = len(stat)
256
+ if nok >= 10:
257
+ st = sorted(stat)
258
+ idx = [min(max(int(math.ceil(a * nok)), 1), nok) for a in (0.90, 0.95, 0.99)]
259
+ out.cv = tuple(st[i - 1] for i in idx)
260
+ ge = sum(1 for v in st if v >= lr_obs)
261
+ out.pvalue = (ge + 1) / (nok + 1)
262
+ return out
263
+
264
+
265
+ @dataclass
266
+ class RankTest:
267
+ spec: object
268
+ names: list
269
+ nobs: int
270
+ fits: list # SearchResult per rank 0..M-1
271
+ infos: list # Ladder/Profile per rank
272
+ npar: list
273
+ ll: list # logL in the original units
274
+ scale: list
275
+ boundary: list
276
+ bootstraps: list = field(default_factory=list)
277
+ boot_n: int = 0 # -bootstrap N (0: none)
278
+ data: object = None # the data as read (for the .out's header)
279
+ drift_how: str = None # "pre" / "nodrift" when case 3's drifts are masked
280
+ provenance: object = None # results.Provenance: who ran the bootstrap
281
+
282
+ def lr(self, r):
283
+ return 2.0 * (self.ll[r + 1] - self.ll[r])
284
+
285
+ def aic(self, r):
286
+ return (-2.0 * self.ll[r] + 2.0 * self.npar[r]) / self.nobs
287
+
288
+ def bic(self, r):
289
+ return (-2.0 * self.ll[r] + self.npar[r] * math.log(self.nobs)) / self.nobs
290
+
291
+ def critical_values(self, r):
292
+ g = self.spec.M - r
293
+ if self.spec.case == 3 or not (1 <= g <= 11):
294
+ return None
295
+ return (CV_NONE if self.spec.case == 1 else CV_CONST)[g - 1]
296
+
297
+ def verdict(self, r, cv=None):
298
+ cv = cv or self.critical_values(r)
299
+ if cv is None:
300
+ return None
301
+ lr = self.lr(r)
302
+ if lr > cv[2]:
303
+ return "reject H0 at 1%"
304
+ if lr > cv[1]:
305
+ return "reject H0 at 5%"
306
+ if lr > cv[0]:
307
+ return "reject H0 at 10%"
308
+ return "H0 not rejected"
309
+
310
+ def out_text(self, base):
311
+ """<base>.out, as the C writes it (report.rank_test_out)."""
312
+ from .report import rank_test_out
313
+ return rank_test_out(self, self.data, base)
314
+
315
+ def boot_text(self, version="0.1.0.dev0"):
316
+ """<base>.boot (drvec.c:9775-9785). The first line differs from the C's in
317
+ the program's name only; tools/golden.sh drops it before hashing."""
318
+ N = len(self.bootstraps[0].reps) if self.bootstraps else 0
319
+ lines = [f"# drvec {version} (python) -bootstrap {N}: the rank test, one line "
320
+ "per replication", "h0,rep,status,lr"]
321
+ for b in self.bootstraps:
322
+ lines += [r.line() for r in b.reps]
323
+ return "\n".join(lines) + "\n"
324
+
325
+
326
+ def rank_test(data, spec, bootstrap=0, boot_path=None, progress=None, njitter=1,
327
+ seed=None, workers=1, boot_hook=None):
328
+ """run_lrtest. `spec` gives p, q, case and the class; its r is ignored (the
329
+ test walks r = 0..M-1). seed: the bootstrap generator's (None: the C's).
330
+ workers: the pool's size for the refits (1: in this process).
331
+ boot_hook(out, ds): when given, it supplies the bootstraps instead of the
332
+ generator (the C engine, drvec.c_engine) and returns their list."""
333
+ M = spec.M
334
+ raw = data.raw
335
+ # BUG-43: a gross mis-scaling (two orders) is undone internally; the LR does
336
+ # not depend on it and the logLs are reported in the original units.
337
+ dsc = [1.0] * M
338
+ T = data.nobs_raw - 1
339
+ for i in range(M):
340
+ d = [float(raw[t, i]) - float(raw[t - 1, i]) for t in range(1, data.nobs_raw)]
341
+ m1 = sum(d) / T
342
+ v = 0.0
343
+ for di in d:
344
+ v += (di - m1) * (di - m1)
345
+ sd = math.sqrt(v / (T - 1 if T > 1 else 1))
346
+ if sd > 0.0 and abs(math.log10(sd)) > 2.0:
347
+ dsc[i] = 10.0 ** (-math.floor(math.log10(sd) + 0.5))
348
+ scaled = any(c != 1.0 for c in dsc)
349
+ ds = data.with_raw(raw * np.array(dsc)) if scaled else data
350
+ fits, infos, npar, ll, bnd = [], [], [], [], []
351
+ for rr in range(M):
352
+ sr = spec.but(r=rr)
353
+ res, info = fit_rank(sr, ds, njitter=njitter) # -multistart (BUG-25)
354
+ fits.append(res); infos.append(info); npar.append(sr.npar())
355
+ ll.append(res.ll if res.ifault == 0 else float("nan"))
356
+ b = False
357
+ if res.ifault == 0 and spec.q > 0:
358
+ _mu, _phi, theta, _qq, _w, _ = shootx(sr, ds.at(rr), res.x)
359
+ b = ma_min_modulus(theta) < 1.0001
360
+ bnd.append(b)
361
+ nobs = ds.at(0).n
362
+ if scaled:
363
+ jac = sum(math.log(c) for c in dsc) * nobs
364
+ ll = [v + jac for v in ll]
365
+ out = RankTest(spec, data.names, nobs, fits, infos, npar, ll, dsc, bnd,
366
+ boot_n=bootstrap, data=data)
367
+ if bootstrap > 0 and boot_hook is not None:
368
+ out.bootstraps = boot_hook(out, ds)
369
+ elif bootstrap > 0:
370
+ rng = Rng() if seed is None else Rng(seed)
371
+ pairs_ok = [rr for rr in range(M - 1) if not (fits[rr].ifault or fits[rr + 1].ifault)]
372
+ # no sample is drawn from a model that is not I(1, r) (drvec.c, step ii)
373
+ noti1 = not_i1r(spec, ds, fits, pairs_ok)
374
+ todo = [rr for rr in pairs_ok if rr not in noti1]
375
+ pairs = None
376
+ if workers > 1:
377
+ # every comparison's samples drawn first, in the generator's order
378
+ # (comparison by comparison, as the sequential path draws them: they
379
+ # come from the OBSERVED fits), then one pool for all the refits, so
380
+ # that no comparison waits for the previous one's slowest refit
381
+ from .parallel import map_refits
382
+ tasks = []
383
+ for rr in todo:
384
+ sr = spec.but(r=rr)
385
+ for _b in range(bootstrap):
386
+ sim = simulate_h0(sr, ds, fits[rr].x, rng)
387
+ tasks.append(None if sim is None else (sr, sim))
388
+ pairs = map_refits(_refit_pair, ds, tasks, workers)
389
+ for rr in pairs_ok:
390
+ if rr in noti1:
391
+ out.bootstraps.append(Bootstrap(rr, out.lr(rr), [], i1r=noti1[rr]))
392
+ continue
393
+ out.bootstraps.append(bootstrap_rank(spec.but(r=rr), ds, fits[rr].x,
394
+ bootstrap, out.lr(rr), rng, progress,
395
+ workers=workers, pairs=pairs))
396
+ if pairs is not None:
397
+ pairs.close()
398
+ if boot_path:
399
+ with open(boot_path, "w") as f:
400
+ f.write(out.boot_text())
401
+ return out