drvec 0.1.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- drvec/__init__.py +22 -0
- drvec/_bin/drvec.exe +0 -0
- drvec/_gsl.py +83 -0
- drvec/api.py +942 -0
- drvec/c_engine.py +391 -0
- drvec/cast.py +196 -0
- drvec/cli.py +592 -0
- drvec/data.py +295 -0
- drvec/diagnose.py +439 -0
- drvec/fdhess.py +130 -0
- drvec/fit.py +528 -0
- drvec/forecast.py +314 -0
- drvec/irfboot.py +216 -0
- drvec/levels.py +158 -0
- drvec/loglik.py +45 -0
- drvec/nla.py +266 -0
- drvec/parallel.py +86 -0
- drvec/params.py +311 -0
- drvec/rank.py +401 -0
- drvec/report.py +1712 -0
- drvec/results.py +167 -0
- drvec/rng.py +51 -0
- drvec/rungtable.py +82 -0
- drvec/start.py +317 -0
- drvec/stderr.py +33 -0
- drvec/xlink.py +226 -0
- drvec-0.1.0.dist-info/METADATA +205 -0
- drvec-0.1.0.dist-info/RECORD +32 -0
- drvec-0.1.0.dist-info/WHEEL +5 -0
- drvec-0.1.0.dist-info/entry_points.txt +2 -0
- drvec-0.1.0.dist-info/licenses/COPYING +339 -0
- drvec-0.1.0.dist-info/top_level.txt +1 -0
drvec/fit.py
ADDED
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
"""Estimation, as the C does it: est(), the admissible-start ladder, the search
|
|
2
|
+
over starts, the r = 0 ladder with its entry gate, and the route-(B) profile.
|
|
3
|
+
|
|
4
|
+
C references (engines/drvec at drvec-v0.12): `est` drvmlest.c:33-145 and
|
|
5
|
+
`objcfunc` 149-181; `make_admissible` drvec.c:4919; `est_run` 4967;
|
|
6
|
+
`fit_search` 5079; `pack_r0` 5326; `fit_r0_ladder` 5345; `ladder_seed_r0`
|
|
7
|
+
5443; `gate_profile_seed` 5508; `gate_contract` 3014.
|
|
8
|
+
|
|
9
|
+
The order of the fits is the C's, call for call (24 est() calls for case 1, 11
|
|
10
|
+
per data set for the rank test), because the winner of the search is decided by
|
|
11
|
+
a strict ">" between optima that differ by 1e-10 in logL and by 1e-4 in the
|
|
12
|
+
parameters (docs/PORT.md §5): reproducing the optimum is not enough, the
|
|
13
|
+
trajectories have to be reproduced too.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import math
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
from drvarma import _qnewt
|
|
22
|
+
from drvarma._engine import elf_c
|
|
23
|
+
|
|
24
|
+
from .cast import shootx
|
|
25
|
+
from .loglik import LOG2PI, XITOL, concentrated, elf
|
|
26
|
+
from .params import embed_ma, theta_pack
|
|
27
|
+
from .start import init_guess
|
|
28
|
+
from .stderr import bfgs_covariance
|
|
29
|
+
|
|
30
|
+
MAXITS, GRADTOL, STEPTOL = 500, 1.0e-5, 1.0e-7 # drvec.c:2898-2899
|
|
31
|
+
|
|
32
|
+
CRITERION = {1: "norm of scaled gradient <= gradtol",
|
|
33
|
+
2: "scaled distance between last two steps <= steptol",
|
|
34
|
+
3: "last global step failed to locate a lower point",
|
|
35
|
+
4: "iteration limit reached",
|
|
36
|
+
5: "five consecutive steps of maximum length taken"}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class LineSearchRoundoff(RuntimeError):
|
|
40
|
+
"""The line search's `disc < 0` branch (lib/optim/lnsrch.c). In the C it is
|
|
41
|
+
`nrerror("ROUNDOFF PROBLEM IN LINE SEARCH")`, which EXITS THE PROGRAM with
|
|
42
|
+
status 1 (nlatools.c:367), whatever was running -- a start of the search, a
|
|
43
|
+
rung of the ladder, a bootstrap replication. A library cannot exit, so it is
|
|
44
|
+
raised and NOT caught anywhere in drvec: it reaches the caller of `fit` or
|
|
45
|
+
`rank_test` exactly as the C's run ends there, with no partial result.
|
|
46
|
+
It does not fire on the milestone's cases."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# --------------------------------------------------------------------------- #
|
|
50
|
+
# est() #
|
|
51
|
+
# --------------------------------------------------------------------------- #
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class EstResult:
|
|
55
|
+
x: list
|
|
56
|
+
ifault: int
|
|
57
|
+
ll: float = float("-inf")
|
|
58
|
+
s2: float = float("nan")
|
|
59
|
+
fk: float = float("nan")
|
|
60
|
+
iters: int = -1
|
|
61
|
+
termcode: int = 0
|
|
62
|
+
dev: np.ndarray = None
|
|
63
|
+
cov: np.ndarray = None
|
|
64
|
+
|
|
65
|
+
def banner(self):
|
|
66
|
+
"""The text report() writes into the .out (qnewtopt.c:98-120)."""
|
|
67
|
+
tc = self.termcode
|
|
68
|
+
lines = ["", "=" * 61,
|
|
69
|
+
" OPTIMIZER %s after %d iterations" % (
|
|
70
|
+
"CONVERGED" if tc in (1, 2) else "STOPPED", self.iters),
|
|
71
|
+
" Objective function = %.12f" % self.fk]
|
|
72
|
+
if tc in CRITERION:
|
|
73
|
+
lines.append(" Convergence criterion: %s" % CRITERION[tc])
|
|
74
|
+
lines += ["=" * 61, "", ""]
|
|
75
|
+
return "\n".join(lines)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def est(spec, rd, x0, hold=None, maxits=MAXITS, gradtol=GRADTOL, steptol=STEPTOL,
|
|
79
|
+
xitol=XITOL):
|
|
80
|
+
"""drvmlest.c:est. The objective is the concentrated likelihood normalised
|
|
81
|
+
at the start, (f1/f1_0)^m (f2/f2_0); a refused point is 1.0."""
|
|
82
|
+
x0 = [float(v) for v in x0]
|
|
83
|
+
npar = len(x0)
|
|
84
|
+
m, n = spec.M, rd.n
|
|
85
|
+
f10, f20, _a, ifa = elf(spec, rd, x0, xitol, True, hold)
|
|
86
|
+
if ifa:
|
|
87
|
+
return EstResult(x=x0, ifault=1 if ifa == 6 else ifa)
|
|
88
|
+
|
|
89
|
+
def objc(xk1):
|
|
90
|
+
f1, f2, _a, ifa = elf(spec, rd, xk1[1:].tolist(), xitol, False, hold)
|
|
91
|
+
if ifa:
|
|
92
|
+
return 1.0
|
|
93
|
+
f = pow(f1 / f10, m) * (f2 / f20)
|
|
94
|
+
return f if math.isfinite(f) else 1.0
|
|
95
|
+
|
|
96
|
+
xk = np.zeros(npar + 1)
|
|
97
|
+
xk[1:] = x0
|
|
98
|
+
try:
|
|
99
|
+
fk, b, k, tc = _qnewt.raxopt(objc, npar, xk, maxits, gradtol, steptol)
|
|
100
|
+
except RuntimeError as e: # pragma: no cover (not reached)
|
|
101
|
+
if "ROUNDOFF" in str(e):
|
|
102
|
+
raise LineSearchRoundoff(str(e)) from e
|
|
103
|
+
raise
|
|
104
|
+
x = xk[1:].tolist()
|
|
105
|
+
dev, cov = bfgs_covariance(b, fk, npar, n)
|
|
106
|
+
f1, f2, _a, ifa = elf(spec, rd, x, xitol, True, hold)
|
|
107
|
+
res = EstResult(x=x, ifault=0, fk=float(fk), iters=int(k), termcode=int(tc),
|
|
108
|
+
dev=dev, cov=cov)
|
|
109
|
+
if ifa:
|
|
110
|
+
# The C returns the ifault of its last (deallocating) cast call, which is
|
|
111
|
+
# only the Sigma check: an elf refusal at the final point would leave a
|
|
112
|
+
# garbage logL. The port refuses the point instead (not reached here).
|
|
113
|
+
res.ifault = 1 if ifa == 6 else ifa
|
|
114
|
+
return res
|
|
115
|
+
res.ll = concentrated(m, n, f1, f2)
|
|
116
|
+
res.s2 = f1 / (n * m)
|
|
117
|
+
return res
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def est_run(spec, rd, x0):
|
|
121
|
+
"""est() plus est_run's two checks on the final point (drvec.c:5020-5060):
|
|
122
|
+
Sigma passes choldcp, and qq is not singular (det qq / prod qq_ii >= 1e-10)."""
|
|
123
|
+
r = est(spec, rd, x0)
|
|
124
|
+
if r.ifault == 0 and spec.M > 1:
|
|
125
|
+
_mu, _phi, _th, qq, _w, ifq = shootx(spec, rd, r.x)
|
|
126
|
+
if ifq:
|
|
127
|
+
r.ifault = 1
|
|
128
|
+
else:
|
|
129
|
+
m = spec.M
|
|
130
|
+
L = [[float(qq[a, b]) for b in range(m)] for a in range(m)]
|
|
131
|
+
det, pd = 1.0, 1.0
|
|
132
|
+
for a in range(m):
|
|
133
|
+
d = L[a][a]
|
|
134
|
+
for c in range(a):
|
|
135
|
+
d -= L[a][c] * L[a][c]
|
|
136
|
+
if not (d > 0.0):
|
|
137
|
+
r.ifault = 1
|
|
138
|
+
break
|
|
139
|
+
L[a][a] = math.sqrt(d)
|
|
140
|
+
for bb in range(a + 1, m):
|
|
141
|
+
e = L[bb][a]
|
|
142
|
+
for c in range(a):
|
|
143
|
+
e -= L[bb][c] * L[a][c]
|
|
144
|
+
L[bb][a] = e / L[a][a]
|
|
145
|
+
det *= d
|
|
146
|
+
pd *= float(qq[a, a])
|
|
147
|
+
if r.ifault == 0 and pd > 0.0 and det / pd < 1e-10:
|
|
148
|
+
r.ifault = 1
|
|
149
|
+
return r
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# --------------------------------------------------------------------------- #
|
|
153
|
+
# the search #
|
|
154
|
+
# --------------------------------------------------------------------------- #
|
|
155
|
+
|
|
156
|
+
_SHRINK = (1.0, 0.8, 0.5, 0.3, 0.1, 0.0)
|
|
157
|
+
# c of the BUG-64 Lambda: the relations it seeds are nearly I(1); 0.02 and 0.05
|
|
158
|
+
# reach the same optimum, 0.2 and 0.5 lower ones (drvec.c, make_admissible)
|
|
159
|
+
_LAMBDA_FALLBACK = 0.05
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def make_admissible(spec, rd, x):
|
|
163
|
+
"""drvec.c:4919. F and Theta shrunk towards zero, and Lambda halved twice if
|
|
164
|
+
that is not enough, until elf accepts the start; then Lambda = c [I; 0]
|
|
165
|
+
(BUG-64). -> (x, step): step 0 when accepted as given, -1 when nothing
|
|
166
|
+
worked (x returned as given)."""
|
|
167
|
+
nmean, nlam, nmid, _ntail = spec.blocks()
|
|
168
|
+
lo, hi = nmean + nlam, nmean + nlam + nmid - spec.nsig()
|
|
169
|
+
x0 = list(x)
|
|
170
|
+
for ps in range(3):
|
|
171
|
+
lamf = (1.0, 0.5, 0.25)[ps]
|
|
172
|
+
for mi in range(6):
|
|
173
|
+
xt = list(x0)
|
|
174
|
+
for i in range(lo, hi):
|
|
175
|
+
xt[i] = _SHRINK[mi] * x0[i]
|
|
176
|
+
for i in range(nmean, nmean + nlam):
|
|
177
|
+
xt[i] = lamf * x0[i]
|
|
178
|
+
_f1, _f2, _a, ifa = elf(spec, rd, xt, XITOL, True)
|
|
179
|
+
if ifa == 0:
|
|
180
|
+
return xt, ps * 6 + mi
|
|
181
|
+
# BUG-64 (drvec.c:5117): scaling cannot fix a Lambda that points the wrong
|
|
182
|
+
# way. The last resort is one admissible by construction: the Y1 rows c I,
|
|
183
|
+
# the Y2 rows zero, so B'Lambda = c I whatever B2 is. Not under -alpha.
|
|
184
|
+
if not spec.alpha and spec.r > 0:
|
|
185
|
+
r = spec.r
|
|
186
|
+
for mi in range(6):
|
|
187
|
+
xt = list(x0)
|
|
188
|
+
for i in range(lo, hi):
|
|
189
|
+
xt[i] = _SHRINK[mi] * x0[i]
|
|
190
|
+
for ii in range(spec.M):
|
|
191
|
+
for jj in range(r):
|
|
192
|
+
xt[nmean + ii * r + jj] = _LAMBDA_FALLBACK if ii == jj else 0.0
|
|
193
|
+
_f1, _f2, _a, ifa = elf(spec, rd, xt, XITOL, True)
|
|
194
|
+
if ifa == 0:
|
|
195
|
+
return xt, 18 + mi
|
|
196
|
+
return x0, -1
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@dataclass
|
|
200
|
+
class Candidate:
|
|
201
|
+
label: str
|
|
202
|
+
ok: bool
|
|
203
|
+
ll: float
|
|
204
|
+
termcode: int
|
|
205
|
+
iters: int
|
|
206
|
+
adm: int
|
|
207
|
+
est: EstResult = None
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@dataclass
|
|
211
|
+
class SearchResult:
|
|
212
|
+
ifault: int
|
|
213
|
+
x: list
|
|
214
|
+
ll: float
|
|
215
|
+
s2: float
|
|
216
|
+
best: Candidate
|
|
217
|
+
candidates: list
|
|
218
|
+
seed: list = None # the start brought in (fit_search's x on entry)
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def nok(self):
|
|
222
|
+
return sum(c.ok for c in self.candidates)
|
|
223
|
+
|
|
224
|
+
def table(self):
|
|
225
|
+
"""The search table of the .out (fit_search 5240-5265)."""
|
|
226
|
+
ok = [c.ll for c in self.candidates if c.ok]
|
|
227
|
+
out = ["", "Search: %d starting points, %d converged; logL from %.6f to %.6f. "
|
|
228
|
+
"The best is `%s'." % (len(self.candidates), self.nok,
|
|
229
|
+
min(ok) if ok else 0.0,
|
|
230
|
+
self.best.ll if self.best else 0.0,
|
|
231
|
+
self.best.label if self.best else "none"),
|
|
232
|
+
" start logL stop iters admissible start"]
|
|
233
|
+
for c in self.candidates:
|
|
234
|
+
s = " %-10s " % c.label
|
|
235
|
+
s += ("%15.6f " % c.ll) if c.ok else " (failed) "
|
|
236
|
+
s += " %4s " % ({1: "grad", 2: "step", 3: "tc3", 4: "iter", 5: "maxl"}
|
|
237
|
+
.get(c.termcode, " - "))
|
|
238
|
+
s += ("%5d" % c.iters) if c.iters >= 0 else "%5s" % "-"
|
|
239
|
+
s += " %s" % ("as given" if c.adm == 0 else "shrunk" if c.adm > 0 else "not found")
|
|
240
|
+
out.append(s)
|
|
241
|
+
out.append(" The SPREAD is the diagnostic: on a well-behaved surface every "
|
|
242
|
+
"start lands in the same place.")
|
|
243
|
+
return "\n".join(out)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def fit_search(spec, rd, seed, njitter=1, log=None):
|
|
247
|
+
"""drvec.c:fit_search: the best of seed / cold / johansen / nested.
|
|
248
|
+
|
|
249
|
+
`log`, when a list, receives every est() call as (label, spec, start, result),
|
|
250
|
+
for the trajectory tests."""
|
|
251
|
+
cands = []
|
|
252
|
+
best = None
|
|
253
|
+
|
|
254
|
+
def TRY(label, xs):
|
|
255
|
+
nonlocal best
|
|
256
|
+
xc, adm = make_admissible(spec, rd, list(xs))
|
|
257
|
+
if adm < 0:
|
|
258
|
+
c = Candidate(label, False, 0.0, 0, -1, adm)
|
|
259
|
+
else:
|
|
260
|
+
er = est_run(spec, rd, xc)
|
|
261
|
+
if log is not None:
|
|
262
|
+
log.append((label, spec, list(xc), er))
|
|
263
|
+
c = Candidate(label, er.ifault == 0, er.ll if er.ifault == 0 else 0.0,
|
|
264
|
+
er.termcode, er.iters, adm, er)
|
|
265
|
+
cands.append(c)
|
|
266
|
+
if c.ok and (best is None or c.ll > best.ll):
|
|
267
|
+
best = c
|
|
268
|
+
|
|
269
|
+
seed = [float(v) for v in seed]
|
|
270
|
+
TRY("seed", seed)
|
|
271
|
+
cold = init_guess(spec, rd)
|
|
272
|
+
if any(abs(a - b) > 1e-12 for a, b in zip(cold, seed)):
|
|
273
|
+
TRY("cold", cold)
|
|
274
|
+
if spec.b2_fix is not None and spec.fixb2_given and spec.r > 0:
|
|
275
|
+
# -fixb2 v: the old seed too, Lambda, F and Sigma from the static-OLS W
|
|
276
|
+
# (inconsistent with v, BUG-33, but a search must not end below it)
|
|
277
|
+
TRY("ols-b2", init_guess(spec, rd, consistent=False))
|
|
278
|
+
if spec.r > 0 and spec.b2_fix is None:
|
|
279
|
+
TRY("johansen", init_guess(spec, rd, seedjoh=True))
|
|
280
|
+
cls = spec.theta_class()
|
|
281
|
+
if spec.q > 0 and cls != "q0":
|
|
282
|
+
if cls == "free":
|
|
283
|
+
low = "matri" if spec.r > 0 else "q0"
|
|
284
|
+
elif cls == "matri":
|
|
285
|
+
low = "marow" if spec.r > 0 else "q0"
|
|
286
|
+
else: # marow, diag
|
|
287
|
+
low = "q0"
|
|
288
|
+
lspec = spec.but(q=0, ma="free") if low == "q0" else spec.but(ma=low)
|
|
289
|
+
sub = fit_search(lspec, rd, init_guess(lspec, rd), log=log)
|
|
290
|
+
if sub.ifault == 0:
|
|
291
|
+
TRY("nested", embed_ma(sub.x, lspec, spec))
|
|
292
|
+
if njitter > 1:
|
|
293
|
+
# -multistart n: n - 1 perturbations of the caller's seed, from the C's
|
|
294
|
+
# own generator (drvec.c:5202-5218): a 64-bit LCG, as `unsigned long'
|
|
295
|
+
# is on Linux (32-bit on Windows, where the C's jitters differ)
|
|
296
|
+
rng = 20260818
|
|
297
|
+
for k in range(1, njitter):
|
|
298
|
+
amp = 0.05 * float(1 + (k - 1) % 20)
|
|
299
|
+
xt = []
|
|
300
|
+
for v in seed:
|
|
301
|
+
rng = (rng * 6364136223846793005 + 1442695040888963407) & 0xFFFFFFFFFFFFFFFF
|
|
302
|
+
u = float((rng >> 33) & 0x7FFFFFFF) / 2147483647.0
|
|
303
|
+
u = 2.0 * u - 1.0
|
|
304
|
+
xt.append(v + amp * u * (abs(v) if abs(v) > 1.0e-8 else 0.1))
|
|
305
|
+
TRY("jitter %d" % k, xt)
|
|
306
|
+
if best is None:
|
|
307
|
+
first = cands[0]
|
|
308
|
+
return SearchResult(first.est.ifault if first.est else 6, seed, 0.0, 0.0, None, cands,
|
|
309
|
+
seed)
|
|
310
|
+
return SearchResult(0, best.est.x, best.ll, best.est.s2, best, cands, seed)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
# --------------------------------------------------------------------------- #
|
|
314
|
+
# the ladder below the rank, and its gate #
|
|
315
|
+
# --------------------------------------------------------------------------- #
|
|
316
|
+
|
|
317
|
+
@dataclass
|
|
318
|
+
class Gate:
|
|
319
|
+
univariate: list
|
|
320
|
+
sum: float
|
|
321
|
+
joint: float
|
|
322
|
+
fit_ll: float
|
|
323
|
+
p: int
|
|
324
|
+
q: int
|
|
325
|
+
ll_start: float = None # the optimality certificate's "values brought in" (a fit)
|
|
326
|
+
|
|
327
|
+
@property
|
|
328
|
+
def gap(self):
|
|
329
|
+
return self.joint - self.sum
|
|
330
|
+
|
|
331
|
+
@property
|
|
332
|
+
def verified(self):
|
|
333
|
+
return abs(self.gap) < 1.0e-6
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def gate_contract(rs, rd0, x, ll):
|
|
337
|
+
"""drvec.c:3014. At r = 0 with F, Theta and Sigma diagonal the likelihood
|
|
338
|
+
factorises: the joint fit must equal the sum of its univariate components,
|
|
339
|
+
both evaluated with the WHOLE xi sequence (xitol < 0). It is not a comparison
|
|
340
|
+
with the .pre models, which this route does not read."""
|
|
341
|
+
M, n, p, q = rs.M, rd0.n, rs.p, rs.q
|
|
342
|
+
mu, phi, theta, qq, w, _ = shootx(rs, rd0, x)
|
|
343
|
+
xt = -abs(XITOL)
|
|
344
|
+
uni = []
|
|
345
|
+
for i in range(M):
|
|
346
|
+
_lg, f1, f2, _a, ifa = elf_c(1, n, p, q, mu[i:i + 1], phi[:, i:i + 1, i:i + 1],
|
|
347
|
+
theta[:, i:i + 1, i:i + 1], np.ones((1, 1)),
|
|
348
|
+
w[:, i:i + 1], 1.0, xt, False)
|
|
349
|
+
if ifa:
|
|
350
|
+
uni.append(None)
|
|
351
|
+
continue
|
|
352
|
+
uni.append(-0.5 * n * (LOG2PI - math.log(n) + 1.0)
|
|
353
|
+
- 0.5 * n * (math.log(f1) + math.log(f2)))
|
|
354
|
+
tot = 0.0
|
|
355
|
+
for v in uni:
|
|
356
|
+
if v is not None:
|
|
357
|
+
tot += v
|
|
358
|
+
_lg, f1, f2, _a, ifa = elf_c(M, n, p, q, mu, phi, theta, qq, w, 1.0, xt, False)
|
|
359
|
+
joint = concentrated(M, n, f1, f2) if ifa == 0 else ll
|
|
360
|
+
return Gate(uni, tot, joint, ll, p, q)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@dataclass
|
|
364
|
+
class Ladder:
|
|
365
|
+
ok: bool
|
|
366
|
+
F: list = None
|
|
367
|
+
Th: list = None
|
|
368
|
+
S: list = None
|
|
369
|
+
ll: float = 0.0
|
|
370
|
+
rungs: list = field(default_factory=list) # (diag_ar, diag_ma, diag_cov, logL)
|
|
371
|
+
gate: Gate = None
|
|
372
|
+
cached: bool = False
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def pack_r0(rs, F, Th, S, mu):
|
|
376
|
+
"""drvec.c:5326: an r = 0 point from full matrices, in rs's walk."""
|
|
377
|
+
M = rs.M
|
|
378
|
+
x = []
|
|
379
|
+
if rs.case == 3:
|
|
380
|
+
x += [mu[i] for i in range(M) if rs.mu_free(i)]
|
|
381
|
+
for k in range(rs.nf):
|
|
382
|
+
x += [float(F[k][i][j]) for i in range(M) for j in range(M) if rs.f_free(i, j)]
|
|
383
|
+
if rs.q > 0:
|
|
384
|
+
x += [float(v) for v in theta_pack(Th, rs.theta_class(), rs.q, M, 0)]
|
|
385
|
+
s11 = float(S[0][0]) if S[0][0] > 1e-24 else 1.0
|
|
386
|
+
x += [float(S[i][i]) / s11 for i in range(1, M)]
|
|
387
|
+
if not rs.diag_cov:
|
|
388
|
+
x += [float(S[i][j]) / s11 for i in range(1, M) for j in range(i)]
|
|
389
|
+
return x
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def fit_r0_ladder(spec, data, log=None):
|
|
393
|
+
"""drvec.c:5345. Rung 0 (F, Theta, Sigma diagonal; the gate), rung 1 (Sigma
|
|
394
|
+
free), rung 2 (the structure asked for), each from the optimum of the one
|
|
395
|
+
below. Cached per data set, as the C's lad_* cache (keyed on the data)."""
|
|
396
|
+
key = (spec.p, spec.q, spec.case, spec.diag_ar, spec.ma, spec.diag_cov,
|
|
397
|
+
tuple(spec.mu_free(i) for i in range(spec.M)) if spec.case == 3 else None,
|
|
398
|
+
spec.xsys, spec.gzero)
|
|
399
|
+
if key in data.ladder_cache:
|
|
400
|
+
c = data.ladder_cache[key]
|
|
401
|
+
return Ladder(True, c.F, c.Th, c.S, c.ll, c.rungs, c.gate, cached=True)
|
|
402
|
+
s0 = spec.but(r=0)
|
|
403
|
+
rd0 = data.at(0)
|
|
404
|
+
dar, dma, dcov = spec.diag_ar, spec.ma == "diag", spec.diag_cov
|
|
405
|
+
rungs = [(True, True, True)]
|
|
406
|
+
if not dcov:
|
|
407
|
+
rungs.append((True, True, False))
|
|
408
|
+
if not dar or not dma:
|
|
409
|
+
rungs.append((dar, dma, dcov))
|
|
410
|
+
out = Ladder(False)
|
|
411
|
+
F = Th = S = mu = None
|
|
412
|
+
for st, (a, b, c) in enumerate(rungs):
|
|
413
|
+
rs = s0.but(diag_ar=a, ma="diag" if b else spec.ma, diag_cov=c)
|
|
414
|
+
x = init_guess(rs, rd0) if st == 0 else pack_r0(rs, F, Th, S, mu)
|
|
415
|
+
res = fit_search(rs, rd0, x, log=log)
|
|
416
|
+
if res.ifault != 0:
|
|
417
|
+
return out
|
|
418
|
+
mu_, phi, theta, qq, _w, _ = shootx(rs, rd0, res.x)
|
|
419
|
+
F = [phi[k].tolist() for k in range(rs.nf)]
|
|
420
|
+
Th = [theta[k].tolist() for k in range(rs.q)]
|
|
421
|
+
S = qq.tolist()
|
|
422
|
+
mu = mu_.tolist()
|
|
423
|
+
out.rungs.append((a, b, c, res.ll))
|
|
424
|
+
if st == 0:
|
|
425
|
+
out.gate = gate_contract(rs, rd0, res.x, res.ll)
|
|
426
|
+
out.ll = res.ll
|
|
427
|
+
out.ok, out.F, out.Th, out.S = True, F, Th, S
|
|
428
|
+
data.ladder_cache[key] = out
|
|
429
|
+
return out
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def ladder_seed_r0(spec, data, x, log=None):
|
|
433
|
+
"""drvec.c:5443: at r = 0 the ladder's top rung IS the model (BUG-50)."""
|
|
434
|
+
if spec.r != 0 or (spec.diag_ar and spec.ma == "diag" and spec.diag_cov):
|
|
435
|
+
return list(x), None
|
|
436
|
+
mu = [0.0] * spec.M
|
|
437
|
+
if spec.case == 3:
|
|
438
|
+
im = 0
|
|
439
|
+
for i in range(spec.M):
|
|
440
|
+
if spec.mu_free(i):
|
|
441
|
+
mu[i] = x[im]; im += 1
|
|
442
|
+
lad = fit_r0_ladder(spec, data, log)
|
|
443
|
+
if not lad.ok:
|
|
444
|
+
return list(x), lad
|
|
445
|
+
return pack_r0(spec, lad.F, lad.Th, lad.S, mu), lad
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# --------------------------------------------------------------------------- #
|
|
449
|
+
# route (B): the VEC block profiled on the rung below #
|
|
450
|
+
# --------------------------------------------------------------------------- #
|
|
451
|
+
|
|
452
|
+
_MULT = (1.0, -1.0, 0.5, -0.5, 0.25, -0.25, 0.1, -0.1, 0.05, -0.05, 0.02, -0.02,
|
|
453
|
+
0.01, -0.01, 2.0, -2.0, 4.0, -4.0)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
@dataclass
|
|
457
|
+
class Profile:
|
|
458
|
+
ok: bool
|
|
459
|
+
ladder: Ladder = None
|
|
460
|
+
mult: float = 0.0
|
|
461
|
+
ll_start: float = 0.0
|
|
462
|
+
ll: float = 0.0
|
|
463
|
+
x2: list = None
|
|
464
|
+
est: EstResult = None
|
|
465
|
+
x2_start: list = None
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def gate_profile_seed(spec, data, x, log=None):
|
|
469
|
+
"""drvec.c:5508. Hold F, Theta and Sigma at the r = 0 optimum (permuted to the
|
|
470
|
+
internal order of rank r, Q renormalised: BUG-32), choose the multiple of the
|
|
471
|
+
conditional-regression Lambda by the likelihood, estimate only the mean,
|
|
472
|
+
Lambda and B2, and return the full start. -> (x, Profile)."""
|
|
473
|
+
r0, M = spec.r, spec.M
|
|
474
|
+
if r0 <= 0:
|
|
475
|
+
return list(x), Profile(False)
|
|
476
|
+
s0 = M - r0
|
|
477
|
+
nmean, nlam, nmid, ntail = spec.blocks()
|
|
478
|
+
nhead = nmean + nlam
|
|
479
|
+
lad = fit_r0_ladder(spec, data, log)
|
|
480
|
+
if not lad.ok:
|
|
481
|
+
return list(x), Profile(False, lad)
|
|
482
|
+
perm = lambda z: s0 + z if z < r0 else z - r0 # noqa: E731
|
|
483
|
+
hold = {
|
|
484
|
+
"F": [[[lad.F[k][perm(a)][perm(b)] for b in range(M)] for a in range(M)]
|
|
485
|
+
for k in range(spec.nf)],
|
|
486
|
+
"Th": [[[lad.Th[k][perm(a)][perm(b)] for b in range(M)] for a in range(M)]
|
|
487
|
+
for k in range(spec.q)],
|
|
488
|
+
}
|
|
489
|
+
s11 = lad.S[perm(0)][perm(0)]
|
|
490
|
+
if s11 <= 1e-24:
|
|
491
|
+
s11 = 1.0
|
|
492
|
+
hold["S"] = [[lad.S[perm(a)][perm(b)] / s11 for b in range(M)] for a in range(M)]
|
|
493
|
+
rd = data.at(r0)
|
|
494
|
+
x2 = [float(v) for v in x[:nhead]] + [float(v) for v in x[nhead + nmid:]]
|
|
495
|
+
lam0 = x2[nmean:nmean + nlam]
|
|
496
|
+
best = None
|
|
497
|
+
for c in _MULT:
|
|
498
|
+
for i in range(nlam):
|
|
499
|
+
x2[nmean + i] = c * lam0[i]
|
|
500
|
+
f1, f2, _a, ifa = elf(spec, rd, x2, XITOL, True, hold)
|
|
501
|
+
if ifa:
|
|
502
|
+
continue
|
|
503
|
+
ll = concentrated(M, rd.n, f1, f2)
|
|
504
|
+
if best is None or ll > best[0]:
|
|
505
|
+
best = (ll, c)
|
|
506
|
+
prof = Profile(False, lad)
|
|
507
|
+
if best is None:
|
|
508
|
+
return list(x), prof
|
|
509
|
+
for i in range(nlam):
|
|
510
|
+
x2[nmean + i] = best[1] * lam0[i]
|
|
511
|
+
prof.mult, prof.ll_start, prof.x2_start = best[1], best[0], list(x2)
|
|
512
|
+
er = est(spec, rd, x2, hold=hold)
|
|
513
|
+
if log is not None:
|
|
514
|
+
log.append(("profile", spec, list(x2), er))
|
|
515
|
+
prof.est = er
|
|
516
|
+
if er.ifault != 0:
|
|
517
|
+
return list(x), prof
|
|
518
|
+
prof.ok, prof.ll, prof.x2 = True, er.ll, er.x
|
|
519
|
+
xf = list(er.x[:nhead])
|
|
520
|
+
for k in range(spec.nf):
|
|
521
|
+
xf += [hold["F"][k][i][j] for i in range(M) for j in range(M) if spec.f_free(i, j)]
|
|
522
|
+
xf += [float(v) for v in theta_pack(hold["Th"], spec.theta_class(), spec.q, M, r0)]
|
|
523
|
+
xf += [hold["S"][i][i] for i in range(1, M)]
|
|
524
|
+
if not spec.diag_cov:
|
|
525
|
+
xf += [hold["S"][i][j] for i in range(1, M) for j in range(i)]
|
|
526
|
+
xf += list(er.x[nhead:])
|
|
527
|
+
assert len(xf) == spec.npar()
|
|
528
|
+
return xf, prof
|