quantlab-factor-agent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- factor_agent/KUNQUANT_LICENSE +201 -0
- factor_agent/__init__.py +8 -0
- factor_agent/_acceptance_model.py +93 -0
- factor_agent/_admission.py +168 -0
- factor_agent/_expression_fold.py +48 -0
- factor_agent/_feedback.py +399 -0
- factor_agent/_max_drawdown.py +51 -0
- factor_agent/_model_cache.py +100 -0
- factor_agent/_scheduler.py +35 -0
- factor_agent/acceptance.py +808 -0
- factor_agent/accepted/__init__.py +1 -0
- factor_agent/aggregations.py +122 -0
- factor_agent/campaign.py +797 -0
- factor_agent/candidate.py +51 -0
- factor_agent/config.py +651 -0
- factor_agent/derived.py +98 -0
- factor_agent/evaluator.py +1096 -0
- factor_agent/events.py +343 -0
- factor_agent/exported.py +177 -0
- factor_agent/gate/__init__.py +359 -0
- factor_agent/gate/_canonical.py +331 -0
- factor_agent/gate/_checks.py +85 -0
- factor_agent/gate/_constraints.py +85 -0
- factor_agent/gate/_dimension.py +177 -0
- factor_agent/gate/_originality.py +146 -0
- factor_agent/gate/_sandbox.py +105 -0
- factor_agent/gate/graph.py +223 -0
- factor_agent/gate/stages.py +383 -0
- factor_agent/gate/vocabulary.py +1097 -0
- factor_agent/ledger.py +85 -0
- factor_agent/llm/__init__.py +0 -0
- factor_agent/llm/_common.py +18 -0
- factor_agent/llm/_raw.py +78 -0
- factor_agent/llm/anthropic.py +100 -0
- factor_agent/llm/base.py +124 -0
- factor_agent/llm/claude_code.py +101 -0
- factor_agent/llm/log.py +74 -0
- factor_agent/llm/openai_compatible.py +113 -0
- factor_agent/llm/replay.py +59 -0
- factor_agent/llm/scripted.py +39 -0
- factor_agent/period_lag.py +91 -0
- factor_agent/prompts.py +557 -0
- factor_agent/researcher.py +615 -0
- factor_agent/scoring.py +143 -0
- factor_agent/segments.py +184 -0
- factor_agent/session.py +500 -0
- factor_agent/snapshot.py +219 -0
- factor_agent/web/__init__.py +7 -0
- factor_agent/web/_acceptance.py +145 -0
- factor_agent/web/_config.py +51 -0
- factor_agent/web/_diagnosis.py +110 -0
- factor_agent/web/_files.py +78 -0
- factor_agent/web/_health.py +207 -0
- factor_agent/web/_ic.py +27 -0
- factor_agent/web/_ledger.py +293 -0
- factor_agent/web/_memory.py +42 -0
- factor_agent/web/_results.py +77 -0
- factor_agent/web/_round.py +219 -0
- factor_agent/web/_trials.py +112 -0
- factor_agent/web/app.py +118 -0
- factor_agent/web/run_view.py +329 -0
- factor_agent/web/static/assets/index-9QjjJgl3.css +1 -0
- factor_agent/web/static/assets/index-iOE9ei_S.js +74 -0
- factor_agent/web/static/index.html +13 -0
- factor_agent/zoo.py +136 -0
- quantlab_factor_agent-0.1.0.dist-info/METADATA +172 -0
- quantlab_factor_agent-0.1.0.dist-info/RECORD +72 -0
- quantlab_factor_agent-0.1.0.dist-info/WHEEL +5 -0
- quantlab_factor_agent-0.1.0.dist-info/entry_points.txt +2 -0
- quantlab_factor_agent-0.1.0.dist-info/licenses/LICENSE +21 -0
- quantlab_factor_agent-0.1.0.dist-info/licenses/factor_agent/KUNQUANT_LICENSE +201 -0
- quantlab_factor_agent-0.1.0.dist-info/top_level.txt +1 -0
factor_agent/session.py
ADDED
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
"""Session columns: intraday graphs computed on intraday bars, one value per stock and session.
|
|
2
|
+
|
|
3
|
+
``SessionFactor`` is a quantlab KunQuant factor over an intraday dataset
|
|
4
|
+
(one-minute trade bars cut by session). Each of its graphs ends in a session
|
|
5
|
+
aggregation (``factor_agent.aggregations``) and is one output; at a
|
|
6
|
+
session's last bar the output holds the aggregation of that session's bars
|
|
7
|
+
and every other bar is NaN, so resampled to ``"1d"`` with ``"last"`` the
|
|
8
|
+
factor is one value per stock and session::
|
|
9
|
+
|
|
10
|
+
daily = SessionFactor(config).resample("1d", "last").compute(start, end)
|
|
11
|
+
|
|
12
|
+
KunQuant has no notion of a session, so the factor adds the session inputs
|
|
13
|
+
before the graph runs, as quantlab's ``IntradayStats`` computes them
|
|
14
|
+
(``quantlab.factor.predefined.intraday.session_inputs``): a bar's position in
|
|
15
|
+
its session (``session_position``, 0 for the first bar), the bars left to the
|
|
16
|
+
session's last (``session_bars_left``, 0 for the last) and the last trade
|
|
17
|
+
price at or before the bar within the session (``last_price``, from the
|
|
18
|
+
bars' ``close``). A graph reads them as it reads the bars' own columns.
|
|
19
|
+
Nothing assumes a number of bars per session: a session is the bars the
|
|
20
|
+
dataset cuts into one ``"1d"`` bar (``resample_labels``).
|
|
21
|
+
|
|
22
|
+
Each aggregation is expanded into KunQuant accumulators: running values
|
|
23
|
+
that restart at each session's first bar and skip a missing value, sums
|
|
24
|
+
(the count, the sum, sums of powers) or folds (the first, last, least and
|
|
25
|
+
greatest value present). Moments are sums of the values less the session's
|
|
26
|
+
first value, as quantlab's session statistics compute them, so that in
|
|
27
|
+
float32 a price's deviations are not lost to its level. An accumulator is
|
|
28
|
+
named after what it accumulates, the hash of what the aggregation's input
|
|
29
|
+
subgraph computes (``computation_hashes``; of each input's, joined, for
|
|
30
|
+
two) and the kind
|
|
31
|
+
(``<hash>_count``, ``<hash>_first``, ``<hash>_shifted_sumsq``...): one
|
|
32
|
+
function builds one accumulator per name, and KunQuant's expression folding
|
|
33
|
+
merges two accumulators only when their names are the same, so two
|
|
34
|
+
aggregations of one subgraph in one function, in one graph or in two, share
|
|
35
|
+
their sums (``SessionMean(x)`` and ``SessionStd(x)``: the count, the first
|
|
36
|
+
value and the shifted sums of ``x`` and of its square).
|
|
37
|
+
|
|
38
|
+
All the graphs of one factor are one KunQuant function, and an operator
|
|
39
|
+
that computes what an earlier one computes (in the same graph or another:
|
|
40
|
+
the same ``computation_hashes``) is that operator, so a subtree two
|
|
41
|
+
candidates share is computed once, and a graph's values are the same
|
|
42
|
+
whichever graphs it is computed with.
|
|
43
|
+
|
|
44
|
+
``SessionColumns`` is the provider of session columns to a daily factor
|
|
45
|
+
(``factor_agent.derived``): it computes the intraday graphs of a stage split
|
|
46
|
+
(``factor_agent.gate.stages``) with ``SessionFactor``, in chunks of sessions,
|
|
47
|
+
and puts each session's value on its day of the daily panel.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
from collections.abc import Callable, Mapping
|
|
51
|
+
|
|
52
|
+
import numpy as np
|
|
53
|
+
import pandas as pd
|
|
54
|
+
import xarray as xr
|
|
55
|
+
from KunQuant.Driver import compileit
|
|
56
|
+
from KunQuant.Op import Builder, ConstantOp, Input, OpBase, Output
|
|
57
|
+
from KunQuant.ops import (
|
|
58
|
+
Accumulator, And, Equals, LessEqual, LessThan, Max, Min, Not, Select, SetAccumulator, SetInfOrNanToValue, Sqrt,
|
|
59
|
+
)
|
|
60
|
+
from KunQuant.Stage import Function
|
|
61
|
+
from quantlab.factor.chunking import ChunkedBuilder
|
|
62
|
+
from quantlab.factor.config import FactorConfig
|
|
63
|
+
from quantlab.factor.kunquant import BATCH_OPTIONS, FactorKunQuant
|
|
64
|
+
from quantlab.factor.predefined.intraday import LAST_PRICE, SESSION_BARS_LEFT, SESSION_POSITION, session_inputs
|
|
65
|
+
from quantlab.utils.date_range import as_label
|
|
66
|
+
from quantlab.utils.resample import resolve_resample_how
|
|
67
|
+
|
|
68
|
+
from factor_agent.aggregations import (
|
|
69
|
+
SessionAggregation, SessionCorr, SessionCount, SessionFirst, SessionLast, SessionMax, SessionMean, SessionMin,
|
|
70
|
+
SessionSkew, SessionStd, SessionSum, SessionWeightedMean,
|
|
71
|
+
)
|
|
72
|
+
from factor_agent.derived import Provider
|
|
73
|
+
from factor_agent.gate._canonical import computation_hashes
|
|
74
|
+
from factor_agent.gate.graph import Graph, Node
|
|
75
|
+
from factor_agent.gate.vocabulary import resolve_classes
|
|
76
|
+
|
|
77
|
+
#: The inputs the factor adds to the bars' own columns, computed from the bar timestamps and prices.
|
|
78
|
+
SESSION_INPUTS = (SESSION_POSITION, SESSION_BARS_LEFT, LAST_PRICE)
|
|
79
|
+
#: The bars' column the last trade price is carried from: the bar's last trade, missing on a bar without one.
|
|
80
|
+
PRICE = "close"
|
|
81
|
+
#: The bytes per bar and symbol of the arrays ``session_inputs`` computes the session inputs through: the float64
|
|
82
|
+
#: price and closing-auction price it is given, its index, mask and price arrays, and the six float32 inputs it
|
|
83
|
+
#: returns, counted from its code (about 150 bytes) and rounded up.
|
|
84
|
+
SESSION_INPUT_WORK = 160
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _zero_if_missing(value: OpBase) -> OpBase:
|
|
88
|
+
"""Return ``value`` with NaN and infinities replaced by 0."""
|
|
89
|
+
return SetInfOrNanToValue(value, 0.0)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _present(value: OpBase) -> OpBase:
|
|
93
|
+
"""Return 1 where ``value`` is finite, else 0."""
|
|
94
|
+
return _zero_if_missing(value * 0.0 + 1.0)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _missing(value: OpBase) -> OpBase:
|
|
98
|
+
"""Whether ``value`` is NaN or infinite."""
|
|
99
|
+
return LessEqual(_present(value), ConstantOp(0.0))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _beside(column: OpBase, value: OpBase) -> OpBase:
|
|
103
|
+
"""``column``, computed in the stage of ``value``.
|
|
104
|
+
|
|
105
|
+
KunQuant keeps a comparison in the stage of what it compares and cannot
|
|
106
|
+
pass its result to another stage, where it is a mask, not a number: a
|
|
107
|
+
mask of the session position must be made beside what it selects.
|
|
108
|
+
"""
|
|
109
|
+
return column + _zero_if_missing(value) * 0.0
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _nan_unless(condition: OpBase, value: OpBase) -> OpBase:
|
|
113
|
+
"""Return ``value`` where ``condition`` holds, else NaN."""
|
|
114
|
+
return Select(condition, value, ConstantOp("nan"))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class _Expansion:
|
|
118
|
+
"""What an aggregation expands into: its running values, named after what they hold, over one function's inputs."""
|
|
119
|
+
|
|
120
|
+
def __init__(self, inputs: dict[str, OpBase]):
|
|
121
|
+
self._inputs = inputs
|
|
122
|
+
self._running: dict[str, OpBase] = {}
|
|
123
|
+
|
|
124
|
+
def input(self, name: str) -> OpBase:
|
|
125
|
+
"""The function's ``Input`` of ``name``, added when the graphs did not read it."""
|
|
126
|
+
if name not in self._inputs:
|
|
127
|
+
self._inputs[name] = Input(name)
|
|
128
|
+
return self._inputs[name]
|
|
129
|
+
|
|
130
|
+
def running(
|
|
131
|
+
self, name: str, step: Callable[[OpBase], OpBase], reads: OpBase, start: float | str = 0.0,
|
|
132
|
+
) -> OpBase:
|
|
133
|
+
"""The running value ``name`` over the session's bars up to the current one: ``step`` of its value before.
|
|
134
|
+
|
|
135
|
+
``step`` reads ``reads`` besides the value before, which is
|
|
136
|
+
``start`` before the session's first bar. It is the KunQuant
|
|
137
|
+
accumulator ``name``, which must say what it holds: a second call
|
|
138
|
+
under the same name in one function gives the first's value,
|
|
139
|
+
whatever its ``step``.
|
|
140
|
+
"""
|
|
141
|
+
if name not in self._running:
|
|
142
|
+
# KunQuant needs an accumulator and its update in one stage, and places the accumulator with its
|
|
143
|
+
# input: on the position alone, it would come before a cross-sectional operator whose values it sums.
|
|
144
|
+
total = Accumulator(reads, name, is_whole_time_required=True)
|
|
145
|
+
position = _beside(self.input(SESSION_POSITION), reads)
|
|
146
|
+
first = LessEqual(position, ConstantOp(0.0))
|
|
147
|
+
always = Equals(position, position) # the position is finite
|
|
148
|
+
self._running[name] = SetAccumulator(total, always, step(Select(first, ConstantOp(start), total)))
|
|
149
|
+
return self._running[name]
|
|
150
|
+
|
|
151
|
+
def session_sum(self, term: OpBase, name: str) -> OpBase:
|
|
152
|
+
"""The running sum of ``term`` over the session's bars up to the current one, a missing term adding 0."""
|
|
153
|
+
value = _zero_if_missing(term)
|
|
154
|
+
return self.running(name, lambda before: before + value, value)
|
|
155
|
+
|
|
156
|
+
def session_fold(self, x: OpBase, name: str, combine: Callable[[OpBase, OpBase], OpBase]) -> OpBase:
|
|
157
|
+
"""``combine(held, x)`` over the values of ``x`` present in the session so far; NaN before the first."""
|
|
158
|
+
def step(held: OpBase) -> OpBase:
|
|
159
|
+
return Select(_missing(x), held, Select(_missing(held), x, combine(held, x)))
|
|
160
|
+
return self.running(name, step, x, "nan")
|
|
161
|
+
|
|
162
|
+
def first(self, x: OpBase, key: str) -> OpBase:
|
|
163
|
+
"""The first value of ``x`` present in the session so far, NaN before it: ``SessionFirst``'s and the shift."""
|
|
164
|
+
return self.session_fold(x, f"{key}_first", lambda held, value: held)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _count(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
168
|
+
[x], [key] = inputs, keys
|
|
169
|
+
return expansion.session_sum(_present(x), f"{key}_count")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _sum(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
173
|
+
[x], [key] = inputs, keys
|
|
174
|
+
return expansion.session_sum(x, f"{key}_sum")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _shifted_sums(expansion: _Expansion, x: OpBase, key: str, powers: int) -> list[OpBase]:
|
|
178
|
+
"""The session sums of the first ``powers`` powers of ``x`` less its session's first value.
|
|
179
|
+
|
|
180
|
+
A price's deviations in a session are orders below its level: float32
|
|
181
|
+
sums of the prices themselves would lose them to cancellation, sums of
|
|
182
|
+
the values shifted by the session's first value keep them (as
|
|
183
|
+
quantlab's session statistics do).
|
|
184
|
+
"""
|
|
185
|
+
shifted = x - expansion.first(x, key)
|
|
186
|
+
terms = {
|
|
187
|
+
"shifted_sum": shifted, "shifted_sumsq": shifted * shifted, "shifted_sumcube": shifted * shifted * shifted,
|
|
188
|
+
}
|
|
189
|
+
return [expansion.session_sum(term, f"{key}_{kind}") for kind, term in list(terms.items())[:powers]]
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _mean(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
193
|
+
[x], [key] = inputs, keys
|
|
194
|
+
count = _count(expansion, inputs, keys)
|
|
195
|
+
[s1] = _shifted_sums(expansion, x, key, 1)
|
|
196
|
+
return _nan_unless(LessThan(ConstantOp(0.0), count), expansion.first(x, key) + s1 / count)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _std(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
200
|
+
[x], [key] = inputs, keys
|
|
201
|
+
n = _count(expansion, inputs, keys)
|
|
202
|
+
s1, s2 = _shifted_sums(expansion, x, key, 2)
|
|
203
|
+
m2 = Max(s2 - s1 * s1 / n, ConstantOp(0.0))
|
|
204
|
+
# The unbiased standard deviation, as pandas and KunQuant's windows give it.
|
|
205
|
+
return _nan_unless(LessThan(ConstantOp(1.0), n), Sqrt(m2 / (n - 1.0)))
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _skew(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
209
|
+
[x], [key] = inputs, keys
|
|
210
|
+
n = _count(expansion, inputs, keys)
|
|
211
|
+
s1, s2, s3 = _shifted_sums(expansion, x, key, 3)
|
|
212
|
+
mean = s1 / n
|
|
213
|
+
m2 = s2 - s1 * mean
|
|
214
|
+
m3 = s3 - mean * s2 * 3.0 + mean * mean * s1 * 2.0
|
|
215
|
+
# The unbiased skewness, as pandas and KunQuant's windows give it.
|
|
216
|
+
skew = Sqrt(n * (n - 1.0)) / (n - 2.0) * Sqrt(n) * m3 / (m2 * Sqrt(m2))
|
|
217
|
+
return _nan_unless(And(LessThan(ConstantOp(2.0), n), LessThan(ConstantOp(0.0), m2)), skew)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _first(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
221
|
+
[x], [key] = inputs, keys
|
|
222
|
+
return expansion.first(x, key)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _last(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
226
|
+
[x], [key] = inputs, keys
|
|
227
|
+
return expansion.session_fold(x, f"{key}_last", lambda held, value: value)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _min(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
231
|
+
[x], [key] = inputs, keys
|
|
232
|
+
return expansion.session_fold(x, f"{key}_min", Min)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _max(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
236
|
+
[x], [key] = inputs, keys
|
|
237
|
+
return expansion.session_fold(x, f"{key}_max", Max)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _weighted_mean(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
241
|
+
(x, w), key = inputs, "_".join(keys)
|
|
242
|
+
both = _present(x) * _present(w)
|
|
243
|
+
weight = expansion.session_sum(w * both, f"{key}_weight")
|
|
244
|
+
shifted = expansion.session_sum((x - expansion.first(x, keys[0])) * w * both, f"{key}_shifted_weighted_sum")
|
|
245
|
+
return _nan_unless(Not(Equals(weight, ConstantOp(0.0))), expansion.first(x, keys[0]) + shifted / weight)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _corr(expansion: _Expansion, inputs: list[OpBase], keys: list[str]) -> OpBase:
|
|
249
|
+
key = "_".join(keys)
|
|
250
|
+
both = _present(inputs[0]) * _present(inputs[1])
|
|
251
|
+
# Each input less its first value present in the session, which comes no later than the pair's first bar.
|
|
252
|
+
x, y = ((value - expansion.first(value, k)) * both for value, k in zip(inputs, keys))
|
|
253
|
+
n = expansion.session_sum(both, f"{key}_count")
|
|
254
|
+
sx, sy = (expansion.session_sum(v, f"{key}_shifted_sum_{i}") for i, v in enumerate((x, y)))
|
|
255
|
+
sxx, syy = (expansion.session_sum(v * v, f"{key}_shifted_sumsq_{i}") for i, v in enumerate((x, y)))
|
|
256
|
+
sxy = expansion.session_sum(x * y, f"{key}_shifted_cross")
|
|
257
|
+
xx, yy, xy = sxx - sx * sx / n, syy - sy * sy / n, sxy - sx * sy / n
|
|
258
|
+
return _nan_unless(And(LessThan(ConstantOp(0.0), xx), LessThan(ConstantOp(0.0), yy)), xy / Sqrt(xx * yy))
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
#: How each aggregation expands: from its input operators and the keys naming their sums (the computation hash of
|
|
262
|
+
#: each input's subgraph), its value at each bar.
|
|
263
|
+
_EXPANSIONS: dict[type, Callable[[_Expansion, list[OpBase], list[str]], OpBase]] = {
|
|
264
|
+
SessionMean: _mean, SessionSum: _sum, SessionCount: _count, SessionStd: _std, SessionSkew: _skew,
|
|
265
|
+
SessionFirst: _first, SessionLast: _last, SessionMin: _min, SessionMax: _max,
|
|
266
|
+
SessionWeightedMean: _weighted_mean, SessionCorr: _corr,
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def session_function(graphs: Mapping[str, Graph]) -> Function:
|
|
271
|
+
"""One KunQuant function with an output per graph, named after it: its value at each session's last bar.
|
|
272
|
+
|
|
273
|
+
Every aggregation is expanded into its session sums; the bars before a
|
|
274
|
+
session's last are NaN. A subtree is computed once however often the
|
|
275
|
+
graphs hold it: an operator computing what an earlier one of any graph
|
|
276
|
+
computes is that operator (``Mul(close, volume)`` and
|
|
277
|
+
``Mul(volume, close)`` are one, ``x + 1e8 - 1e8`` and ``x`` two), so an
|
|
278
|
+
aggregation of one subgraph in two graphs sums it once. Operators in or after a loop are kept as
|
|
279
|
+
built, a loop's body being its own.
|
|
280
|
+
|
|
281
|
+
Raises
|
|
282
|
+
------
|
|
283
|
+
ValueError
|
|
284
|
+
If a graph does not end in a session aggregation.
|
|
285
|
+
"""
|
|
286
|
+
inputs: dict[str, OpBase] = {}
|
|
287
|
+
expansion = _Expansion(inputs)
|
|
288
|
+
ops: list[OpBase] = []
|
|
289
|
+
# The operator computing each computation hash, over every graph so far.
|
|
290
|
+
built: dict[str, OpBase] = {}
|
|
291
|
+
for name, graph in graphs.items():
|
|
292
|
+
if not _is_aggregation(graph, graph.nodes[-1]):
|
|
293
|
+
raise ValueError(f"SessionFactor: graph {name} does not end in a session aggregation")
|
|
294
|
+
hashes = computation_hashes(graph)
|
|
295
|
+
shared = _shareable(graph)
|
|
296
|
+
# Graph.build makes an operator per node other than an Input, in node order.
|
|
297
|
+
nodes = iter([i for i, node in enumerate(graph.nodes) if node.op != "Input"])
|
|
298
|
+
output, new = graph.build(inputs)
|
|
299
|
+
replaced: dict[OpBase, OpBase] = {}
|
|
300
|
+
for op in new:
|
|
301
|
+
if isinstance(op, Input):
|
|
302
|
+
ops.append(op)
|
|
303
|
+
continue
|
|
304
|
+
i = next(nodes)
|
|
305
|
+
if shared[i] and hashes[i] in built:
|
|
306
|
+
replaced[op] = built[hashes[i]]
|
|
307
|
+
continue
|
|
308
|
+
op.replace_inputs(replaced)
|
|
309
|
+
if isinstance(op, SessionAggregation):
|
|
310
|
+
keys = [hashes[j] for j in graph.nodes[i].inputs]
|
|
311
|
+
builder = Builder()
|
|
312
|
+
with builder:
|
|
313
|
+
replaced[op] = _EXPANSIONS[type(op)](expansion, op.inputs, keys)
|
|
314
|
+
ops += builder.ops
|
|
315
|
+
else:
|
|
316
|
+
ops.append(op)
|
|
317
|
+
if shared[i]:
|
|
318
|
+
built[hashes[i]] = replaced.get(op, op)
|
|
319
|
+
value = replaced.get(output, output)
|
|
320
|
+
builder = Builder()
|
|
321
|
+
with builder:
|
|
322
|
+
last = LessEqual(_beside(expansion.input(SESSION_BARS_LEFT), value), ConstantOp(0.0))
|
|
323
|
+
Output(Select(last, value, ConstantOp("nan")), name)
|
|
324
|
+
ops += builder.ops
|
|
325
|
+
return Function(ops)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _shareable(graph: Graph) -> list[bool]:
|
|
329
|
+
"""Whether each node of ``graph`` may be taken from another graph: neither a loop, in one nor after one."""
|
|
330
|
+
loops = {node.parent for node in graph.nodes if node.parent is not None}
|
|
331
|
+
shareable: list[bool] = []
|
|
332
|
+
for i, node in enumerate(graph.nodes):
|
|
333
|
+
shareable.append(i not in loops and node.parent is None and all(shareable[j] for j in node.inputs))
|
|
334
|
+
return shareable
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _is_aggregation(graph: Graph, node: Node) -> bool:
|
|
338
|
+
"""Whether ``node`` of ``graph`` is a session aggregation."""
|
|
339
|
+
return issubclass((graph.classes or {}).get(node.op, object), SessionAggregation)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class SessionFactor(FactorKunQuant):
|
|
343
|
+
"""Intraday graphs, each ending in a session aggregation, on an intraday dataset; see the module docstring.
|
|
344
|
+
|
|
345
|
+
The graphs, name to ``Graph.to_json()``, are ``config.kwargs["graphs"]``
|
|
346
|
+
and the class path of each operator they record
|
|
347
|
+
``config.kwargs["classes"]`` (``Vocabulary.class_paths``).
|
|
348
|
+
``config.data_columns`` must hold the dataset columns they read
|
|
349
|
+
(``data_columns``). Batch mode only: the session inputs are computed
|
|
350
|
+
from a whole panel.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
def _validate_config(self) -> None:
|
|
354
|
+
super()._validate_config()
|
|
355
|
+
if self.config.mode != "batch":
|
|
356
|
+
raise ValueError(f"{self.class_name}: batch mode only; the session inputs are computed from a whole panel")
|
|
357
|
+
|
|
358
|
+
def compute(self, start, end) -> xr.Dataset:
|
|
359
|
+
"""Compute as ``FactorKunQuant`` does, a resample to ``"1d"`` with ``"last"`` taking each session's last bar.
|
|
360
|
+
|
|
361
|
+
Every output holds a value on its session's last bar only, so a
|
|
362
|
+
session's last value is the one on that bar: the bars are read and
|
|
363
|
+
computed as for any resample (the padding around the range
|
|
364
|
+
included), and the sessions labelled in the range keep their last
|
|
365
|
+
bar's values, without grouping the minute panel by day. Any other
|
|
366
|
+
resample is ``FactorKunQuant``'s.
|
|
367
|
+
"""
|
|
368
|
+
config = self.config
|
|
369
|
+
last_only = config.resample_freq == "1d" and set(
|
|
370
|
+
resolve_resample_how(config.resample_how, list(self.get_factor_names()), self.class_name).values()
|
|
371
|
+
) == {"last"}
|
|
372
|
+
if not last_only:
|
|
373
|
+
return super().compute(start, end)
|
|
374
|
+
inputs = config.dataset.panel(*self._input_range(start, end), variables=self._input_variables())
|
|
375
|
+
panel = self._compute_panel(inputs)[list(self.get_factor_names())]
|
|
376
|
+
sessions = np.asarray(config.dataset.resample_labels(panel["timestamp"].values, "1d"))
|
|
377
|
+
last = np.flatnonzero(np.append(sessions[1:] != sessions[:-1], True))
|
|
378
|
+
daily = panel.isel(timestamp=last).assign_coords(timestamp=sessions[last])
|
|
379
|
+
return daily.sel(timestamp=slice(as_label(start), as_label(end))).transpose("timestamp", "symbol")
|
|
380
|
+
|
|
381
|
+
def _recorded(self) -> dict:
|
|
382
|
+
return dict((self.config.kwargs or {})["graphs"])
|
|
383
|
+
|
|
384
|
+
def _graphs(self) -> dict[str, Graph]:
|
|
385
|
+
classes = resolve_classes(dict((self.config.kwargs or {})["classes"]))
|
|
386
|
+
return {name: Graph.from_json(graph, classes) for name, graph in self._recorded().items()}
|
|
387
|
+
|
|
388
|
+
def _get_factor_names(self) -> tuple[str, ...]:
|
|
389
|
+
return tuple(self._recorded())
|
|
390
|
+
|
|
391
|
+
def _get_factor_func(self) -> Function:
|
|
392
|
+
return session_function(self._graphs())
|
|
393
|
+
|
|
394
|
+
def cell_bytes(self) -> tuple[int, int]:
|
|
395
|
+
"""Return the bytes one bar of one symbol takes: at peak, and as output; see ``Factor.cell_bytes``.
|
|
396
|
+
|
|
397
|
+
The peak counts, at 8 bytes each as ``FactorKunQuant`` does, the
|
|
398
|
+
columns read, the session inputs, every array KunQuant holds for
|
|
399
|
+
each bar between two of its stages (a cross-sectional operator's
|
|
400
|
+
values, say: the ``TEMP`` buffers of the generated module) and the
|
|
401
|
+
outputs, and besides them the arrays ``session_inputs`` computes
|
|
402
|
+
through (``SESSION_INPUT_WORK``), as if all were held at once.
|
|
403
|
+
"""
|
|
404
|
+
output = 8 * self.num_factors
|
|
405
|
+
source = "\n".join(compileit(
|
|
406
|
+
self._get_factor_func(), self.class_name, input_layout="TS", output_layout="TS",
|
|
407
|
+
options=dict(BATCH_OPTIONS),
|
|
408
|
+
))
|
|
409
|
+
held = len(self._input_variables()) + len(SESSION_INPUTS) + source.count("BufferKind::TEMP")
|
|
410
|
+
return 8 * held + output + SESSION_INPUT_WORK, output
|
|
411
|
+
|
|
412
|
+
def _kunquant_inputs(self, inputs: xr.Dataset) -> tuple[dict[str, np.ndarray], np.ndarray, np.ndarray]:
|
|
413
|
+
arrays, symbols, timestamps = super()._kunquant_inputs(inputs)
|
|
414
|
+
session = np.asarray(self.config.dataset.resample_labels(np.asarray(timestamps), "1d"))
|
|
415
|
+
close = arrays[PRICE].astype(np.float64)
|
|
416
|
+
# Only the position, the bars left and the last price are read; the opens and auctions do not enter them.
|
|
417
|
+
added = session_inputs(session, close, close, np.full_like(close, np.nan))
|
|
418
|
+
arrays.update({name: added[name] for name in SESSION_INPUTS})
|
|
419
|
+
read = _inputs(self._recorded()) | {SESSION_POSITION, SESSION_BARS_LEFT}
|
|
420
|
+
return {name: values for name, values in arrays.items() if name in read}, symbols, timestamps
|
|
421
|
+
|
|
422
|
+
@staticmethod
|
|
423
|
+
def data_columns(graphs: Mapping[str, list]) -> tuple[str, ...]:
|
|
424
|
+
"""The dataset columns the recorded graphs read besides the session inputs, and the price, sorted."""
|
|
425
|
+
return tuple(sorted((_inputs(graphs) - set(SESSION_INPUTS)) | {PRICE}))
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _inputs(graphs: Mapping[str, list]) -> set[str]:
|
|
429
|
+
"""The names of the ``Input`` nodes of recorded graphs."""
|
|
430
|
+
return {dict(node[1])["name"] for graph in graphs.values() for node in graph if node[0] == "Input"}
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
class SessionColumns(Provider):
|
|
434
|
+
"""The session columns of intraday graphs, filled for a daily factor from an intraday dataset.
|
|
435
|
+
|
|
436
|
+
``graphs`` maps each session column to its intraday graph
|
|
437
|
+
(``Graph.to_json()``, by the names ``stages.split`` gives them) and
|
|
438
|
+
``classes`` gives the class path of every operator they record
|
|
439
|
+
(``Vocabulary.class_paths``). ``fill`` computes them with
|
|
440
|
+
``SessionFactor`` on ``dataset``, the intraday dataset, over the days of
|
|
441
|
+
the daily axes, and puts each session's value on its day: a session's
|
|
442
|
+
label is its date, and a symbol is matched by its label as text, so the
|
|
443
|
+
permatickers of an integer axis find the symbols of a text one. A day
|
|
444
|
+
without a session (before the intraday data begins, say) and a symbol
|
|
445
|
+
the intraday dataset does not hold are NaN. It reads no column of the
|
|
446
|
+
daily dataset and needs no warm-up: an intraday graph sees its own
|
|
447
|
+
session only.
|
|
448
|
+
|
|
449
|
+
The graphs run as one ``SessionFactor``, so a subtree several of them
|
|
450
|
+
hold is computed once, and in chunks of whole sessions through
|
|
451
|
+
quantlab's ``ChunkedBuilder``: only one chunk's bars are in memory at a
|
|
452
|
+
time, beside the daily values. ``granularity`` is the chunks' calendar
|
|
453
|
+
period (``"month"``, ``"day"``...); ``None`` computes the days whole
|
|
454
|
+
when they fit in the memory available (``SessionFactor.cell_bytes``)
|
|
455
|
+
and otherwise in the coarsest period whose chunks do. The values do not depend on it. The dataset must
|
|
456
|
+
read a store (``stored_symbols``), as a campaign's intraday dataset
|
|
457
|
+
does.
|
|
458
|
+
"""
|
|
459
|
+
|
|
460
|
+
def __init__(
|
|
461
|
+
self, dataset, graphs: Mapping[str, list], classes: Mapping[str, str], njobs: int = 1,
|
|
462
|
+
granularity: str | None = None,
|
|
463
|
+
):
|
|
464
|
+
self._dataset = dataset
|
|
465
|
+
self._graphs = {name: list(graph) for name, graph in graphs.items()}
|
|
466
|
+
self._classes = dict(classes)
|
|
467
|
+
self._njobs = njobs
|
|
468
|
+
self._granularity = granularity
|
|
469
|
+
self.columns = tuple(self._graphs)
|
|
470
|
+
|
|
471
|
+
def compile(self) -> None:
|
|
472
|
+
"""Compile the session columns' graphs as ``fill`` does, without reading the dataset.
|
|
473
|
+
|
|
474
|
+
Raises what KunQuant raises for a graph it cannot compile.
|
|
475
|
+
"""
|
|
476
|
+
self._factor()._make()
|
|
477
|
+
|
|
478
|
+
def fill(self, dataset, panel, arrays, symbols, timestamps) -> dict[str, np.ndarray]:
|
|
479
|
+
days = pd.DatetimeIndex(timestamps).normalize()
|
|
480
|
+
factor = self._factor()
|
|
481
|
+
builder = ChunkedBuilder(factor.resample("1d", "last"), granularity=self._granularity)
|
|
482
|
+
daily = builder.compute(days[0].strftime("%Y-%m-%d"), days[-1].strftime("%Y-%m-%d"))
|
|
483
|
+
rows = pd.DatetimeIndex(daily["timestamp"].values).normalize().get_indexer(days)
|
|
484
|
+
columns = pd.Index([str(s) for s in daily["symbol"].values]).get_indexer([str(s) for s in symbols])
|
|
485
|
+
found = np.ix_(rows >= 0, columns >= 0)
|
|
486
|
+
read = np.ix_(rows[rows >= 0], columns[columns >= 0])
|
|
487
|
+
filled = {}
|
|
488
|
+
for name in self.columns:
|
|
489
|
+
values = np.full((len(days), len(symbols)), np.nan, dtype=np.float32)
|
|
490
|
+
values[found] = daily[name].transpose("timestamp", "symbol").values[read]
|
|
491
|
+
filled[name] = values
|
|
492
|
+
return filled
|
|
493
|
+
|
|
494
|
+
def _factor(self) -> SessionFactor:
|
|
495
|
+
"""The session factor of the graphs on the dataset."""
|
|
496
|
+
return SessionFactor(FactorConfig(
|
|
497
|
+
warmup_bars=0, dataset=self._dataset, mode="batch", njobs=self._njobs,
|
|
498
|
+
data_columns=SessionFactor.data_columns(self._graphs),
|
|
499
|
+
kwargs={"graphs": self._graphs, "classes": self._classes},
|
|
500
|
+
))
|