topstep-backtest 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- topstep_backtest/__init__.py +43 -0
- topstep_backtest/clock/__init__.py +1 -0
- topstep_backtest/clock/live_clock.py +82 -0
- topstep_backtest/clock/test_clock.py +133 -0
- topstep_backtest/core/__init__.py +1 -0
- topstep_backtest/core/ids.py +23 -0
- topstep_backtest/core/instruments.py +167 -0
- topstep_backtest/core/money.py +160 -0
- topstep_backtest/core/time.py +125 -0
- topstep_backtest/data/__init__.py +1 -0
- topstep_backtest/data/clean.py +86 -0
- topstep_backtest/data/feed.py +56 -0
- topstep_backtest/data/synthetic.py +137 -0
- topstep_backtest/data/validator.py +215 -0
- topstep_backtest/data/wrangler.py +306 -0
- topstep_backtest/engine/__init__.py +1 -0
- topstep_backtest/engine/backtest.py +209 -0
- topstep_backtest/execution/__init__.py +1 -0
- topstep_backtest/execution/rejections.py +53 -0
- topstep_backtest/execution/sim_broker.py +1436 -0
- topstep_backtest/fills/__init__.py +1 -0
- topstep_backtest/fills/bar_fill.py +268 -0
- topstep_backtest/fills/fees.py +120 -0
- topstep_backtest/fills/path.py +59 -0
- topstep_backtest/harness.py +446 -0
- topstep_backtest/indicators/__init__.py +46 -0
- topstep_backtest/indicators/base.py +57 -0
- topstep_backtest/indicators/library.py +303 -0
- topstep_backtest/indicators/talib_adapter.py +657 -0
- topstep_backtest/metrics/__init__.py +5 -0
- topstep_backtest/metrics/stats.py +153 -0
- topstep_backtest/protocols.py +473 -0
- topstep_backtest/py.typed +0 -0
- topstep_backtest/rules/__init__.py +1 -0
- topstep_backtest/rules/kernel.py +281 -0
- topstep_backtest/rules/params.py +74 -0
- topstep_backtest/strategy/__init__.py +20 -0
- topstep_backtest/strategy/base.py +118 -0
- topstep_backtest/strategy/symbol.py +344 -0
- topstep_backtest/strategy/tracker.py +151 -0
- topstep_backtest-0.1.0.dist-info/METADATA +250 -0
- topstep_backtest-0.1.0.dist-info/RECORD +44 -0
- topstep_backtest-0.1.0.dist-info/WHEEL +4 -0
- topstep_backtest-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Time spine: int-nanosecond UTC hot path, ET (America/New_York) boundaries.
|
|
2
|
+
|
|
3
|
+
The engine's hot path stores time as ``int`` nanoseconds since the UTC epoch.
|
|
4
|
+
The project's canonical timezone is **ET** — every session boundary below is
|
|
5
|
+
defined in ET and converted tz-aware (Topstep publishes rules in CT, but
|
|
6
|
+
ET = CT + 1h always; the two share the identical US DST schedule).
|
|
7
|
+
|
|
8
|
+
Topstep session boundaries (ET):
|
|
9
|
+
- Globex open: 18:00 previous calendar day (Sun 18:00 for Monday)
|
|
10
|
+
- auto-flatten start: 16:08
|
|
11
|
+
- flatten deadline: 16:10 (positions & working orders force-closed)
|
|
12
|
+
- Globex close: 17:00 (EOD balance snapshot for the MLL ratchet)
|
|
13
|
+
- day reset: 18:00 (DLL reset; next trading day begins)
|
|
14
|
+
|
|
15
|
+
A bar timestamped at/after 18:00 ET belongs to the *next* trading day.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from datetime import date, datetime, time, timedelta
|
|
21
|
+
from zoneinfo import ZoneInfo
|
|
22
|
+
|
|
23
|
+
import msgspec
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"ET",
|
|
27
|
+
"NS_PER_MIN",
|
|
28
|
+
"NS_PER_SEC",
|
|
29
|
+
"TOPSTEP_SESSION",
|
|
30
|
+
"SessionTimes",
|
|
31
|
+
"dt_to_ns",
|
|
32
|
+
"et_time_of",
|
|
33
|
+
"ns_to_dt",
|
|
34
|
+
"ns_to_et",
|
|
35
|
+
"trading_day_of",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
ET = ZoneInfo("America/New_York")
|
|
39
|
+
UTC = ZoneInfo("UTC")
|
|
40
|
+
|
|
41
|
+
NS_PER_SEC = 1_000_000_000
|
|
42
|
+
NS_PER_MIN = 60 * NS_PER_SEC
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def dt_to_ns(dt: datetime) -> int:
|
|
49
|
+
"""tz-aware datetime -> int UTC nanoseconds. Naive datetimes are rejected.
|
|
50
|
+
|
|
51
|
+
Computed with exact integer arithmetic (datetime carries microseconds);
|
|
52
|
+
a float round-trip would lose sub-microsecond exactness at 2026 epochs.
|
|
53
|
+
"""
|
|
54
|
+
if dt.tzinfo is None:
|
|
55
|
+
raise ValueError(f"naive datetime {dt!r}: all engine datetimes must be tz-aware")
|
|
56
|
+
delta = dt.astimezone(UTC) - _EPOCH
|
|
57
|
+
return ((delta.days * 86_400 + delta.seconds) * NS_PER_SEC) + delta.microseconds * 1_000
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def ns_to_dt(ns: int) -> datetime:
|
|
61
|
+
"""int UTC nanoseconds -> tz-aware UTC datetime.
|
|
62
|
+
|
|
63
|
+
Exact integer arithmetic, truncating sub-microsecond remainders toward the
|
|
64
|
+
past (datetime resolution is 1 us). Truncation — never float rounding — so
|
|
65
|
+
an instant strictly before a boundary (e.g. 17:59:59.999999999 ET) can
|
|
66
|
+
never classify as at/after it (the 18:00 trading-day boundary hinges on this).
|
|
67
|
+
"""
|
|
68
|
+
seconds, rem = divmod(ns, NS_PER_SEC)
|
|
69
|
+
return datetime.fromtimestamp(seconds, tz=UTC) + timedelta(microseconds=rem // 1_000)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def ns_to_et(ns: int) -> datetime:
|
|
73
|
+
"""int UTC nanoseconds -> tz-aware ET datetime."""
|
|
74
|
+
return ns_to_dt(ns).astimezone(ET)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def et_time_of(d: date, t: time) -> int:
|
|
78
|
+
"""An ET wall-clock instant on calendar date ``d`` -> int UTC nanoseconds."""
|
|
79
|
+
return dt_to_ns(datetime.combine(d, t, tzinfo=ET))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def trading_day_of(ns: int) -> date:
|
|
83
|
+
"""The Topstep trading day containing UTC-ns instant ``ns``.
|
|
84
|
+
|
|
85
|
+
The day boundary is 18:00 ET: at/after 18:00 the instant belongs to the
|
|
86
|
+
next calendar day's session (Sunday 18:00+ belongs to Monday).
|
|
87
|
+
"""
|
|
88
|
+
local = ns_to_et(ns)
|
|
89
|
+
if local.time() >= time(18, 0):
|
|
90
|
+
return local.date() + timedelta(days=1)
|
|
91
|
+
return local.date()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class SessionTimes(msgspec.Struct, frozen=True):
|
|
95
|
+
"""The ET wall-clock boundaries of a Topstep trading day (all `datetime.time`)."""
|
|
96
|
+
|
|
97
|
+
open_prev_day: time = time(18, 0) # on the PREVIOUS calendar day
|
|
98
|
+
auto_flatten: time = time(16, 8)
|
|
99
|
+
flatten: time = time(16, 10)
|
|
100
|
+
close: time = time(17, 0) # Globex close; EOD-MLL snapshot instant
|
|
101
|
+
reset: time = time(18, 0) # next trading day begins
|
|
102
|
+
|
|
103
|
+
def open_ns(self, day: date) -> int:
|
|
104
|
+
"""Session open (18:00 ET on the previous calendar day)."""
|
|
105
|
+
return et_time_of(day - timedelta(days=1), self.open_prev_day)
|
|
106
|
+
|
|
107
|
+
def auto_flatten_ns(self, day: date) -> int:
|
|
108
|
+
return et_time_of(day, self.auto_flatten)
|
|
109
|
+
|
|
110
|
+
def flatten_ns(self, day: date) -> int:
|
|
111
|
+
return et_time_of(day, self.flatten)
|
|
112
|
+
|
|
113
|
+
def close_ns(self, day: date) -> int:
|
|
114
|
+
return et_time_of(day, self.close)
|
|
115
|
+
|
|
116
|
+
def reset_ns(self, day: date) -> int:
|
|
117
|
+
return et_time_of(day, self.reset)
|
|
118
|
+
|
|
119
|
+
def in_no_trade_window(self, ns: int) -> bool:
|
|
120
|
+
"""True inside the daily no-trade window [flatten, reset) = [16:10, 18:00) ET."""
|
|
121
|
+
t = ns_to_et(ns).time()
|
|
122
|
+
return self.flatten <= t < self.reset
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
TOPSTEP_SESSION = SessionTimes()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""topstep_backtest.data"""
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Pre-run input hygiene: filename symbol inference.
|
|
2
|
+
|
|
3
|
+
``infer_symbol_from_filename`` guards the boundary where user files meet the
|
|
4
|
+
engine, powering the examples' ``--symbol`` cross-check. Feeding one product's
|
|
5
|
+
file under another's tick economics passes every price-grid check and silently
|
|
6
|
+
scales P&L (an ES file run as MNQ is exactly 25x wrong).
|
|
7
|
+
|
|
8
|
+
This module used to also ship ``drop_closed_session_bars``, which discarded
|
|
9
|
+
bars the CME calendar said could not be a real session. It was removed along
|
|
10
|
+
with the holiday calendar itself (see ``core/time.py``): the table it consulted
|
|
11
|
+
was known to disagree with CME on several dates a year, so it silently deleted
|
|
12
|
+
tradable sessions. Filter exchange holidays upstream, in whatever produces your
|
|
13
|
+
bar files.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from pathlib import PurePath
|
|
20
|
+
|
|
21
|
+
from ..core.instruments import SPECS
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"AmbiguousSymbolError",
|
|
25
|
+
"infer_symbol_from_filename",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
_TOKEN_SPLIT = re.compile(r"[^A-Za-z0-9]+")
|
|
29
|
+
# Delivery-month code + 1-, 2-, or 4-digit year: the tail of a separator-free
|
|
30
|
+
# vendor contract-code basename (MNQZ25, esu6, gcm2026). Three digits is no
|
|
31
|
+
# known convention and stays unmatched.
|
|
32
|
+
_MONTH_YEAR = re.compile(r"[FGHJKMNQUVXZ](?:\d{1,2}|\d{4})")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class AmbiguousSymbolError(ValueError):
|
|
36
|
+
"""Distinct product symbols matched distinct tokens of one filename."""
|
|
37
|
+
|
|
38
|
+
def __init__(self, name: str, symbols: tuple[str, ...]) -> None:
|
|
39
|
+
self.symbols = symbols
|
|
40
|
+
super().__init__(
|
|
41
|
+
f"filename {name!r} names several known products {list(symbols)} — "
|
|
42
|
+
"cannot infer which one the file contains"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _token_symbols(token: str) -> frozenset[str]:
|
|
47
|
+
"""SPECS symbols evidenced by one uppercased token: an exact symbol, or a
|
|
48
|
+
contract code (symbol + delivery-month code + year digits, e.g. MNQZ25)."""
|
|
49
|
+
if token in SPECS:
|
|
50
|
+
return frozenset((token,))
|
|
51
|
+
return frozenset(
|
|
52
|
+
symbol
|
|
53
|
+
for symbol in SPECS
|
|
54
|
+
if token.startswith(symbol) and _MONTH_YEAR.fullmatch(token, len(symbol))
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def infer_symbol_from_filename(name: str) -> str | None:
|
|
59
|
+
"""The unique SPECS product symbol evidenced by ``name``'s basename.
|
|
60
|
+
|
|
61
|
+
Token-aware, never substring: the basename splits on non-alphanumeric
|
|
62
|
+
boundaries and each token compares uppercased, matching either a bare
|
|
63
|
+
symbol by EXACT equality (``"sample_mnq_1m.csv"`` matches MNQ, never NQ)
|
|
64
|
+
or a contract code — symbol + delivery-month code + 1/2/4-digit year, the
|
|
65
|
+
separator-free shape vendor exports use (``"MNQZ25.csv"`` matches MNQ,
|
|
66
|
+
``"esu6.parquet"`` matches ES). Both anchor at the token's start, so a
|
|
67
|
+
micro's code can never be misread as its mini's (``"MESU6.csv"`` is MES,
|
|
68
|
+
not ES) and ``"esoteric.csv"`` matches nothing. Only the basename is
|
|
69
|
+
inspected — directory names carry no evidence about the file's contents.
|
|
70
|
+
|
|
71
|
+
Returns ``None`` when no token carries evidence.
|
|
72
|
+
|
|
73
|
+
Raises:
|
|
74
|
+
AmbiguousSymbolError: several DISTINCT symbols matched (e.g.
|
|
75
|
+
``"es_nq_spread.csv"``) — the caller must name the product
|
|
76
|
+
explicitly instead of trusting a guess.
|
|
77
|
+
"""
|
|
78
|
+
basename = PurePath(name).name
|
|
79
|
+
tokens = {token.upper() for token in _TOKEN_SPLIT.split(basename) if token}
|
|
80
|
+
found: set[str] = set()
|
|
81
|
+
for token in tokens:
|
|
82
|
+
found |= _token_symbols(token)
|
|
83
|
+
matches = tuple(sorted(found))
|
|
84
|
+
if len(matches) > 1:
|
|
85
|
+
raise AmbiguousSymbolError(basename, matches)
|
|
86
|
+
return matches[0] if matches else None
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""In-memory bar feed with the ordering invariant enforced at construction.
|
|
2
|
+
|
|
3
|
+
``ListBarFeed`` is the Tier-0 backtest feed: a pre-built sequence of bars,
|
|
4
|
+
validated once up front so the engine can trust the ``DataFeed`` contract —
|
|
5
|
+
non-decreasing ``ts_init`` across the merged stream, strictly increasing
|
|
6
|
+
``ts_init`` per instrument (equal timestamps across DIFFERENT instruments are
|
|
7
|
+
legal and expected; duplicate bars for the same instrument are data corruption
|
|
8
|
+
and rejected loudly).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
from ..protocols import Bar
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from collections.abc import Iterator, Sequence
|
|
19
|
+
|
|
20
|
+
__all__ = ["ListBarFeed"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class ListBarFeed:
|
|
24
|
+
"""A validated, time-ordered in-memory ``DataFeed`` over a bar sequence."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, bars: Sequence[Bar]) -> None:
|
|
27
|
+
self._bars: tuple[Bar, ...] = tuple(bars)
|
|
28
|
+
last_ts: int | None = None
|
|
29
|
+
last_per_instrument: dict[str, int] = {}
|
|
30
|
+
for index, bar in enumerate(self._bars):
|
|
31
|
+
if last_ts is not None and bar.ts_init < last_ts:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f"bars out of order at index {index}: ts_init {bar.ts_init} < "
|
|
34
|
+
f"previous {last_ts} (feed must be non-decreasing in ts_init)"
|
|
35
|
+
)
|
|
36
|
+
contract_id = bar.bar_type.contract_id
|
|
37
|
+
prev = last_per_instrument.get(contract_id)
|
|
38
|
+
if prev is not None and bar.ts_init <= prev:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
f"duplicate ts_init {bar.ts_init} for instrument {contract_id} at "
|
|
41
|
+
f"index {index}: per-instrument ts_init must be strictly increasing"
|
|
42
|
+
)
|
|
43
|
+
last_per_instrument[contract_id] = bar.ts_init
|
|
44
|
+
last_ts = bar.ts_init
|
|
45
|
+
# dict preserves insertion order -> instruments in first-appearance order.
|
|
46
|
+
self._instruments: tuple[str, ...] = tuple(last_per_instrument)
|
|
47
|
+
|
|
48
|
+
def __iter__(self) -> Iterator[Bar]:
|
|
49
|
+
return iter(self._bars)
|
|
50
|
+
|
|
51
|
+
def __len__(self) -> int:
|
|
52
|
+
return len(self._bars)
|
|
53
|
+
|
|
54
|
+
def instruments(self) -> Sequence[str]:
|
|
55
|
+
"""The contract ids this feed emits, in first-appearance order."""
|
|
56
|
+
return self._instruments
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Deterministic synthetic OHLCV bars for examples and tests.
|
|
2
|
+
|
|
3
|
+
A seeded ``random.Random`` walk in INTEGER TICKS — prices are reconstructed
|
|
4
|
+
via ``from_ticks`` so every value is exactly on the instrument's grid and no
|
|
5
|
+
float ever touches a price. Bars are stamped in ET regular trading hours
|
|
6
|
+
starting 09:30 (one per ``unit x unit_number`` step), weekends are skipped,
|
|
7
|
+
and the same seed always reproduces the identical bar tuple.
|
|
8
|
+
|
|
9
|
+
Exchange holidays are NOT skipped — this package ships no holiday calendar
|
|
10
|
+
(see ``core/time.py``). A synthetic tape spanning a market holiday will emit
|
|
11
|
+
a session that would not exist in real data.
|
|
12
|
+
|
|
13
|
+
``days`` counts WEEKDAYS emitted: the generator scans forward from
|
|
14
|
+
``start_day`` and skips Sat/Sun until ``days`` sessions exist.
|
|
15
|
+
``bars_per_day`` is capped at 450 so the last one-minute bar closes at
|
|
16
|
+
17:00 ET — no synthetic bar can ever sit inside the 17:00-18:00 ET
|
|
17
|
+
maintenance halt.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from datetime import time, timedelta
|
|
23
|
+
from decimal import Decimal
|
|
24
|
+
from random import Random
|
|
25
|
+
from typing import TYPE_CHECKING
|
|
26
|
+
|
|
27
|
+
from topstep_sdk import AggregateBarUnit
|
|
28
|
+
|
|
29
|
+
from ..core.money import from_ticks, to_ticks
|
|
30
|
+
from ..core.time import et_time_of
|
|
31
|
+
from ..protocols import Bar, BarType
|
|
32
|
+
from .wrangler import step_ns
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from datetime import date
|
|
36
|
+
|
|
37
|
+
from ..core.instruments import InstrumentSpec
|
|
38
|
+
|
|
39
|
+
__all__ = ["synthetic_bars"]
|
|
40
|
+
|
|
41
|
+
_RTH_OPEN = time(9, 30)
|
|
42
|
+
# 09:30 ET + 450 one-minute bars = 17:00 ET, the start of the 17:00-18:00 ET
|
|
43
|
+
# maintenance halt. More bars would land inside the halt and be rejected by
|
|
44
|
+
# this package's own validator (``in_maintenance_halt``).
|
|
45
|
+
_MAX_BARS_PER_DAY = 450
|
|
46
|
+
_MIN_VOLUME = 100
|
|
47
|
+
_MAX_VOLUME = 5_000
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def synthetic_bars(
|
|
51
|
+
*,
|
|
52
|
+
contract_id: str,
|
|
53
|
+
spec: InstrumentSpec,
|
|
54
|
+
start_day: date,
|
|
55
|
+
days: int,
|
|
56
|
+
seed: int,
|
|
57
|
+
start_price: Decimal,
|
|
58
|
+
bars_per_day: int = 390,
|
|
59
|
+
unit: AggregateBarUnit = AggregateBarUnit.MINUTE,
|
|
60
|
+
unit_number: int = 1,
|
|
61
|
+
drift_ticks_per_day: int = 0,
|
|
62
|
+
vol_ticks: int = 8,
|
|
63
|
+
) -> tuple[Bar, ...]:
|
|
64
|
+
"""Generate ``days`` trading sessions of consistent, on-grid OHLCV bars.
|
|
65
|
+
|
|
66
|
+
``start_price`` must lie on ``spec.tick_size`` (raises ``OffGridError``
|
|
67
|
+
otherwise) and be comfortably above zero for the chosen ``vol_ticks``.
|
|
68
|
+
``drift_ticks_per_day`` is distributed across the day's bars in exact
|
|
69
|
+
integer ticks; ``vol_ticks`` bounds the per-bar random move.
|
|
70
|
+
``bars_per_day`` must be <= 450 (09:30 + 450 minutes = 17:00 ET) so no
|
|
71
|
+
bar lands inside the 17:00-18:00 ET maintenance halt.
|
|
72
|
+
"""
|
|
73
|
+
if days < 0:
|
|
74
|
+
raise ValueError(f"days must be >= 0, got {days}")
|
|
75
|
+
if bars_per_day < 1:
|
|
76
|
+
raise ValueError(f"bars_per_day must be >= 1, got {bars_per_day}")
|
|
77
|
+
if bars_per_day > _MAX_BARS_PER_DAY:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"bars_per_day must be <= {_MAX_BARS_PER_DAY}, got {bars_per_day}: bars "
|
|
80
|
+
"are stamped from the 09:30 ET session open and 09:30 + 450 minutes = "
|
|
81
|
+
"17:00 ET, the start of the 17:00-18:00 ET maintenance halt — extra "
|
|
82
|
+
"bars would land inside the halt and fail validation"
|
|
83
|
+
)
|
|
84
|
+
if vol_ticks < 0:
|
|
85
|
+
raise ValueError(f"vol_ticks must be >= 0, got {vol_ticks}")
|
|
86
|
+
|
|
87
|
+
step = step_ns(unit, unit_number)
|
|
88
|
+
tick = spec.tick_size
|
|
89
|
+
start_ticks = to_ticks(start_price, tick) # raises OffGridError off-grid
|
|
90
|
+
|
|
91
|
+
wick_max = max(1, vol_ticks // 2)
|
|
92
|
+
# Keep the walk high enough that low = min(o, c) - wick stays positive.
|
|
93
|
+
floor_ticks = vol_ticks + wick_max + 1
|
|
94
|
+
if start_ticks < floor_ticks:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"start_price {start_price} ({start_ticks} ticks) is too low for "
|
|
97
|
+
f"vol_ticks={vol_ticks}; need at least {floor_ticks} ticks"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
rng = Random(seed)
|
|
101
|
+
bar_type = BarType(contract_id=contract_id, unit=unit, unit_number=unit_number)
|
|
102
|
+
# Exact integer split of the daily drift across bars (works for negatives).
|
|
103
|
+
base_drift, extra_drift_bars = divmod(drift_ticks_per_day, bars_per_day)
|
|
104
|
+
|
|
105
|
+
bars: list[Bar] = []
|
|
106
|
+
current = start_ticks
|
|
107
|
+
day = start_day
|
|
108
|
+
emitted = 0
|
|
109
|
+
while emitted < days:
|
|
110
|
+
if day.weekday() >= 5:
|
|
111
|
+
day += timedelta(days=1)
|
|
112
|
+
continue
|
|
113
|
+
session_open_ns = et_time_of(day, _RTH_OPEN)
|
|
114
|
+
for i in range(bars_per_day):
|
|
115
|
+
open_ticks = current
|
|
116
|
+
delta = base_drift + (1 if i < extra_drift_bars else 0)
|
|
117
|
+
delta += rng.randint(-vol_ticks, vol_ticks)
|
|
118
|
+
close_ticks = max(open_ticks + delta, floor_ticks)
|
|
119
|
+
high_ticks = max(open_ticks, close_ticks) + rng.randint(0, wick_max)
|
|
120
|
+
low_ticks = min(open_ticks, close_ticks) - rng.randint(0, wick_max)
|
|
121
|
+
ts_event = session_open_ns + i * step
|
|
122
|
+
bars.append(
|
|
123
|
+
Bar(
|
|
124
|
+
bar_type=bar_type,
|
|
125
|
+
ts_event=ts_event,
|
|
126
|
+
ts_init=ts_event + step,
|
|
127
|
+
open=from_ticks(open_ticks, tick),
|
|
128
|
+
high=from_ticks(high_ticks, tick),
|
|
129
|
+
low=from_ticks(low_ticks, tick),
|
|
130
|
+
close=from_ticks(close_ticks, tick),
|
|
131
|
+
volume=rng.randint(_MIN_VOLUME, _MAX_VOLUME),
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
current = close_ticks
|
|
135
|
+
emitted += 1
|
|
136
|
+
day += timedelta(days=1)
|
|
137
|
+
return tuple(bars)
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Bar-stream sanity validation: catch bad data BEFORE it reaches the engine.
|
|
2
|
+
|
|
3
|
+
Severity encoding (documented contract): severity is derived from the issue
|
|
4
|
+
``code`` — codes in ``INFO_CODES`` (currently only ``"session_gap"``) are
|
|
5
|
+
informational; every other code is an ERROR. ``ValidationReport.ok`` is True
|
|
6
|
+
iff no ERROR-severity issue is present.
|
|
7
|
+
|
|
8
|
+
Codes emitted:
|
|
9
|
+
- ``ohlc_inconsistent`` ERROR high < max(open, close) or low > min(open, close)
|
|
10
|
+
- ``stamp_inverted`` ERROR ``ts_event`` >= ``ts_init`` — the bar's open
|
|
11
|
+
does not precede its close (the classic
|
|
12
|
+
off-by-one look-ahead stamp bug)
|
|
13
|
+
- ``negative_volume`` ERROR volume < 0
|
|
14
|
+
- ``duplicate_ts`` ERROR same instrument, repeated ``ts_init``
|
|
15
|
+
- ``non_monotonic`` ERROR ``ts_init`` decreases along the sequence
|
|
16
|
+
- ``off_grid`` ERROR a price off the instrument's tick grid
|
|
17
|
+
- ``in_maintenance_halt`` ERROR ``ts_event`` inside the 17:00-18:00 ET halt
|
|
18
|
+
- ``weekend_bar`` ERROR bar's trading day falls on Sat/Sun
|
|
19
|
+
- ``session_gap`` INFO > 3x step missing between consecutive bars of
|
|
20
|
+
the same instrument within one trading day
|
|
21
|
+
|
|
22
|
+
Ordering checks (``duplicate_ts``, ``non_monotonic``) anchor ``ts_ns`` to the
|
|
23
|
+
bar's ``ts_init``; event-time checks (halt/weekend) anchor to ``ts_event``.
|
|
24
|
+
|
|
25
|
+
There is deliberately no exchange-holiday check: this package ships no holiday
|
|
26
|
+
calendar (see the note in ``core/time.py``), so a bar landing on a market
|
|
27
|
+
holiday is indistinguishable from any other weekday bar here.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from datetime import time
|
|
33
|
+
from typing import TYPE_CHECKING
|
|
34
|
+
|
|
35
|
+
import msgspec
|
|
36
|
+
|
|
37
|
+
from ..core.money import is_on_grid
|
|
38
|
+
from ..core.time import ns_to_et, trading_day_of
|
|
39
|
+
from .wrangler import step_ns
|
|
40
|
+
|
|
41
|
+
if TYPE_CHECKING:
|
|
42
|
+
from collections.abc import Sequence
|
|
43
|
+
|
|
44
|
+
from ..core.instruments import InstrumentSpec
|
|
45
|
+
from ..protocols import Bar, BarType
|
|
46
|
+
|
|
47
|
+
__all__ = ["INFO_CODES", "ValidationIssue", "ValidationReport", "validate_bars"]
|
|
48
|
+
|
|
49
|
+
INFO_CODES: frozenset[str] = frozenset({"session_gap"})
|
|
50
|
+
|
|
51
|
+
_HALT_START = time(17, 0)
|
|
52
|
+
_HALT_END = time(18, 0)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class ValidationIssue(msgspec.Struct, frozen=True):
|
|
56
|
+
"""One finding; severity is derived from ``code`` (see module docstring)."""
|
|
57
|
+
|
|
58
|
+
code: str
|
|
59
|
+
message: str
|
|
60
|
+
ts_ns: int | None = None
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ValidationReport(msgspec.Struct, frozen=True):
|
|
64
|
+
"""All findings for a bar sequence; ``ok`` means no ERROR-severity issues."""
|
|
65
|
+
|
|
66
|
+
issues: tuple[ValidationIssue, ...]
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def ok(self) -> bool:
|
|
70
|
+
return all(issue.code in INFO_CODES for issue in self.issues)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _bar_step_ns(bar_type: BarType, cache: dict[BarType, int | None]) -> int | None:
|
|
74
|
+
"""Step in ns for a bar type, or None if the unit has no fixed step."""
|
|
75
|
+
if bar_type not in cache:
|
|
76
|
+
try:
|
|
77
|
+
cache[bar_type] = step_ns(bar_type.unit, bar_type.unit_number)
|
|
78
|
+
except ValueError:
|
|
79
|
+
cache[bar_type] = None
|
|
80
|
+
return cache[bar_type]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def validate_bars(bars: Sequence[Bar], spec: InstrumentSpec) -> ValidationReport:
|
|
84
|
+
"""Run every check over ``bars`` and return the full report (never raises)."""
|
|
85
|
+
issues: list[ValidationIssue] = []
|
|
86
|
+
last_ts_init: int | None = None
|
|
87
|
+
last_per_instrument: dict[str, Bar] = {}
|
|
88
|
+
step_cache: dict[BarType, int | None] = {}
|
|
89
|
+
|
|
90
|
+
for index, bar in enumerate(bars):
|
|
91
|
+
# --- stamp sanity: open must strictly precede close ----------------
|
|
92
|
+
if bar.ts_event >= bar.ts_init:
|
|
93
|
+
issues.append(
|
|
94
|
+
ValidationIssue(
|
|
95
|
+
code="stamp_inverted",
|
|
96
|
+
message=(
|
|
97
|
+
f"bar {index}: ts_event {bar.ts_event} >= ts_init "
|
|
98
|
+
f"{bar.ts_init} — the bar's open does not precede its "
|
|
99
|
+
"close (stamp-inverted bar; acting on it hands the "
|
|
100
|
+
"strategy one bar of the future)"
|
|
101
|
+
),
|
|
102
|
+
ts_ns=bar.ts_init,
|
|
103
|
+
)
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
# --- volume sanity -------------------------------------------------
|
|
107
|
+
if bar.volume < 0:
|
|
108
|
+
issues.append(
|
|
109
|
+
ValidationIssue(
|
|
110
|
+
code="negative_volume",
|
|
111
|
+
message=f"bar {index}: volume {bar.volume} is negative",
|
|
112
|
+
ts_ns=bar.ts_init,
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
# --- OHLC internal consistency -----------------------------------
|
|
117
|
+
if bar.high < max(bar.open, bar.close) or bar.low > min(bar.open, bar.close):
|
|
118
|
+
issues.append(
|
|
119
|
+
ValidationIssue(
|
|
120
|
+
code="ohlc_inconsistent",
|
|
121
|
+
message=(
|
|
122
|
+
f"bar {index}: OHLC inconsistent (o={bar.open} h={bar.high} "
|
|
123
|
+
f"l={bar.low} c={bar.close})"
|
|
124
|
+
),
|
|
125
|
+
ts_ns=bar.ts_init,
|
|
126
|
+
)
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# --- tick grid ----------------------------------------------------
|
|
130
|
+
for field, price in (
|
|
131
|
+
("open", bar.open),
|
|
132
|
+
("high", bar.high),
|
|
133
|
+
("low", bar.low),
|
|
134
|
+
("close", bar.close),
|
|
135
|
+
):
|
|
136
|
+
if not is_on_grid(price, spec.tick_size):
|
|
137
|
+
issues.append(
|
|
138
|
+
ValidationIssue(
|
|
139
|
+
code="off_grid",
|
|
140
|
+
message=(
|
|
141
|
+
f"bar {index}: {field}={price} is not on the {spec.tick_size} tick grid"
|
|
142
|
+
),
|
|
143
|
+
ts_ns=bar.ts_init,
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
# --- ordering (global) and per-instrument duplicates/gaps ---------
|
|
148
|
+
if last_ts_init is not None and bar.ts_init < last_ts_init:
|
|
149
|
+
issues.append(
|
|
150
|
+
ValidationIssue(
|
|
151
|
+
code="non_monotonic",
|
|
152
|
+
message=(f"bar {index}: ts_init {bar.ts_init} < previous {last_ts_init}"),
|
|
153
|
+
ts_ns=bar.ts_init,
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
last_ts_init = bar.ts_init
|
|
157
|
+
|
|
158
|
+
contract_id = bar.bar_type.contract_id
|
|
159
|
+
prev = last_per_instrument.get(contract_id)
|
|
160
|
+
if prev is not None:
|
|
161
|
+
if bar.ts_init == prev.ts_init:
|
|
162
|
+
issues.append(
|
|
163
|
+
ValidationIssue(
|
|
164
|
+
code="duplicate_ts",
|
|
165
|
+
message=(
|
|
166
|
+
f"bar {index}: duplicate ts_init {bar.ts_init} for "
|
|
167
|
+
f"instrument {contract_id}"
|
|
168
|
+
),
|
|
169
|
+
ts_ns=bar.ts_init,
|
|
170
|
+
)
|
|
171
|
+
)
|
|
172
|
+
else:
|
|
173
|
+
step = _bar_step_ns(bar.bar_type, step_cache)
|
|
174
|
+
same_day = trading_day_of(prev.ts_init) == trading_day_of(bar.ts_init)
|
|
175
|
+
if step is not None and same_day and bar.ts_init - prev.ts_init > 3 * step:
|
|
176
|
+
missing = (bar.ts_init - prev.ts_init) // step - 1
|
|
177
|
+
issues.append(
|
|
178
|
+
ValidationIssue(
|
|
179
|
+
code="session_gap",
|
|
180
|
+
message=(
|
|
181
|
+
f"bar {index}: ~{missing} bars missing before ts_init "
|
|
182
|
+
f"{bar.ts_init} for {contract_id} (intraday gap > 3x step)"
|
|
183
|
+
),
|
|
184
|
+
ts_ns=bar.ts_init,
|
|
185
|
+
)
|
|
186
|
+
)
|
|
187
|
+
last_per_instrument[contract_id] = bar
|
|
188
|
+
|
|
189
|
+
# --- calendar / session placement (keyed off ts_event) ------------
|
|
190
|
+
event_et = ns_to_et(bar.ts_event)
|
|
191
|
+
if _HALT_START <= event_et.time() < _HALT_END:
|
|
192
|
+
issues.append(
|
|
193
|
+
ValidationIssue(
|
|
194
|
+
code="in_maintenance_halt",
|
|
195
|
+
message=(
|
|
196
|
+
f"bar {index}: ts_event {event_et:%Y-%m-%d %H:%M:%S %Z} is "
|
|
197
|
+
"inside the 17:00-18:00 ET maintenance halt"
|
|
198
|
+
),
|
|
199
|
+
ts_ns=bar.ts_event,
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
session_day = trading_day_of(bar.ts_event)
|
|
203
|
+
if session_day.weekday() >= 5:
|
|
204
|
+
issues.append(
|
|
205
|
+
ValidationIssue(
|
|
206
|
+
code="weekend_bar",
|
|
207
|
+
message=(
|
|
208
|
+
f"bar {index}: ts_event {event_et:%Y-%m-%d %H:%M:%S %Z} falls "
|
|
209
|
+
f"in the weekend session gap (trading day {session_day})"
|
|
210
|
+
),
|
|
211
|
+
ts_ns=bar.ts_event,
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
return ValidationReport(issues=tuple(issues))
|