lobkit 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lobkit/__init__.py +18 -0
- lobkit/analysis.py +262 -0
- lobkit/book.py +113 -0
- lobkit/features.py +85 -0
- lobkit/market_making.py +168 -0
- lobkit/plotting.py +147 -0
- lobkit/recorder.py +222 -0
- lobkit/simulator.py +234 -0
- lobkit-0.2.1.dist-info/METADATA +289 -0
- lobkit-0.2.1.dist-info/RECORD +13 -0
- lobkit-0.2.1.dist-info/WHEEL +5 -0
- lobkit-0.2.1.dist-info/licenses/LICENSE +21 -0
- lobkit-0.2.1.dist-info/top_level.txt +1 -0
lobkit/plotting.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Diagnostic plots (requires matplotlib)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from lobkit.analysis import RegressionResult
|
|
9
|
+
|
|
10
|
+
C = ["#4C72B0", "#DD8452", "#55A868", "#C44E52"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _save(fig, ax, path, title):
|
|
14
|
+
ax.set_title(title, fontweight="bold", loc="left")
|
|
15
|
+
ax.spines[["top", "right"]].set_visible(False)
|
|
16
|
+
fig.tight_layout()
|
|
17
|
+
fig.savefig(path, dpi=150, bbox_inches="tight")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def plot_ofi_impact(buckets: pd.DataFrame, fit: RegressionResult, path: str, title: str, unit: str = "ticks"):
|
|
21
|
+
import matplotlib.pyplot as plt
|
|
22
|
+
|
|
23
|
+
fig, ax = plt.subplots(figsize=(8, 5.5))
|
|
24
|
+
ax.scatter(buckets["ofi"], buckets["d_mid"], s=8, alpha=0.35, color=C[0], edgecolor="none")
|
|
25
|
+
x = np.linspace(buckets["ofi"].min(), buckets["ofi"].max(), 50)
|
|
26
|
+
ax.plot(x, fit.alpha + fit.beta * x, color=C[3], lw=2,
|
|
27
|
+
label=f"slope {fit.beta:.3f} {unit}/unit OFI, R² = {fit.r2:.2f}")
|
|
28
|
+
ax.axhline(0, color="grey", lw=0.6), ax.axvline(0, color="grey", lw=0.6)
|
|
29
|
+
ax.set_xlabel("order-flow imbalance in bucket")
|
|
30
|
+
ax.set_ylabel(f"mid-price change ({unit})")
|
|
31
|
+
ax.legend(frameon=False)
|
|
32
|
+
_save(fig, ax, path, title)
|
|
33
|
+
plt.close(fig)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def plot_imbalance(table: pd.DataFrame, path: str, title: str, min_n: int = 200):
|
|
37
|
+
import matplotlib.pyplot as plt
|
|
38
|
+
|
|
39
|
+
t = table[table["n"] >= min_n]
|
|
40
|
+
fig, ax = plt.subplots(figsize=(8, 5))
|
|
41
|
+
ax.plot(t["imbalance_mid"], t["p_up"], "o-", color=C[0], lw=2)
|
|
42
|
+
ax.axhline(0.5, color="grey", ls="--", lw=1)
|
|
43
|
+
ax.set_xlabel("queue imbalance bid size / (bid size + ask size)")
|
|
44
|
+
ax.set_ylabel("P(next mid move is up)")
|
|
45
|
+
ax.set_ylim(0, 1)
|
|
46
|
+
_save(fig, ax, path, title)
|
|
47
|
+
plt.close(fig)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def plot_impact_vs_depth(table: pd.DataFrame, fit: RegressionResult, path: str, title: str):
|
|
51
|
+
import matplotlib.pyplot as plt
|
|
52
|
+
|
|
53
|
+
fig, ax = plt.subplots(figsize=(8, 5))
|
|
54
|
+
inv = 1 / table["mean_depth"]
|
|
55
|
+
ax.plot(inv, table["beta"], "o", ms=8, color=C[0])
|
|
56
|
+
x = np.linspace(0, inv.max() * 1.05, 20)
|
|
57
|
+
ax.plot(x, fit.alpha + fit.beta * x, "--", color=C[3], label=f"linear fit, R² = {fit.r2:.2f}")
|
|
58
|
+
ax.set_xlabel("1 / average depth at best quotes")
|
|
59
|
+
ax.set_ylabel("price impact coefficient β")
|
|
60
|
+
ax.legend(frameon=False)
|
|
61
|
+
_save(fig, ax, path, title)
|
|
62
|
+
plt.close(fig)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def plot_as(results: dict, path: str, title: str):
|
|
66
|
+
import matplotlib.pyplot as plt
|
|
67
|
+
|
|
68
|
+
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 4.8))
|
|
69
|
+
bins = np.linspace(min(r.pnl.min() for r in results.values()), max(r.pnl.max() for r in results.values()), 50)
|
|
70
|
+
for i, (name, r) in enumerate(results.items()):
|
|
71
|
+
label = f"{name}: mean {r.pnl.mean():.1f}, std {r.pnl.std(ddof=1):.1f}"
|
|
72
|
+
ax1.hist(r.pnl, bins=bins, alpha=0.6, color=C[i], label=label)
|
|
73
|
+
t = np.linspace(0, 1, r.inventory_paths.shape[1])
|
|
74
|
+
ax2.plot(t, r.inventory_paths[0], ["-", "--"][i], color=C[i], lw=1.4, label=name)
|
|
75
|
+
ax1.set_xlabel("terminal P&L"), ax1.set_ylabel("simulations"), ax1.legend(frameon=False)
|
|
76
|
+
ax2.set_xlabel("time / T"), ax2.set_ylabel("inventory (one sample path)"), ax2.legend(frameon=False)
|
|
77
|
+
ax2.spines[["top", "right"]].set_visible(False)
|
|
78
|
+
_save(fig, ax1, path, title)
|
|
79
|
+
plt.close(fig)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def plot_multi_level(table: pd.DataFrame, path: str, title: str):
|
|
83
|
+
import matplotlib.pyplot as plt
|
|
84
|
+
|
|
85
|
+
fig, ax = plt.subplots(figsize=(8, 5))
|
|
86
|
+
ax.plot(table["levels"], table["oos_r2"], "o-", color=C[0], lw=2, label="out-of-sample")
|
|
87
|
+
ax.plot(table["levels"], table["r2"], "s--", color=C[1], lw=1.2, label="in-sample")
|
|
88
|
+
ax.set_xlabel("OFI from the best m price levels")
|
|
89
|
+
ax.set_ylabel("R² of mid-price change")
|
|
90
|
+
ax.set_xticks(table["levels"])
|
|
91
|
+
ax.legend(frameon=False)
|
|
92
|
+
_save(fig, ax, path, title)
|
|
93
|
+
plt.close(fig)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def plot_microprice(adjustment: np.ndarray, path: str, title: str, unit: str = "ticks"):
|
|
97
|
+
import matplotlib.pyplot as plt
|
|
98
|
+
|
|
99
|
+
fig, ax = plt.subplots(figsize=(8, 5))
|
|
100
|
+
n_imb = adjustment.shape[1]
|
|
101
|
+
x = (np.arange(n_imb) + 0.5) / n_imb
|
|
102
|
+
for s, row in enumerate(adjustment):
|
|
103
|
+
ax.plot(x, row, "o-", color=C[s % len(C)], lw=2, label=f"spread {s + 1} tick{'s' if s else ''}")
|
|
104
|
+
ax.plot(x, (x - 0.5), ":", color="grey", label="naive microprice, spread 1 tick")
|
|
105
|
+
ax.axhline(0, color="grey", lw=0.6)
|
|
106
|
+
ax.set_xlabel("queue imbalance bid size / (bid size + ask size)")
|
|
107
|
+
ax.set_ylabel(f"micro-price minus mid ({unit})")
|
|
108
|
+
ax.legend(frameon=False)
|
|
109
|
+
_save(fig, ax, path, title)
|
|
110
|
+
plt.close(fig)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def plot_as_lob(table: pd.DataFrame, path: str, title: str):
|
|
114
|
+
"""Per-fill edge by strategy and risk aversion: spread captured, lost to adverse selection, kept."""
|
|
115
|
+
import matplotlib.pyplot as plt
|
|
116
|
+
|
|
117
|
+
fig, ax = plt.subplots(figsize=(9, 5))
|
|
118
|
+
labels = [f"γ={g}\n{s}" for g, s in zip(table["gamma"], table["strategy"], strict=True)]
|
|
119
|
+
x = np.arange(len(table))
|
|
120
|
+
ax.bar(x - 0.27, table["spread_capture"], 0.27, color=C[0], label="spread captured")
|
|
121
|
+
ax.bar(x, table["adverse_1s"], 0.27, color=C[1], label="adverse selection after 1 s")
|
|
122
|
+
ax.bar(x + 0.27, table["adverse_10s"], 0.27, color=C[3], label="adverse selection after 10 s")
|
|
123
|
+
ax.plot(x, table["spread_capture"] + table["adverse_10s"], "D", color="black", label="kept after 10 s")
|
|
124
|
+
ax.axhline(0, color="grey", lw=0.6)
|
|
125
|
+
ax.set_xticks(x, labels)
|
|
126
|
+
ax.set_ylabel("ticks per fill")
|
|
127
|
+
ax.legend(frameon=False)
|
|
128
|
+
_save(fig, ax, path, title)
|
|
129
|
+
plt.close(fig)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def plot_lead_lag(tables: dict, path: str, title: str):
|
|
133
|
+
"""tables: label -> lead_lag() output. A peak right of zero means the first venue leads."""
|
|
134
|
+
import matplotlib.pyplot as plt
|
|
135
|
+
|
|
136
|
+
fig, ax = plt.subplots(figsize=(8, 5))
|
|
137
|
+
for i, (name, t) in enumerate(tables.items()):
|
|
138
|
+
ax.plot(t["lag_s"] * 1000, t["corr"], "-", color=C[i % len(C)], lw=2, label=name)
|
|
139
|
+
peak = t.loc[t["corr"].idxmax()]
|
|
140
|
+
ax.plot(peak["lag_s"] * 1000, peak["corr"], "o", color=C[i % len(C)])
|
|
141
|
+
ax.axvline(0, color="grey", lw=0.6)
|
|
142
|
+
ax.axhline(0, color="grey", lw=0.6)
|
|
143
|
+
ax.set_xlabel("lag (ms): correlation of venue A's return now with venue B's return this much later")
|
|
144
|
+
ax.set_ylabel("correlation of mid returns")
|
|
145
|
+
ax.legend(frameon=False)
|
|
146
|
+
_save(fig, ax, path, title)
|
|
147
|
+
plt.close(fig)
|
lobkit/recorder.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Record live spot market data from Binance and Bybit (requires `pip install -e '.[live]'`).
|
|
2
|
+
|
|
3
|
+
Binance, three streams on one websocket:
|
|
4
|
+
<sym>@bookTicker every best bid/ask change -> l1.csv
|
|
5
|
+
<sym>@aggTrade trades with aggressor side -> trades.csv
|
|
6
|
+
<sym>@depth@100ms L2 diffs, synchronised with a -> depth.csv (top N levels, sampled;
|
|
7
|
+
REST snapshot via load with load_depth)
|
|
8
|
+
BinanceDepthSync
|
|
9
|
+
Bybit (optional, same session and clock, for cross-venue studies):
|
|
10
|
+
orderbook.1.<SYM> best bid/ask -> bybit_l1.csv
|
|
11
|
+
publicTrade.<SYM> trades with aggressor side -> bybit_trades.csv
|
|
12
|
+
|
|
13
|
+
Timestamps: `ts` is the local receive time for quotes on both venues, so the
|
|
14
|
+
two venues share one clock; trades carry the exchange time in `ts` and the
|
|
15
|
+
local time in `ts_local`. A dropped connection is reopened automatically (and
|
|
16
|
+
the Binance depth book resynchronised); the count is printed at the end.
|
|
17
|
+
Binance may be unavailable in some jurisdictions; adjust WS_BASE/REST_BASE
|
|
18
|
+
(e.g. to Binance.US endpoints) if needed.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import asyncio
|
|
24
|
+
import csv
|
|
25
|
+
import json
|
|
26
|
+
import time
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
import pandas as pd
|
|
30
|
+
|
|
31
|
+
from lobkit.book import BinanceDepthSync, SequenceGap
|
|
32
|
+
|
|
33
|
+
WS_BASE = "wss://stream.binance.com:9443"
|
|
34
|
+
REST_BASE = "https://api.binance.com"
|
|
35
|
+
BYBIT_WS = "wss://stream.bybit.com/v5/public/spot"
|
|
36
|
+
|
|
37
|
+
L1_HEADER = ["ts", "update_id", "bid_px", "bid_sz", "ask_px", "ask_sz"]
|
|
38
|
+
TRADES_HEADER = ["ts", "ts_local", "price", "qty", "side"]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _fetch_snapshot(symbol: str, limit: int = 1000) -> dict:
|
|
42
|
+
import requests
|
|
43
|
+
|
|
44
|
+
r = requests.get(f"{REST_BASE}/api/v3/depth", params={"symbol": symbol.upper(), "limit": limit}, timeout=10)
|
|
45
|
+
r.raise_for_status()
|
|
46
|
+
return r.json()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _writer(out: Path, name: str, header: list[str], files: list):
|
|
50
|
+
f = open(out / f"{name}.csv", "w", newline="") # closed by record_binance
|
|
51
|
+
files.append(f)
|
|
52
|
+
w = csv.writer(f)
|
|
53
|
+
w.writerow(header)
|
|
54
|
+
return w
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
async def record_binance(symbol: str = "btcusdt", seconds: int = 900, out_dir: str | Path = "data",
|
|
58
|
+
depth_levels: int = 10, depth_every: float = 1.0, bybit: bool = False) -> Path:
|
|
59
|
+
"""Record `seconds` of data into out_dir/<symbol>_<timestamp>/; bybit=True adds the Bybit streams."""
|
|
60
|
+
sym = symbol.lower()
|
|
61
|
+
out = Path(out_dir) / f"{sym}_{time.strftime('%Y%m%d_%H%M%S')}"
|
|
62
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
t_end = time.time() + seconds
|
|
64
|
+
files: list = []
|
|
65
|
+
try:
|
|
66
|
+
jobs = [_binance(sym, t_end, out, depth_levels, depth_every, files)]
|
|
67
|
+
if bybit:
|
|
68
|
+
jobs.append(_bybit(sym.upper(), t_end, out, files))
|
|
69
|
+
for line in await asyncio.gather(*jobs):
|
|
70
|
+
print(line)
|
|
71
|
+
finally:
|
|
72
|
+
for f in files:
|
|
73
|
+
f.close()
|
|
74
|
+
print(f"recorded to {out}")
|
|
75
|
+
return out
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
async def _recv(ws) -> tuple[dict, float]:
|
|
79
|
+
"""Next message and its arrival time. The clock is read after recv() returns, never before:
|
|
80
|
+
a time taken before waiting would stamp each message with the previous one's arrival."""
|
|
81
|
+
raw = await asyncio.wait_for(ws.recv(), timeout=30)
|
|
82
|
+
return json.loads(raw), time.time()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
async def _reconnecting(name: str, url: str, t_end: float, session) -> int:
|
|
86
|
+
"""Run session(ws) until t_end, reopening the connection if it drops. Returns the reconnect count."""
|
|
87
|
+
import websockets
|
|
88
|
+
|
|
89
|
+
reconnects = -1
|
|
90
|
+
while time.time() < t_end:
|
|
91
|
+
reconnects += 1
|
|
92
|
+
try:
|
|
93
|
+
async with websockets.connect(url, max_size=None, ping_interval=20) as ws:
|
|
94
|
+
await session(ws)
|
|
95
|
+
except (websockets.ConnectionClosed, asyncio.TimeoutError, OSError) as e:
|
|
96
|
+
print(f"{name}: connection lost ({e!r}); reconnecting")
|
|
97
|
+
await asyncio.sleep(1)
|
|
98
|
+
return max(reconnects, 0)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
async def _binance(sym: str, t_end: float, out: Path, depth_levels: int, depth_every: float, files: list) -> str:
|
|
102
|
+
l1 = _writer(out, "l1", L1_HEADER, files)
|
|
103
|
+
trades = _writer(out, "trades", TRADES_HEADER, files)
|
|
104
|
+
depth = _writer(out, "depth", ["ts", "side", "level", "price", "qty"], files)
|
|
105
|
+
resyncs = 0
|
|
106
|
+
|
|
107
|
+
async def session(ws):
|
|
108
|
+
nonlocal resyncs
|
|
109
|
+
sync, snapshot_task, last_depth, retry_at = BinanceDepthSync(), None, 0.0, 0.0 # fresh book per connection
|
|
110
|
+
while time.time() < t_end:
|
|
111
|
+
msg, now = await _recv(ws)
|
|
112
|
+
stream, d = msg["stream"], msg["data"]
|
|
113
|
+
if not stream.endswith("@depth@100ms"):
|
|
114
|
+
name, row = _binance_row(stream, d, now)
|
|
115
|
+
(l1 if name == "l1" else trades).writerow(row)
|
|
116
|
+
else:
|
|
117
|
+
try:
|
|
118
|
+
sync.on_event(d)
|
|
119
|
+
if snapshot_task is not None and snapshot_task.done():
|
|
120
|
+
task, snapshot_task = snapshot_task, None
|
|
121
|
+
if task.exception() is not None:
|
|
122
|
+
print(f"snapshot fetch failed: {task.exception()!r}; retrying in 5 s")
|
|
123
|
+
retry_at = now + 5 # don't hammer the REST API (rate limits, IP bans)
|
|
124
|
+
else:
|
|
125
|
+
sync.on_snapshot(task.result()) # False = too old; refetched below
|
|
126
|
+
except SequenceGap: # from the stream, or replaying the buffer
|
|
127
|
+
resyncs += 1
|
|
128
|
+
if snapshot_task is not None:
|
|
129
|
+
snapshot_task.cancel() # its snapshot belongs to the abandoned book
|
|
130
|
+
sync, snapshot_task = BinanceDepthSync(), None
|
|
131
|
+
sync.on_event(d)
|
|
132
|
+
if not sync.synced and snapshot_task is None and now >= retry_at:
|
|
133
|
+
snapshot_task = asyncio.create_task(asyncio.to_thread(_fetch_snapshot, sym))
|
|
134
|
+
if sync.synced and now - last_depth >= depth_every:
|
|
135
|
+
last_depth = now
|
|
136
|
+
for side in ("bid", "ask"):
|
|
137
|
+
for lvl, (p, q) in enumerate(sync.book.levels(side, depth_levels)):
|
|
138
|
+
depth.writerow([now, side, lvl, p, q])
|
|
139
|
+
|
|
140
|
+
url = f"{WS_BASE}/stream?streams={sym}@bookTicker/{sym}@aggTrade/{sym}@depth@100ms"
|
|
141
|
+
reconnects = await _reconnecting("binance", url, t_end, session)
|
|
142
|
+
return f"binance: {resyncs} depth resyncs, {reconnects} reconnects"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
async def _bybit(sym: str, t_end: float, out: Path, files: list) -> str:
|
|
146
|
+
l1 = _writer(out, "bybit_l1", L1_HEADER, files)
|
|
147
|
+
trades = _writer(out, "bybit_trades", TRADES_HEADER, files)
|
|
148
|
+
|
|
149
|
+
async def session(ws):
|
|
150
|
+
await ws.send(json.dumps({"op": "subscribe", "args": [f"orderbook.1.{sym}", f"publicTrade.{sym}"]}))
|
|
151
|
+
book = {"bid": None, "ask": None} # fresh best quotes per connection
|
|
152
|
+
last_ping = time.time()
|
|
153
|
+
while time.time() < t_end:
|
|
154
|
+
if time.time() - last_ping > 20: # Bybit drops connections without app-level pings
|
|
155
|
+
await ws.send(json.dumps({"op": "ping"}))
|
|
156
|
+
last_ping = time.time()
|
|
157
|
+
msg, now = await _recv(ws)
|
|
158
|
+
for name, row in _bybit_rows(msg, book, now):
|
|
159
|
+
(l1 if name == "l1" else trades).writerow(row)
|
|
160
|
+
|
|
161
|
+
reconnects = await _reconnecting("bybit", BYBIT_WS, t_end, session)
|
|
162
|
+
return f"bybit: {reconnects} reconnects"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _binance_row(stream: str, d: dict, now: float) -> tuple[str, list]:
|
|
166
|
+
"""("l1" | "trades", csv row) for a bookTicker or aggTrade message."""
|
|
167
|
+
if stream.endswith("@bookTicker"):
|
|
168
|
+
return "l1", [now, d["u"], d["b"], d["B"], d["a"], d["A"]]
|
|
169
|
+
return "trades", [d["T"] / 1000, now, d["p"], d["q"], -1 if d["m"] else 1] # m: buyer is maker
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _bybit_rows(msg: dict, book: dict, now: float) -> list[tuple[str, list]]:
|
|
173
|
+
"""CSV rows for one Bybit message; book holds the current best bid/ask and is updated.
|
|
174
|
+
|
|
175
|
+
A snapshot replaces both sides (an empty side means no quote); a delta only
|
|
176
|
+
replaces the sides it carries. No L1 row is written while a side is missing.
|
|
177
|
+
"""
|
|
178
|
+
topic = msg.get("topic", "")
|
|
179
|
+
if topic.startswith("orderbook"):
|
|
180
|
+
d = msg["data"]
|
|
181
|
+
for side, key in (("bid", "b"), ("ask", "a")):
|
|
182
|
+
if d[key] or msg.get("type") == "snapshot":
|
|
183
|
+
top = d[key][0] if d[key] else None
|
|
184
|
+
book[side] = top if top and float(top[1]) > 0 else None # size 0 = level deleted
|
|
185
|
+
bid, ask = book["bid"], book["ask"]
|
|
186
|
+
return [("l1", [now, d["u"], bid[0], bid[1], ask[0], ask[1]])] if bid and ask else []
|
|
187
|
+
if topic.startswith("publicTrade"):
|
|
188
|
+
return [("trades", [tr["T"] / 1000, now, tr["p"], tr["v"], 1 if tr["S"] == "Buy" else -1])
|
|
189
|
+
for tr in msg["data"]] # S: taker side
|
|
190
|
+
if msg.get("success") is False:
|
|
191
|
+
raise RuntimeError(f"bybit subscription failed: {msg}")
|
|
192
|
+
return [] # pong, subscription ack
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def load_recording(path: str | Path, venue: str = "binance") -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
196
|
+
"""Return (quotes, trades) in lobkit's standard format for venue "binance" or "bybit".
|
|
197
|
+
|
|
198
|
+
Trades are stamped with their local receive time, the clock the quotes use:
|
|
199
|
+
the local clock can be off the exchange's by a second or more, which would
|
|
200
|
+
misalign trade and quote buckets.
|
|
201
|
+
"""
|
|
202
|
+
prefix = {"binance": "", "bybit": "bybit_"}[venue]
|
|
203
|
+
path = Path(path)
|
|
204
|
+
quotes = pd.read_csv(path / f"{prefix}l1.csv").drop(columns="update_id")
|
|
205
|
+
trades = pd.read_csv(path / f"{prefix}trades.csv")
|
|
206
|
+
trades = trades.assign(ts=trades["ts_local"]).drop(columns="ts_local")
|
|
207
|
+
changed = quotes[["bid_px", "bid_sz", "ask_px", "ask_sz"]].diff().abs().sum(axis=1) > 0
|
|
208
|
+
changed.iloc[0] = True
|
|
209
|
+
return quotes[changed].reset_index(drop=True), trades
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def load_depth(path: str | Path) -> pd.DataFrame:
|
|
213
|
+
"""Return the sampled L2 snapshots as a quotes frame with levels bid_px, ..., bid_px_2, ..."""
|
|
214
|
+
d = pd.read_csv(Path(path) / "depth.csv")
|
|
215
|
+
w = d.pivot(index="ts", columns=["side", "level"], values=["price", "qty"]).dropna()
|
|
216
|
+
out = pd.DataFrame({"ts": w.index.to_numpy()})
|
|
217
|
+
for lvl in sorted(d["level"].unique()):
|
|
218
|
+
sfx = "" if lvl == 0 else f"_{lvl + 1}"
|
|
219
|
+
for side in ("bid", "ask"):
|
|
220
|
+
out[f"{side}_px{sfx}"] = w[("price", side, lvl)].to_numpy()
|
|
221
|
+
out[f"{side}_sz{sfx}"] = w[("qty", side, lvl)].to_numpy()
|
|
222
|
+
return out
|
lobkit/simulator.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Zero-intelligence limit order book (Cont, Stoikov & Talreja 2010).
|
|
2
|
+
|
|
3
|
+
Unit-size orders on a tick grid:
|
|
4
|
+
* limit buy (sell) orders arrive i ticks below the best ask (above the best
|
|
5
|
+
bid) at rate lambda(i), i = 1..K;
|
|
6
|
+
* market buy and sell orders arrive at rate mu each;
|
|
7
|
+
* each order resting i ticks from the opposite best is cancelled at rate
|
|
8
|
+
theta(i), so a queue of size Q is cancelled at rate theta(i) * Q.
|
|
9
|
+
|
|
10
|
+
No agent has information, yet the model reproduces key empirical facts:
|
|
11
|
+
order-flow imbalance moves prices linearly, and queue imbalance predicts the
|
|
12
|
+
next mid-price move. That makes it a ground truth for testing analytics code.
|
|
13
|
+
|
|
14
|
+
An optional quoter (e.g. a market maker) can rest one unit on each side. Its
|
|
15
|
+
orders join the back of the queue, move up as orders ahead of them trade or
|
|
16
|
+
cancel, and fill only when a market order reaches them.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import math
|
|
22
|
+
import random
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
import pandas as pd
|
|
28
|
+
|
|
29
|
+
CST_LAMBDA = (1.85, 1.51, 1.09, 0.88, 0.77) # orders/second at distance 1..5 ticks
|
|
30
|
+
CST_THETA = (0.71, 0.81, 0.68, 0.56, 0.47) # cancellation rate per resting order
|
|
31
|
+
CST_MU = 0.94 # market orders/second per side
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class SimulationResult:
|
|
36
|
+
quotes: pd.DataFrame # ts, bid_px, bid_sz, ask_px, ask_sz (+ bid_px_2, ... if n_levels > 1) after every event
|
|
37
|
+
trades: pd.DataFrame # ts, price, qty, side (+1 buyer-initiated)
|
|
38
|
+
fills: pd.DataFrame | None = None # quoter's fills: ts, side (+1 quoter bought), price, mid before the fill
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# quoter(t, mid, inventory) -> (bid, ask) in price units, or None to stay out of the book
|
|
42
|
+
Quoter = Callable[[float, float, int], "tuple[float, float] | None"]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def simulate_lob(n_events: int = 300_000, lam=CST_LAMBDA, theta=CST_THETA, mu: float = CST_MU,
|
|
46
|
+
initial_depth: int = 3, n_ticks: int = 20_000, tick: float = 1.0,
|
|
47
|
+
seed: int | None = 0, n_levels: int = 1, quoter: Quoter | None = None,
|
|
48
|
+
requote_every: float = 1.0, t_end: float | None = None) -> SimulationResult:
|
|
49
|
+
"""Simulate up to n_events book events, stopping early at time t_end if given.
|
|
50
|
+
|
|
51
|
+
n_levels > 1 also records the 2nd..n-th best non-empty levels. A quoter is
|
|
52
|
+
asked for new quotes every requote_every seconds; its old orders are
|
|
53
|
+
cancelled first, and a quote that would cross the book is placed one tick
|
|
54
|
+
inside it instead (post-only). Without a quoter the random stream, and so
|
|
55
|
+
every result, is the same as before quoters existed.
|
|
56
|
+
"""
|
|
57
|
+
rng = random.Random(seed)
|
|
58
|
+
K = len(lam)
|
|
59
|
+
bid = [0] * n_ticks
|
|
60
|
+
ask = [0] * n_ticks
|
|
61
|
+
centre = n_ticks // 2
|
|
62
|
+
for p in range(centre):
|
|
63
|
+
bid[p] = initial_depth
|
|
64
|
+
for p in range(centre + 1, n_ticks):
|
|
65
|
+
ask[p] = initial_depth
|
|
66
|
+
bb, ba = centre - 1, centre + 1
|
|
67
|
+
lam_total = sum(lam)
|
|
68
|
+
|
|
69
|
+
ts = np.empty(n_events)
|
|
70
|
+
rec = np.empty((n_events, 4 * n_levels), dtype=np.int64)
|
|
71
|
+
trades, fills = [], []
|
|
72
|
+
t = 0.0
|
|
73
|
+
my_b = my_a = -1 # quoter's bid / ask level, -1 = none
|
|
74
|
+
ahead_b = ahead_a = inv = 0 # orders queued ahead of it, its inventory
|
|
75
|
+
next_quote = 0.0
|
|
76
|
+
|
|
77
|
+
for n in range(n_events):
|
|
78
|
+
if bb < K + 1 or ba > n_ticks - K - 2:
|
|
79
|
+
raise RuntimeError("price walked off the grid; increase n_ticks")
|
|
80
|
+
c_bid = [theta[i] * bid[ba - 1 - i] for i in range(K)]
|
|
81
|
+
c_ask = [theta[i] * ask[bb + 1 + i] for i in range(K)]
|
|
82
|
+
if 0 <= ba - 1 - my_b < K: # the quoter's own orders don't cancel randomly
|
|
83
|
+
c_bid[ba - 1 - my_b] -= theta[ba - 1 - my_b]
|
|
84
|
+
if 0 <= my_a - bb - 1 < K:
|
|
85
|
+
c_ask[my_a - bb - 1] -= theta[my_a - bb - 1]
|
|
86
|
+
sc_bid, sc_ask = sum(c_bid), sum(c_ask)
|
|
87
|
+
total = 2 * lam_total + 2 * mu + sc_bid + sc_ask
|
|
88
|
+
t += rng.expovariate(total)
|
|
89
|
+
if t_end is not None and t >= t_end:
|
|
90
|
+
break
|
|
91
|
+
u = rng.random() * total
|
|
92
|
+
|
|
93
|
+
if u < lam_total: # limit buy
|
|
94
|
+
for i in range(K):
|
|
95
|
+
u -= lam[i]
|
|
96
|
+
if u < 0:
|
|
97
|
+
p = ba - 1 - i
|
|
98
|
+
bid[p] += 1
|
|
99
|
+
bb = max(bb, p)
|
|
100
|
+
break
|
|
101
|
+
else:
|
|
102
|
+
bid[ba - K] += 1
|
|
103
|
+
elif (u := u - lam_total) < lam_total: # limit sell
|
|
104
|
+
for i in range(K):
|
|
105
|
+
u -= lam[i]
|
|
106
|
+
if u < 0:
|
|
107
|
+
p = bb + 1 + i
|
|
108
|
+
ask[p] += 1
|
|
109
|
+
ba = min(ba, p)
|
|
110
|
+
break
|
|
111
|
+
else:
|
|
112
|
+
ask[bb + K] += 1
|
|
113
|
+
elif (u := u - lam_total) < mu: # market buy
|
|
114
|
+
trades.append((t, ba * tick, 1.0, 1))
|
|
115
|
+
if ba == my_a:
|
|
116
|
+
if ahead_a == 0:
|
|
117
|
+
fills.append((t, -1, ba * tick, (bb + ba) * tick / 2))
|
|
118
|
+
inv, my_a = inv - 1, -1
|
|
119
|
+
else:
|
|
120
|
+
ahead_a -= 1
|
|
121
|
+
ask[ba] -= 1
|
|
122
|
+
while ask[ba] == 0:
|
|
123
|
+
ba += 1
|
|
124
|
+
elif (u := u - mu) < mu: # market sell
|
|
125
|
+
trades.append((t, bb * tick, 1.0, -1))
|
|
126
|
+
if bb == my_b:
|
|
127
|
+
if ahead_b == 0:
|
|
128
|
+
fills.append((t, 1, bb * tick, (bb + ba) * tick / 2))
|
|
129
|
+
inv, my_b = inv + 1, -1
|
|
130
|
+
else:
|
|
131
|
+
ahead_b -= 1
|
|
132
|
+
bid[bb] -= 1
|
|
133
|
+
while bid[bb] == 0:
|
|
134
|
+
bb -= 1
|
|
135
|
+
elif (u := u - mu) < sc_bid: # cancel a bid
|
|
136
|
+
for i in range(K):
|
|
137
|
+
u -= c_bid[i]
|
|
138
|
+
if u < 0:
|
|
139
|
+
break
|
|
140
|
+
else: # float rounding: never cancel an empty queue
|
|
141
|
+
i = max(j for j in range(K) if c_bid[j] > 0)
|
|
142
|
+
p = ba - 1 - i
|
|
143
|
+
if p == my_b and rng.random() * (bid[p] - 1) < ahead_b: # the cancelled order was ahead of ours
|
|
144
|
+
ahead_b -= 1
|
|
145
|
+
bid[p] -= 1
|
|
146
|
+
while bid[bb] == 0:
|
|
147
|
+
bb -= 1
|
|
148
|
+
else: # cancel an ask
|
|
149
|
+
u -= sc_bid
|
|
150
|
+
for i in range(K):
|
|
151
|
+
u -= c_ask[i]
|
|
152
|
+
if u < 0:
|
|
153
|
+
break
|
|
154
|
+
else:
|
|
155
|
+
i = max(j for j in range(K) if c_ask[j] > 0)
|
|
156
|
+
p = bb + 1 + i
|
|
157
|
+
if p == my_a and rng.random() * (ask[p] - 1) < ahead_a:
|
|
158
|
+
ahead_a -= 1
|
|
159
|
+
ask[p] -= 1
|
|
160
|
+
while ask[ba] == 0:
|
|
161
|
+
ba += 1
|
|
162
|
+
|
|
163
|
+
if quoter is not None and t >= next_quote:
|
|
164
|
+
next_quote = t + requote_every
|
|
165
|
+
xb, xa = bb, ba # best quotes of everyone else
|
|
166
|
+
if xb == my_b and bid[xb] == 1:
|
|
167
|
+
xb -= 1
|
|
168
|
+
while bid[xb] == 0:
|
|
169
|
+
xb -= 1
|
|
170
|
+
if xa == my_a and ask[xa] == 1:
|
|
171
|
+
xa += 1
|
|
172
|
+
while ask[xa] == 0:
|
|
173
|
+
xa += 1
|
|
174
|
+
q = quoter(t, (xb + xa) * tick / 2, inv)
|
|
175
|
+
b = a = -1
|
|
176
|
+
if q is not None:
|
|
177
|
+
b = min(math.floor(q[0] / tick + 1e-9), xa - 1)
|
|
178
|
+
a = max(math.ceil(q[1] / tick - 1e-9), xb + 1)
|
|
179
|
+
if b >= a:
|
|
180
|
+
b = a = -1
|
|
181
|
+
if b != my_b: # an unchanged level keeps its queue priority
|
|
182
|
+
if my_b >= 0:
|
|
183
|
+
bid[my_b] -= 1
|
|
184
|
+
while bid[bb] == 0:
|
|
185
|
+
bb -= 1
|
|
186
|
+
if b >= 0:
|
|
187
|
+
ahead_b = bid[b]
|
|
188
|
+
bid[b] += 1
|
|
189
|
+
bb = max(bb, b)
|
|
190
|
+
my_b = b
|
|
191
|
+
if a != my_a:
|
|
192
|
+
if my_a >= 0:
|
|
193
|
+
ask[my_a] -= 1
|
|
194
|
+
while ask[ba] == 0:
|
|
195
|
+
ba += 1
|
|
196
|
+
if a >= 0:
|
|
197
|
+
ahead_a = ask[a]
|
|
198
|
+
ask[a] += 1
|
|
199
|
+
ba = min(ba, a)
|
|
200
|
+
my_a = a
|
|
201
|
+
|
|
202
|
+
ts[n] = t
|
|
203
|
+
if n_levels == 1:
|
|
204
|
+
rec[n] = (bb, bid[bb], ba, ask[ba])
|
|
205
|
+
else:
|
|
206
|
+
rec[n] = _top_levels(bid, ask, bb, ba, n_levels)
|
|
207
|
+
|
|
208
|
+
else:
|
|
209
|
+
n = n_events # no early stop at t_end
|
|
210
|
+
ts, rec = ts[:n], rec[:n]
|
|
211
|
+
quotes = pd.DataFrame({"ts": ts})
|
|
212
|
+
for m in range(n_levels):
|
|
213
|
+
sfx = "" if m == 0 else f"_{m + 1}"
|
|
214
|
+
quotes[f"bid_px{sfx}"] = rec[:, 4 * m] * tick
|
|
215
|
+
quotes[f"bid_sz{sfx}"] = rec[:, 4 * m + 1].astype(float)
|
|
216
|
+
quotes[f"ask_px{sfx}"] = rec[:, 4 * m + 2] * tick
|
|
217
|
+
quotes[f"ask_sz{sfx}"] = rec[:, 4 * m + 3].astype(float)
|
|
218
|
+
trade_df = pd.DataFrame(trades, columns=["ts", "price", "qty", "side"])
|
|
219
|
+
fill_df = pd.DataFrame(fills, columns=["ts", "side", "price", "mid"]) if quoter is not None else None
|
|
220
|
+
return SimulationResult(quotes, trade_df, fill_df)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _top_levels(bid: list[int], ask: list[int], bb: int, ba: int, n: int) -> list[int]:
|
|
224
|
+
"""(bid_px, bid_sz, ask_px, ask_sz) for the n best non-empty levels, flattened."""
|
|
225
|
+
out = []
|
|
226
|
+
for _ in range(n):
|
|
227
|
+
out += [bb, bid[bb], ba, ask[ba]]
|
|
228
|
+
bb -= 1
|
|
229
|
+
while bid[bb] == 0:
|
|
230
|
+
bb -= 1
|
|
231
|
+
ba += 1
|
|
232
|
+
while ask[ba] == 0:
|
|
233
|
+
ba += 1
|
|
234
|
+
return out
|