alphaboard 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- alphaboard/__init__.py +30 -0
- alphaboard/_backfill.py +481 -0
- alphaboard/_cli.py +184 -0
- alphaboard/_config.py +239 -0
- alphaboard/_doctor.py +481 -0
- alphaboard/_exporter.py +455 -0
- alphaboard/_spool.py +245 -0
- alphaboard/client.py +429 -0
- alphaboard/py.typed +0 -0
- alphaboard-0.1.0.dist-info/METADATA +190 -0
- alphaboard-0.1.0.dist-info/RECORD +14 -0
- alphaboard-0.1.0.dist-info/WHEEL +4 -0
- alphaboard-0.1.0.dist-info/entry_points.txt +2 -0
- alphaboard-0.1.0.dist-info/licenses/LICENSE +202 -0
alphaboard/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""alphaboard — Python SDK for AlphaBoard.
|
|
2
|
+
|
|
3
|
+
Quick start:
|
|
4
|
+
|
|
5
|
+
from alphaboard import AlphaBoard
|
|
6
|
+
|
|
7
|
+
tracker = AlphaBoard(
|
|
8
|
+
strategy="momentum_eth",
|
|
9
|
+
broker="hyperliquid",
|
|
10
|
+
account="main",
|
|
11
|
+
# api_url and api_key fall back to ALPHABOARD_API_URL / ALPHABOARD_API_KEY
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
tracker.log_trade(symbol="ETH-PERP", side="buy", quantity=0.5, price=3800)
|
|
15
|
+
tracker.heartbeat(open_positions=1)
|
|
16
|
+
|
|
17
|
+
The SDK is observation-path only. `log_trade()` enqueues and returns in
|
|
18
|
+
microseconds — it never blocks the trading loop and never raises. A daemon
|
|
19
|
+
background thread batches events, posts them with exponential backoff, and
|
|
20
|
+
suppresses all exceptions.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from alphaboard._backfill import BackfillError, BackfillResult
|
|
24
|
+
|
|
25
|
+
# `StratOps` is the deprecated pre-rename alias (OPM-762), re-exported only so
|
|
26
|
+
# the burn-in VPS keeps importing until it picks up the rename.
|
|
27
|
+
from alphaboard.client import AlphaBoard, StratOps
|
|
28
|
+
|
|
29
|
+
__version__ = "0.1.0"
|
|
30
|
+
__all__ = ["AlphaBoard", "BackfillError", "BackfillResult", "StratOps", "__version__"]
|
alphaboard/_backfill.py
ADDED
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
"""Backfill engine (OPM-729) — re-runnable historical ingestion.
|
|
2
|
+
|
|
3
|
+
Reads a CSV export (a Zorro log, a broker statement, an engine's own journal)
|
|
4
|
+
and feeds rows through the server's `/batch` endpoints with DETERMINISTIC
|
|
5
|
+
idempotency keys — UUIDv5 of the row's business key — so re-running the same
|
|
6
|
+
file or window is a no-op (`deduped`), never a duplicate.
|
|
7
|
+
|
|
8
|
+
That no-op used to be unconditional, which made the tool useless for the one
|
|
9
|
+
job an operator actually needs it for: fixing rows that were ingested wrong.
|
|
10
|
+
A re-run carrying corrected numbers reported `deduped` — the same word as a
|
|
11
|
+
re-run that changed nothing — so a fix that never landed was indistinguishable
|
|
12
|
+
from a fix that was not needed. Since OPM-1309 a collision whose payload
|
|
13
|
+
disagrees with the stored row is counted as `conflicts`, and `update_existing`
|
|
14
|
+
moves the stored row and counts it as `updated`. The default is unchanged:
|
|
15
|
+
without the flag, history is never rewritten.
|
|
16
|
+
|
|
17
|
+
Business keys (per spec §3.6 / OPM-729):
|
|
18
|
+
trades -> strategy + external_trade_id when present, else
|
|
19
|
+
strategy + symbol + executed_at + quantity + price + side
|
|
20
|
+
equity -> strategy + snapshot_at
|
|
21
|
+
positions -> strategy + account + symbol + snapshot_at
|
|
22
|
+
|
|
23
|
+
Unlike the live path this is a FOREGROUND operator tool: it runs
|
|
24
|
+
synchronously, reports per-row outcomes, and raises on persistent transport
|
|
25
|
+
failure (a re-run is cheap precisely because the keys are deterministic).
|
|
26
|
+
|
|
27
|
+
Deliberately NO `Idempotency-Key` header on backfill batches: the record keys
|
|
28
|
+
are deterministic, so the middleware's 24h response cache would replay the
|
|
29
|
+
first run's "created" counts on a re-run and mask the real record-level
|
|
30
|
+
"deduped" report — the signal that tells the operator the re-run was a no-op.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import csv
|
|
36
|
+
import json
|
|
37
|
+
import logging
|
|
38
|
+
import time
|
|
39
|
+
import uuid
|
|
40
|
+
from collections.abc import Callable, Mapping
|
|
41
|
+
from dataclasses import dataclass, field
|
|
42
|
+
from datetime import datetime, timezone
|
|
43
|
+
from pathlib import Path
|
|
44
|
+
from typing import Any
|
|
45
|
+
|
|
46
|
+
import httpx
|
|
47
|
+
|
|
48
|
+
from alphaboard._config import Config
|
|
49
|
+
|
|
50
|
+
logger = logging.getLogger("alphaboard")
|
|
51
|
+
|
|
52
|
+
# UUIDv5 namespace for deterministic backfill keys. Fixed forever — changing
|
|
53
|
+
# it would re-key history and duplicate every previously backfilled row.
|
|
54
|
+
BACKFILL_NS = uuid.uuid5(uuid.NAMESPACE_URL, "https://alphaboard.dev/sdk/backfill")
|
|
55
|
+
|
|
56
|
+
# Server-side cap on /batch items; our chunks must stay at or under it.
|
|
57
|
+
SERVER_BATCH_CAP = 1000
|
|
58
|
+
|
|
59
|
+
# Canonical fields per kind — anything else in the CSV is dropped (the batch
|
|
60
|
+
# schemas are extra="forbid"; one stray column must not 422 a whole batch).
|
|
61
|
+
_FIELDS: dict[str, frozenset[str]] = {
|
|
62
|
+
"trades": frozenset(
|
|
63
|
+
{
|
|
64
|
+
"strategy",
|
|
65
|
+
"broker",
|
|
66
|
+
"account",
|
|
67
|
+
"symbol",
|
|
68
|
+
"side",
|
|
69
|
+
"quantity",
|
|
70
|
+
"price",
|
|
71
|
+
"executed_at",
|
|
72
|
+
"idempotency_key",
|
|
73
|
+
"external_trade_id",
|
|
74
|
+
"external_order_id",
|
|
75
|
+
"external_exec_id",
|
|
76
|
+
"engine_trade_id",
|
|
77
|
+
"notional_value",
|
|
78
|
+
"currency",
|
|
79
|
+
"exec_type",
|
|
80
|
+
"order_status",
|
|
81
|
+
"cum_qty",
|
|
82
|
+
"leaves_qty",
|
|
83
|
+
"commission",
|
|
84
|
+
"fees",
|
|
85
|
+
"slippage",
|
|
86
|
+
"order_type",
|
|
87
|
+
"time_in_force",
|
|
88
|
+
"is_close",
|
|
89
|
+
}
|
|
90
|
+
),
|
|
91
|
+
"equity": frozenset(
|
|
92
|
+
{
|
|
93
|
+
"strategy",
|
|
94
|
+
"equity",
|
|
95
|
+
"snapshot_at",
|
|
96
|
+
"cash",
|
|
97
|
+
"positions_value",
|
|
98
|
+
"unrealized_pnl",
|
|
99
|
+
"realized_pnl_day",
|
|
100
|
+
"currency",
|
|
101
|
+
"equity_usd",
|
|
102
|
+
"allocated_capital",
|
|
103
|
+
"utilization_pct",
|
|
104
|
+
"idempotency_key",
|
|
105
|
+
}
|
|
106
|
+
),
|
|
107
|
+
"positions": frozenset(
|
|
108
|
+
{
|
|
109
|
+
"strategy",
|
|
110
|
+
"broker",
|
|
111
|
+
"account",
|
|
112
|
+
"symbol",
|
|
113
|
+
"quantity",
|
|
114
|
+
"snapshot_at",
|
|
115
|
+
"avg_entry_price",
|
|
116
|
+
"current_price",
|
|
117
|
+
"unrealized_pnl",
|
|
118
|
+
"realized_pnl",
|
|
119
|
+
"market_value",
|
|
120
|
+
"idempotency_key",
|
|
121
|
+
}
|
|
122
|
+
),
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
_REQUIRED: dict[str, frozenset[str]] = {
|
|
126
|
+
# The timestamp is required even though the server would default it to
|
|
127
|
+
# NOW(): a backfill row without its original time can't have a stable
|
|
128
|
+
# key and would corrupt the curve with wall-clock-at-import.
|
|
129
|
+
"trades": frozenset(
|
|
130
|
+
{"strategy", "broker", "account", "symbol", "side", "quantity", "price", "executed_at"}
|
|
131
|
+
),
|
|
132
|
+
"equity": frozenset({"strategy", "equity", "snapshot_at"}),
|
|
133
|
+
"positions": frozenset({"strategy", "broker", "account", "symbol", "quantity", "snapshot_at"}),
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
_TIME_FIELD = {"trades": "executed_at", "equity": "snapshot_at", "positions": "snapshot_at"}
|
|
137
|
+
_ENDPOINT = {
|
|
138
|
+
"trades": "/api/v1/trades/batch",
|
|
139
|
+
"equity": "/api/v1/equity/batch",
|
|
140
|
+
"positions": "/api/v1/positions/batch",
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
# Fields the server parses as booleans; CSV gives us strings.
|
|
144
|
+
_BOOL_FIELDS = frozenset({"is_close"})
|
|
145
|
+
_TRUTHY = frozenset({"true", "1", "yes", "y", "t"})
|
|
146
|
+
_FALSY = frozenset({"false", "0", "no", "n", "f", ""})
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class BackfillError(Exception):
|
|
150
|
+
"""Unrecoverable backfill failure (bad input file, exhausted retries).
|
|
151
|
+
Re-running after the cause is fixed is safe — keys are deterministic."""
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class BackfillResult:
|
|
156
|
+
"""Aggregate outcome.
|
|
157
|
+
|
|
158
|
+
`created + updated + conflicts + deduped + server_errors == sent`;
|
|
159
|
+
`skipped` rows never left the client (time filter or validation).
|
|
160
|
+
|
|
161
|
+
`conflicts` is the one worth reading on a re-run: those rows exist on the
|
|
162
|
+
server carrying DIFFERENT values to the file you just fed it. Either the
|
|
163
|
+
file is now right and the server is stale (re-run with `update_existing`),
|
|
164
|
+
or the server is right and the file has drifted. Silence on that question
|
|
165
|
+
is what OPM-1309 fixed.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
kind: str
|
|
169
|
+
rows_read: int = 0
|
|
170
|
+
filtered_out: int = 0 # outside [since, until]
|
|
171
|
+
invalid: int = 0 # failed client-side validation (see `row_errors`)
|
|
172
|
+
sent: int = 0
|
|
173
|
+
created: int = 0
|
|
174
|
+
updated: int = 0 # corrected in place (needs update_existing)
|
|
175
|
+
conflicts: int = 0 # stored row disagrees and was left alone
|
|
176
|
+
deduped: int = 0 # byte-identical re-send, nothing to do
|
|
177
|
+
server_errors: int = 0
|
|
178
|
+
batches: int = 0
|
|
179
|
+
row_errors: list[str] = field(default_factory=list) # capped at 50
|
|
180
|
+
warnings: list[str] = field(default_factory=list) # advisory, never fatal
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def ok(self) -> bool:
|
|
184
|
+
return self.invalid == 0 and self.server_errors == 0
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def derive_backfill_key(kind: str, row: Mapping[str, Any]) -> str:
|
|
188
|
+
"""Deterministic UUIDv5 idempotency key from the row's business key.
|
|
189
|
+
|
|
190
|
+
Timestamps are normalised to UTC ISO before hashing so '...Z' and
|
|
191
|
+
'...+00:00' spellings of the same instant derive the same key.
|
|
192
|
+
"""
|
|
193
|
+
if kind == "trades":
|
|
194
|
+
if row.get("external_trade_id"):
|
|
195
|
+
basis = f"trades|{row['strategy']}|{row['external_trade_id']}"
|
|
196
|
+
else:
|
|
197
|
+
ts = _norm_ts(row["executed_at"])
|
|
198
|
+
basis = (
|
|
199
|
+
f"trades|{row['strategy']}|{row['symbol']}|{ts}"
|
|
200
|
+
f"|{row['quantity']}|{row['price']}|{row['side']}"
|
|
201
|
+
)
|
|
202
|
+
elif kind == "equity":
|
|
203
|
+
basis = f"equity|{row['strategy']}|{_norm_ts(row['snapshot_at'])}"
|
|
204
|
+
elif kind == "positions":
|
|
205
|
+
basis = (
|
|
206
|
+
f"positions|{row['strategy']}|{row.get('account', '')}"
|
|
207
|
+
f"|{row['symbol']}|{_norm_ts(row['snapshot_at'])}"
|
|
208
|
+
)
|
|
209
|
+
else:
|
|
210
|
+
raise BackfillError(f"unknown kind: {kind!r}")
|
|
211
|
+
return str(uuid.uuid5(BACKFILL_NS, basis))
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def parse_dt(value: str) -> datetime:
|
|
215
|
+
"""Parse an ISO-8601 timestamp ('Z' accepted). Naive values are taken as
|
|
216
|
+
UTC — the least-surprising reading for engine logs."""
|
|
217
|
+
dt = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
|
218
|
+
if dt.tzinfo is None:
|
|
219
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
220
|
+
return dt
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _norm_ts(value: str) -> str:
|
|
224
|
+
return parse_dt(str(value)).astimezone(timezone.utc).isoformat()
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _map_row(
|
|
228
|
+
raw: Mapping[str, str],
|
|
229
|
+
kind: str,
|
|
230
|
+
mapping: Mapping[str, str] | None,
|
|
231
|
+
defaults: Mapping[str, str | None],
|
|
232
|
+
) -> dict[str, Any]:
|
|
233
|
+
"""Rename headers via `mapping`, inject `defaults` for absent routing
|
|
234
|
+
fields, coerce booleans, and drop empty/unknown columns.
|
|
235
|
+
|
|
236
|
+
Values are `str` except the coerced booleans, hence `dict[str, Any]`."""
|
|
237
|
+
row: dict[str, Any] = {}
|
|
238
|
+
for col, value in raw.items():
|
|
239
|
+
if col is None:
|
|
240
|
+
continue # ragged row — extra cells beyond the header
|
|
241
|
+
name = (mapping or {}).get(col, col).strip()
|
|
242
|
+
cell = (value or "").strip()
|
|
243
|
+
if name not in _FIELDS[kind] or not cell:
|
|
244
|
+
continue
|
|
245
|
+
if name in _BOOL_FIELDS:
|
|
246
|
+
lowered = cell.lower()
|
|
247
|
+
if lowered in _TRUTHY:
|
|
248
|
+
row[name] = True
|
|
249
|
+
elif lowered in _FALSY:
|
|
250
|
+
row[name] = False
|
|
251
|
+
else:
|
|
252
|
+
row[name] = cell # let the server report it per-row
|
|
253
|
+
else:
|
|
254
|
+
row[name] = cell
|
|
255
|
+
for name, default_value in defaults.items():
|
|
256
|
+
if default_value and name in _FIELDS[kind]:
|
|
257
|
+
row.setdefault(name, default_value)
|
|
258
|
+
return row
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def run_backfill(
|
|
262
|
+
config: Config,
|
|
263
|
+
source: Path | str,
|
|
264
|
+
*,
|
|
265
|
+
kind: str,
|
|
266
|
+
mapping: Mapping[str, str] | None = None,
|
|
267
|
+
defaults: Mapping[str, str | None] | None = None,
|
|
268
|
+
since: datetime | None = None,
|
|
269
|
+
until: datetime | None = None,
|
|
270
|
+
batch_size: int = 500,
|
|
271
|
+
dry_run: bool = False,
|
|
272
|
+
update_existing: bool = False,
|
|
273
|
+
progress: Callable[[str], None] | None = None,
|
|
274
|
+
) -> BackfillResult:
|
|
275
|
+
"""Backfill one CSV file. Returns the aggregate result; raises
|
|
276
|
+
`BackfillError` on unreadable input or exhausted transport retries
|
|
277
|
+
(safe to re-run — deterministic keys make the retry a no-op).
|
|
278
|
+
|
|
279
|
+
With `update_existing`, a row that already exists under the same
|
|
280
|
+
deterministic key is CORRECTED to match the file rather than ignored. Off
|
|
281
|
+
by default, because the ordinary reason to re-run is to resume an
|
|
282
|
+
interrupted load, and that must not rewrite anything."""
|
|
283
|
+
if kind not in _FIELDS:
|
|
284
|
+
raise BackfillError(f"kind must be one of {sorted(_FIELDS)}, got {kind!r}")
|
|
285
|
+
if not 1 <= batch_size <= SERVER_BATCH_CAP:
|
|
286
|
+
raise BackfillError(f"batch_size must be 1..{SERVER_BATCH_CAP}")
|
|
287
|
+
if since and since.tzinfo is None:
|
|
288
|
+
since = since.replace(tzinfo=timezone.utc)
|
|
289
|
+
if until and until.tzinfo is None:
|
|
290
|
+
until = until.replace(tzinfo=timezone.utc)
|
|
291
|
+
|
|
292
|
+
result = BackfillResult(kind=kind)
|
|
293
|
+
time_field = _TIME_FIELD[kind]
|
|
294
|
+
rows: list[dict[str, Any]] = []
|
|
295
|
+
|
|
296
|
+
try:
|
|
297
|
+
with Path(source).open(newline="", encoding="utf-8-sig") as fh:
|
|
298
|
+
for line_no, raw in enumerate(csv.DictReader(fh), start=1):
|
|
299
|
+
result.rows_read += 1
|
|
300
|
+
row = _map_row(raw, kind, mapping, defaults or {})
|
|
301
|
+
|
|
302
|
+
missing = _REQUIRED[kind] - row.keys()
|
|
303
|
+
if missing:
|
|
304
|
+
result.invalid += 1
|
|
305
|
+
if len(result.row_errors) < 50:
|
|
306
|
+
result.row_errors.append(f"row {line_no}: missing {sorted(missing)}")
|
|
307
|
+
continue
|
|
308
|
+
try:
|
|
309
|
+
ts = parse_dt(row[time_field])
|
|
310
|
+
except ValueError as exc:
|
|
311
|
+
result.invalid += 1
|
|
312
|
+
if len(result.row_errors) < 50:
|
|
313
|
+
result.row_errors.append(f"row {line_no}: bad {time_field}: {exc}")
|
|
314
|
+
continue
|
|
315
|
+
|
|
316
|
+
if (since and ts < since) or (until and ts > until):
|
|
317
|
+
result.filtered_out += 1
|
|
318
|
+
continue
|
|
319
|
+
|
|
320
|
+
row.setdefault("idempotency_key", derive_backfill_key(kind, row))
|
|
321
|
+
rows.append(row)
|
|
322
|
+
|
|
323
|
+
if kind == "equity":
|
|
324
|
+
_warn_on_non_positive_equity(rows, result)
|
|
325
|
+
except OSError as exc:
|
|
326
|
+
raise BackfillError(f"cannot read {source}: {exc}") from exc
|
|
327
|
+
|
|
328
|
+
if dry_run:
|
|
329
|
+
if progress:
|
|
330
|
+
progress(
|
|
331
|
+
f"dry-run: {len(rows)} row(s) would be sent in {_n_batches(rows, batch_size)} batch(es)"
|
|
332
|
+
)
|
|
333
|
+
return result
|
|
334
|
+
|
|
335
|
+
with httpx.Client(
|
|
336
|
+
base_url=config.api_url,
|
|
337
|
+
headers=_headers(config),
|
|
338
|
+
timeout=config.export_timeout_s,
|
|
339
|
+
) as client:
|
|
340
|
+
for start in range(0, len(rows), batch_size):
|
|
341
|
+
chunk = rows[start : start + batch_size]
|
|
342
|
+
body = _post_batch(
|
|
343
|
+
client,
|
|
344
|
+
_ENDPOINT[kind],
|
|
345
|
+
chunk,
|
|
346
|
+
config,
|
|
347
|
+
result,
|
|
348
|
+
update_existing=update_existing,
|
|
349
|
+
)
|
|
350
|
+
result.batches += 1
|
|
351
|
+
result.sent += len(chunk)
|
|
352
|
+
result.created += body["created"]
|
|
353
|
+
result.deduped += body["deduped"]
|
|
354
|
+
result.server_errors += body["errors"]
|
|
355
|
+
# `.get` because `positions` predates OPM-1309 and its response
|
|
356
|
+
# body carries neither counter. Absent means zero, not an error.
|
|
357
|
+
result.updated += body.get("updated", 0)
|
|
358
|
+
result.conflicts += body.get("conflicts", 0)
|
|
359
|
+
for item in body["items"]:
|
|
360
|
+
if item["status"] == "error" and len(result.row_errors) < 50:
|
|
361
|
+
result.row_errors.append(
|
|
362
|
+
f"batch {result.batches} item {item.get('index', item.get('row'))}: {item.get('error')}"
|
|
363
|
+
)
|
|
364
|
+
if progress:
|
|
365
|
+
# Only mention updated/conflicts once there are some: on the
|
|
366
|
+
# overwhelmingly common clean load both are zero, and two more
|
|
367
|
+
# always-zero counters is noise in front of the one that moves.
|
|
368
|
+
extra = ""
|
|
369
|
+
if result.updated:
|
|
370
|
+
extra += f" updated={result.updated}"
|
|
371
|
+
if result.conflicts:
|
|
372
|
+
extra += f" conflicts={result.conflicts}"
|
|
373
|
+
progress(
|
|
374
|
+
f"batch {result.batches}: sent={result.sent}/{len(rows)} "
|
|
375
|
+
f"created={result.created} deduped={result.deduped}{extra} "
|
|
376
|
+
f"errors={result.server_errors}"
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
return result
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _warn_on_non_positive_equity(rows: list[dict[str, Any]], result: BackfillResult) -> None:
|
|
383
|
+
"""Flag an equity file that is really a PnL file (OPM-1302).
|
|
384
|
+
|
|
385
|
+
`equity` means *account equity* — capital base plus PnL — because every
|
|
386
|
+
ratio metric downstream is a percentage change off the previous point. A
|
|
387
|
+
realised-PnL series starts at 0.00 and can cross it, which makes those
|
|
388
|
+
returns undefined at best and sign-inverted at worst. The rows are still
|
|
389
|
+
sent (the curve itself is real and worth charting); the warning is so the
|
|
390
|
+
trader learns it here rather than from a Sharpe with the wrong sign.
|
|
391
|
+
"""
|
|
392
|
+
zeros = 0
|
|
393
|
+
negatives = 0
|
|
394
|
+
for row in rows:
|
|
395
|
+
try:
|
|
396
|
+
value = float(row.get("equity", ""))
|
|
397
|
+
except (TypeError, ValueError):
|
|
398
|
+
continue # let the server report an unparseable value per-row
|
|
399
|
+
if value == 0:
|
|
400
|
+
zeros += 1
|
|
401
|
+
elif value < 0:
|
|
402
|
+
negatives += 1
|
|
403
|
+
if not zeros and not negatives:
|
|
404
|
+
return
|
|
405
|
+
result.warnings.append(
|
|
406
|
+
f"{zeros} zero and {negatives} negative equity value(s): 'equity' means "
|
|
407
|
+
"account equity (capital base + PnL), not PnL. Percentage metrics "
|
|
408
|
+
"(Sharpe, CAGR, drawdown) will be hidden for this strategy until the "
|
|
409
|
+
"curve is strictly positive."
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _headers(config: Config) -> dict[str, str]:
|
|
414
|
+
headers = {"User-Agent": "alphaboard-sdk/0.1 backfill"}
|
|
415
|
+
if config.api_key:
|
|
416
|
+
headers["Authorization"] = f"Bearer {config.api_key}"
|
|
417
|
+
return headers
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _n_batches(rows: list[Any], batch_size: int) -> int:
|
|
421
|
+
return (len(rows) + batch_size - 1) // batch_size
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _post_batch(
|
|
425
|
+
client: httpx.Client,
|
|
426
|
+
endpoint: str,
|
|
427
|
+
chunk: list[dict[str, Any]],
|
|
428
|
+
config: Config,
|
|
429
|
+
result: BackfillResult,
|
|
430
|
+
*,
|
|
431
|
+
update_existing: bool = False,
|
|
432
|
+
) -> dict[str, Any]:
|
|
433
|
+
"""POST one chunk with bounded retries on 5xx/transport errors. 4xx is
|
|
434
|
+
terminal — it means our payload is wrong and a retry can't fix it."""
|
|
435
|
+
# Sent only when set, so the request an older server sees is byte-for-byte
|
|
436
|
+
# the one it saw before this flag existed.
|
|
437
|
+
params = {"update_existing": "true"} if update_existing else None
|
|
438
|
+
last_error = ""
|
|
439
|
+
for attempt in range(config.max_retry_attempts):
|
|
440
|
+
try:
|
|
441
|
+
resp = client.post(endpoint, json={"items": chunk}, params=params)
|
|
442
|
+
if resp.status_code < 400:
|
|
443
|
+
body: dict[str, Any] = resp.json()
|
|
444
|
+
return body
|
|
445
|
+
if 400 <= resp.status_code < 500:
|
|
446
|
+
raise BackfillError(
|
|
447
|
+
f"{endpoint} rejected the batch ({resp.status_code}): "
|
|
448
|
+
f"{resp.text[:300]} — after {result.sent} row(s) accepted; "
|
|
449
|
+
"fix the input and re-run (already-accepted rows will dedupe)"
|
|
450
|
+
)
|
|
451
|
+
last_error = f"{resp.status_code}: {resp.text[:200]}"
|
|
452
|
+
except httpx.HTTPError as exc:
|
|
453
|
+
last_error = repr(exc)
|
|
454
|
+
backoff = min(config.max_backoff_s, 2**attempt)
|
|
455
|
+
logger.warning(
|
|
456
|
+
"backfill %s attempt %d failed (%s); retrying in %.0fs",
|
|
457
|
+
endpoint,
|
|
458
|
+
attempt + 1,
|
|
459
|
+
last_error,
|
|
460
|
+
backoff,
|
|
461
|
+
)
|
|
462
|
+
time.sleep(backoff)
|
|
463
|
+
raise BackfillError(
|
|
464
|
+
f"{endpoint} unreachable after {config.max_retry_attempts} attempts ({last_error}) — "
|
|
465
|
+
f"{result.sent} row(s) were accepted before the failure; re-run to resume (no duplicates)"
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def parse_mapping(text: str | None) -> dict[str, str] | None:
|
|
470
|
+
"""Parse the CLI's --mapping JSON (`{csv_column: canonical_field}`)."""
|
|
471
|
+
if not text:
|
|
472
|
+
return None
|
|
473
|
+
try:
|
|
474
|
+
mapping = json.loads(text)
|
|
475
|
+
except json.JSONDecodeError as exc:
|
|
476
|
+
raise BackfillError(f"--mapping is not valid JSON: {exc}") from exc
|
|
477
|
+
if not isinstance(mapping, dict) or not all(
|
|
478
|
+
isinstance(k, str) and isinstance(v, str) for k, v in mapping.items()
|
|
479
|
+
):
|
|
480
|
+
raise BackfillError("--mapping must be a JSON object of {csv_column: canonical_field}")
|
|
481
|
+
return mapping
|
alphaboard/_cli.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""`alphaboard` command-line interface (OPM-729, OPM-1218).
|
|
2
|
+
|
|
3
|
+
alphaboard doctor
|
|
4
|
+
alphaboard backfill trades.csv --kind trades --strategy momo --broker ib --account main
|
|
5
|
+
alphaboard backfill equity.csv --kind equity --strategy momo --since 2025-01-01
|
|
6
|
+
alphaboard backfill positions.csv --kind positions --mapping '{"Time": "snapshot_at"}' --dry-run
|
|
7
|
+
|
|
8
|
+
`doctor` is the pre-flight: it checks that the API is reachable, that the key
|
|
9
|
+
authenticates, that ingestion actually accepts a write, and then prints the
|
|
10
|
+
coverage census so "am I being watched?" has an answer before anything is
|
|
11
|
+
trusted. `backfill` reads a CSV export and replays it through the server's
|
|
12
|
+
/batch endpoints with deterministic UUIDv5 idempotency keys — re-running the
|
|
13
|
+
same file is a no-op. `--strategy/--broker/--account` fill in routing fields
|
|
14
|
+
the CSV doesn't carry; columns present in the CSV win.
|
|
15
|
+
|
|
16
|
+
If a re-run reports `conflicts=N`, N rows already on the server disagree with
|
|
17
|
+
the file you just fed it. Re-run with `--update-existing` to make the server
|
|
18
|
+
match the file.
|
|
19
|
+
|
|
20
|
+
api_url/api_key resolve like the SDK does everywhere else
|
|
21
|
+
(flags > ALPHABOARD_* env > ~/.alphaboard/config.toml).
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import argparse
|
|
27
|
+
import sys
|
|
28
|
+
from typing import IO, Any
|
|
29
|
+
|
|
30
|
+
from alphaboard._backfill import BackfillError, parse_dt, parse_mapping, run_backfill
|
|
31
|
+
from alphaboard._config import resolve
|
|
32
|
+
from alphaboard._doctor import render, run_doctor
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
36
|
+
parser = argparse.ArgumentParser(prog="alphaboard", description="AlphaBoard SDK utilities.")
|
|
37
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
38
|
+
|
|
39
|
+
bf = sub.add_parser(
|
|
40
|
+
"backfill",
|
|
41
|
+
help="Replay a historical CSV (trades/equity/positions) — safe to re-run.",
|
|
42
|
+
)
|
|
43
|
+
bf.add_argument("file", help="CSV file; first row is the header.")
|
|
44
|
+
bf.add_argument("--kind", required=True, choices=["trades", "equity", "positions"])
|
|
45
|
+
bf.add_argument("--strategy", help="Strategy name for rows without a strategy column.")
|
|
46
|
+
bf.add_argument("--broker", help="Broker name for rows without a broker column.")
|
|
47
|
+
bf.add_argument("--account", help="Account id for rows without an account column.")
|
|
48
|
+
bf.add_argument(
|
|
49
|
+
"--mapping",
|
|
50
|
+
help='JSON {csv_column: canonical_field} to rename headers, e.g. \'{"Time": "executed_at"}\'.',
|
|
51
|
+
)
|
|
52
|
+
bf.add_argument("--since", help="Only rows at/after this ISO timestamp.")
|
|
53
|
+
bf.add_argument("--until", help="Only rows at/before this ISO timestamp.")
|
|
54
|
+
bf.add_argument(
|
|
55
|
+
"--api-url", help="AlphaBoard server (default: ALPHABOARD_API_URL / config file)."
|
|
56
|
+
)
|
|
57
|
+
bf.add_argument("--api-key", help="API key (default: ALPHABOARD_API_KEY / config file).")
|
|
58
|
+
bf.add_argument("--batch-size", type=int, default=500, help="Rows per request (max 1000).")
|
|
59
|
+
bf.add_argument(
|
|
60
|
+
"--dry-run",
|
|
61
|
+
action="store_true",
|
|
62
|
+
help="Parse, validate and count rows without sending anything.",
|
|
63
|
+
)
|
|
64
|
+
bf.add_argument(
|
|
65
|
+
"--update-existing",
|
|
66
|
+
action="store_true",
|
|
67
|
+
help=(
|
|
68
|
+
"Correct rows that already exist so they match this file. Without "
|
|
69
|
+
"it a disagreeing row is counted as a conflict and left alone. Use "
|
|
70
|
+
"after fixing the source of a bad load — not to resume one."
|
|
71
|
+
),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
doc = sub.add_parser(
|
|
75
|
+
"doctor",
|
|
76
|
+
help="Check the API, the key, ingestion and coverage — then print the census.",
|
|
77
|
+
)
|
|
78
|
+
doc.add_argument(
|
|
79
|
+
"--api-url", help="AlphaBoard server (default: ALPHABOARD_API_URL / config file)."
|
|
80
|
+
)
|
|
81
|
+
doc.add_argument("--api-key", help="API key (default: ALPHABOARD_API_KEY / config file).")
|
|
82
|
+
doc.add_argument(
|
|
83
|
+
"--no-probe",
|
|
84
|
+
action="store_true",
|
|
85
|
+
help=(
|
|
86
|
+
"Skip the probe heartbeat. The probe is a strategy-less "
|
|
87
|
+
"infrastructure ping, so it cannot mask a silent strategy, but "
|
|
88
|
+
"read-only environments may want the checks without the write."
|
|
89
|
+
),
|
|
90
|
+
)
|
|
91
|
+
return parser
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _ascii_only(stream: IO[Any]) -> bool:
|
|
95
|
+
"""Can this console print a tick?
|
|
96
|
+
|
|
97
|
+
Windows consoles still default to a legacy codepage, and a `doctor` that
|
|
98
|
+
dies with `UnicodeEncodeError` on the machine you are diagnosing is worse
|
|
99
|
+
than no doctor at all.
|
|
100
|
+
"""
|
|
101
|
+
encoding = getattr(stream, "encoding", None) or "ascii"
|
|
102
|
+
try:
|
|
103
|
+
"✓✗·•".encode(encoding)
|
|
104
|
+
except (LookupError, UnicodeEncodeError):
|
|
105
|
+
return True
|
|
106
|
+
return False
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _doctor(args: argparse.Namespace) -> int:
|
|
110
|
+
config = resolve(
|
|
111
|
+
# `doctor` inspects the deployment, not one strategy, so this name
|
|
112
|
+
# only satisfies config resolution. Nothing is ever posted under it —
|
|
113
|
+
# the probe heartbeat deliberately carries no strategy at all.
|
|
114
|
+
strategy="doctor",
|
|
115
|
+
api_url=args.api_url,
|
|
116
|
+
api_key=args.api_key,
|
|
117
|
+
)
|
|
118
|
+
report = run_doctor(config, probe=not args.no_probe)
|
|
119
|
+
print(render(report, ascii_only=_ascii_only(sys.stdout)))
|
|
120
|
+
return report.exit_code
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def main(argv: list[str] | None = None) -> int:
|
|
124
|
+
args = _build_parser().parse_args(argv)
|
|
125
|
+
if args.command == "doctor":
|
|
126
|
+
return _doctor(args)
|
|
127
|
+
try:
|
|
128
|
+
config = resolve(
|
|
129
|
+
# `strategy` here only satisfies config resolution (url/key/retry
|
|
130
|
+
# tuning); row routing comes from `defaults` + the CSV itself.
|
|
131
|
+
strategy=args.strategy or "backfill",
|
|
132
|
+
api_url=args.api_url,
|
|
133
|
+
api_key=args.api_key,
|
|
134
|
+
)
|
|
135
|
+
result = run_backfill(
|
|
136
|
+
config,
|
|
137
|
+
args.file,
|
|
138
|
+
kind=args.kind,
|
|
139
|
+
mapping=parse_mapping(args.mapping),
|
|
140
|
+
defaults={
|
|
141
|
+
"strategy": args.strategy,
|
|
142
|
+
"broker": args.broker,
|
|
143
|
+
"account": args.account,
|
|
144
|
+
},
|
|
145
|
+
since=parse_dt(args.since) if args.since else None,
|
|
146
|
+
until=parse_dt(args.until) if args.until else None,
|
|
147
|
+
batch_size=args.batch_size,
|
|
148
|
+
dry_run=args.dry_run,
|
|
149
|
+
update_existing=args.update_existing,
|
|
150
|
+
progress=print,
|
|
151
|
+
)
|
|
152
|
+
except BackfillError as exc:
|
|
153
|
+
print(f"backfill failed: {exc}", file=sys.stderr)
|
|
154
|
+
return 2
|
|
155
|
+
except ValueError as exc: # bad --since/--until
|
|
156
|
+
print(f"backfill failed: {exc}", file=sys.stderr)
|
|
157
|
+
return 2
|
|
158
|
+
|
|
159
|
+
print(
|
|
160
|
+
f"{result.kind}: read={result.rows_read} sent={result.sent} "
|
|
161
|
+
f"created={result.created} updated={result.updated} "
|
|
162
|
+
f"conflicts={result.conflicts} deduped={result.deduped} "
|
|
163
|
+
f"filtered={result.filtered_out} invalid={result.invalid} "
|
|
164
|
+
f"server_errors={result.server_errors}"
|
|
165
|
+
)
|
|
166
|
+
if result.conflicts:
|
|
167
|
+
# Loud, on stderr, and last: a silent conflict count is the whole
|
|
168
|
+
# defect OPM-1309 exists to fix, and a number buried in a nine-field
|
|
169
|
+
# summary line is close enough to silent.
|
|
170
|
+
print(
|
|
171
|
+
f" warning: {result.conflicts} row(s) already exist with DIFFERENT "
|
|
172
|
+
"values and were left unchanged. Re-run with --update-existing to "
|
|
173
|
+
"make the server match this file.",
|
|
174
|
+
file=sys.stderr,
|
|
175
|
+
)
|
|
176
|
+
for line in result.warnings:
|
|
177
|
+
print(f" warning: {line}", file=sys.stderr)
|
|
178
|
+
for line in result.row_errors:
|
|
179
|
+
print(f" {line}", file=sys.stderr)
|
|
180
|
+
return 0 if result.ok else 1
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
if __name__ == "__main__": # pragma: no cover
|
|
184
|
+
sys.exit(main())
|