nlalpha-data 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nlalpha_data/__init__.py +30 -0
- nlalpha_data/bars_aligner.py +82 -0
- nlalpha_data/base.py +90 -0
- nlalpha_data/calendar/__init__.py +15 -0
- nlalpha_data/calendar/cn_equity.py +27 -0
- nlalpha_data/calendar/futures.py +54 -0
- nlalpha_data/calendar/us_equity.py +24 -0
- nlalpha_data/calendar_base.py +201 -0
- nlalpha_data/calendar_tools.py +133 -0
- nlalpha_data/discovery.py +129 -0
- nlalpha_data/knowledge/loading.md +17 -0
- nlalpha_data/knowledge/local-provider.md +17 -0
- nlalpha_data/loader.py +210 -0
- nlalpha_data/local_provider.py +407 -0
- nlalpha_data/mock_provider.py +123 -0
- nlalpha_data/parquet_store.py +134 -0
- nlalpha_data/server.py +392 -0
- nlalpha_data/sqlite_store.py +174 -0
- nlalpha_data/store.py +316 -0
- nlalpha_data-0.3.0.dist-info/METADATA +149 -0
- nlalpha_data-0.3.0.dist-info/RECORD +24 -0
- nlalpha_data-0.3.0.dist-info/WHEEL +4 -0
- nlalpha_data-0.3.0.dist-info/entry_points.txt +2 -0
- nlalpha_data-0.3.0.dist-info/licenses/LICENSE +661 -0
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
"""本地文件数据提供者(CSV / Excel)。
|
|
2
|
+
|
|
3
|
+
支持从本地 CSV 和 Excel 文件加载历史 K 线数据。
|
|
4
|
+
|
|
5
|
+
文件查找规则(按优先级):
|
|
6
|
+
1. {data_dir}/{symbol}.csv
|
|
7
|
+
2. {data_dir}/{symbol}.xlsx
|
|
8
|
+
3. {data_dir}/{symbol}.xls
|
|
9
|
+
|
|
10
|
+
CSV 读取:标准库 csv.DictReader(零额外依赖)
|
|
11
|
+
Excel 读取:懒导入 pandas(如果未安装,Excel 不可用但 CSV 仍可用)
|
|
12
|
+
|
|
13
|
+
列名自动映射:
|
|
14
|
+
- 日期列:datetime / date / trade_date / 日期 / 交易日期
|
|
15
|
+
- 开盘列:open / 开盘 / 开盘价
|
|
16
|
+
- 最高列:high / 最高 / 最高价
|
|
17
|
+
- 最低列:low / 最低 / 最低价
|
|
18
|
+
- 收盘列:close / 收盘 / 收盘价
|
|
19
|
+
- 成交量列:volume / vol / 成交量
|
|
20
|
+
|
|
21
|
+
用法:
|
|
22
|
+
provider = LocalFileDataProvider("workspace/data")
|
|
23
|
+
bars = provider.load_bars("000300.SH", "1d", "2023-01-01", "2023-06-30")
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import asyncio
|
|
27
|
+
import csv
|
|
28
|
+
import json
|
|
29
|
+
import math
|
|
30
|
+
import re
|
|
31
|
+
from datetime import date, datetime
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import Any
|
|
34
|
+
|
|
35
|
+
from nlalpha.core.contracts.market import Bar
|
|
36
|
+
from nlalpha.core.errors import DataError
|
|
37
|
+
from nlalpha.core.contracts.time import get_timezone, normalize_utc, timezone_metadata
|
|
38
|
+
from nlalpha_data.calendar_base import exchange_calendar, session_bounds
|
|
39
|
+
from nlalpha_data.base import _interval_delta
|
|
40
|
+
from nlalpha_data.store import filter_bar_dates
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# 列名映射表:所有可能的列名 -> 标准列名
|
|
44
|
+
_COLUMN_ALIASES = {
|
|
45
|
+
# 日期
|
|
46
|
+
"datetime": "datetime",
|
|
47
|
+
"date": "session_date",
|
|
48
|
+
"trade_date": "session_date",
|
|
49
|
+
"日期": "session_date",
|
|
50
|
+
"交易日期": "session_date",
|
|
51
|
+
"open_time": "open_time",
|
|
52
|
+
"close_time": "close_time",
|
|
53
|
+
"session_date": "session_date",
|
|
54
|
+
"open_interest": "open_interest",
|
|
55
|
+
"oi": "open_interest",
|
|
56
|
+
# 开盘
|
|
57
|
+
"open": "open",
|
|
58
|
+
"开盘": "open",
|
|
59
|
+
"开盘价": "open",
|
|
60
|
+
# 最高
|
|
61
|
+
"high": "high",
|
|
62
|
+
"最高": "high",
|
|
63
|
+
"最高价": "high",
|
|
64
|
+
# 最低
|
|
65
|
+
"low": "low",
|
|
66
|
+
"最低": "low",
|
|
67
|
+
"最低价": "low",
|
|
68
|
+
# 收盘
|
|
69
|
+
"close": "close",
|
|
70
|
+
"收盘": "close",
|
|
71
|
+
"收盘价": "close",
|
|
72
|
+
# 成交量
|
|
73
|
+
"volume": "volume",
|
|
74
|
+
"vol": "volume",
|
|
75
|
+
"成交量": "volume",
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _normalize_columns(headers: list[str]) -> dict[str, str]:
|
|
80
|
+
"""将原始列名映射为标准列名。
|
|
81
|
+
|
|
82
|
+
返回 {标准列名: 原始列名} 的映射字典。
|
|
83
|
+
如果某个标准列名找不到,对应的值为空字符串。
|
|
84
|
+
"""
|
|
85
|
+
# 建立原始列名 -> 标准列名的映射(小写匹配)
|
|
86
|
+
raw_to_std = {}
|
|
87
|
+
for h in headers:
|
|
88
|
+
lower = h.strip().lower()
|
|
89
|
+
if lower in _COLUMN_ALIASES:
|
|
90
|
+
raw_to_std[h] = _COLUMN_ALIASES[lower]
|
|
91
|
+
|
|
92
|
+
# 反向映射:标准列名 -> 原始列名(取第一个匹配的)
|
|
93
|
+
std_to_raw = {}
|
|
94
|
+
for raw, std in raw_to_std.items():
|
|
95
|
+
if std not in std_to_raw:
|
|
96
|
+
std_to_raw[std] = raw
|
|
97
|
+
|
|
98
|
+
return std_to_raw
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _parse_datetime(value: Any) -> datetime:
|
|
102
|
+
"""Parse exact microseconds and whole-second source UTC offsets."""
|
|
103
|
+
if not isinstance(value, datetime):
|
|
104
|
+
if not isinstance(value, str):
|
|
105
|
+
raise DataError(f"Invalid local datetime: {value!r}.", details={"time_contract": True})
|
|
106
|
+
value = value.strip()
|
|
107
|
+
if any(any(digit != "0" for digit in match.group(1)[6:]) for match in re.finditer(r"[.,](\d+)", value)):
|
|
108
|
+
raise DataError("Local Bar transport cannot truncate sub-microsecond timestamps.", details={"time_contract": True})
|
|
109
|
+
try:
|
|
110
|
+
parsed = datetime.fromisoformat(value)
|
|
111
|
+
except ValueError:
|
|
112
|
+
parsed = None
|
|
113
|
+
for pattern in ("%Y%m%d", "%Y/%m/%d"):
|
|
114
|
+
try:
|
|
115
|
+
parsed = datetime.strptime(value, pattern)
|
|
116
|
+
break
|
|
117
|
+
except ValueError:
|
|
118
|
+
continue
|
|
119
|
+
if parsed is None:
|
|
120
|
+
raise DataError(f"Unable to parse datetime: {value!r}.", details={"time_contract": True})
|
|
121
|
+
value = parsed
|
|
122
|
+
if getattr(value, "nanosecond", 0):
|
|
123
|
+
raise DataError("Local Bar transport cannot truncate sub-microsecond timestamps.", details={"time_contract": True})
|
|
124
|
+
offset = value.utcoffset()
|
|
125
|
+
if offset is not None and offset.microseconds:
|
|
126
|
+
raise DataError("Local source UTC offset must use whole seconds.", details={"time_contract": True})
|
|
127
|
+
return value.to_pydatetime() if hasattr(value, "to_pydatetime") else value
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _parse_session_date(value: Any) -> date:
|
|
131
|
+
"""Parse an actual date label without discarding a clock component."""
|
|
132
|
+
if isinstance(value, datetime):
|
|
133
|
+
if getattr(value, "nanosecond", 0) or value.time() != datetime.min.time() or value.tzinfo is not None:
|
|
134
|
+
raise DataError("Trading date labels must contain only a civil date.", details={"time_contract": True})
|
|
135
|
+
return value.date()
|
|
136
|
+
if isinstance(value, date):
|
|
137
|
+
return value
|
|
138
|
+
if not isinstance(value, str):
|
|
139
|
+
raise DataError(f"Invalid trading date: {value!r}.", details={"time_contract": True})
|
|
140
|
+
for pattern in ("%Y-%m-%d", "%Y%m%d", "%Y/%m/%d"):
|
|
141
|
+
try:
|
|
142
|
+
return datetime.strptime(value.strip(), pattern).date()
|
|
143
|
+
except ValueError:
|
|
144
|
+
continue
|
|
145
|
+
raise DataError(f"Invalid trading date: {value!r}.", details={"time_contract": True})
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _parse_float(value: Any, *, optional: bool = False) -> float | None:
|
|
149
|
+
"""Require a finite numeric price, volume, or optional open interest."""
|
|
150
|
+
if value is None or value == "":
|
|
151
|
+
if optional:
|
|
152
|
+
return None
|
|
153
|
+
raise DataError("Missing required numeric local bar value.")
|
|
154
|
+
try:
|
|
155
|
+
result = float(str(value).replace(",", "").strip())
|
|
156
|
+
except (ValueError, TypeError) as exc:
|
|
157
|
+
raise DataError(f"Invalid numeric local bar value: {value!r}.") from exc
|
|
158
|
+
if not math.isfinite(result):
|
|
159
|
+
raise DataError(f"Non-finite numeric local bar value: {value!r}.")
|
|
160
|
+
return result
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class LocalFileDataProvider:
|
|
164
|
+
"""本地文件数据提供者。"""
|
|
165
|
+
|
|
166
|
+
def __init__(self, data_dir: str | Path = "workspace/data", *, time_contracts: dict[str, dict] | None = None,
|
|
167
|
+
price_adjustments: dict[str, str] | None = None):
|
|
168
|
+
self.data_dir = Path(data_dir).resolve()
|
|
169
|
+
self.time_contracts = dict(time_contracts or {})
|
|
170
|
+
if price_adjustments is not None and (not isinstance(price_adjustments, dict) or any(
|
|
171
|
+
not isinstance(symbol, str) or not symbol or symbol != symbol.strip()
|
|
172
|
+
or not isinstance(value, str) or value not in {"unadjusted", "forward", "backward"}
|
|
173
|
+
for symbol, value in price_adjustments.items()
|
|
174
|
+
)):
|
|
175
|
+
raise DataError("Local price_adjustments must map symbols to unadjusted, forward, or backward.")
|
|
176
|
+
self.price_adjustments = dict(price_adjustments or {})
|
|
177
|
+
|
|
178
|
+
def cache_identity(self) -> dict:
|
|
179
|
+
"""Declare local storage and host configuration used to isolate cached bars."""
|
|
180
|
+
return {"data_dir": str(self.data_dir), "time_contracts": self.time_contracts,
|
|
181
|
+
"price_adjustments": self.price_adjustments}
|
|
182
|
+
|
|
183
|
+
async def describe_request(self, symbol: str, interval: str,
|
|
184
|
+
price_adjustment: str | None = None) -> dict:
|
|
185
|
+
"""Declare host-configured price adjustment for the requested symbol."""
|
|
186
|
+
if symbol in self.price_adjustments:
|
|
187
|
+
return {"price_adjustment": self.price_adjustments[symbol],
|
|
188
|
+
"evidence": {"type": "host_declaration", "reference": f"local.price_adjustments[{symbol}]"}}
|
|
189
|
+
return {"price_adjustment": "unknown", "evidence": {"type": "unknown", "reference": "No host price-adjustment declaration."}}
|
|
190
|
+
|
|
191
|
+
def is_available(self) -> bool:
|
|
192
|
+
"""检查数据目录是否存在。"""
|
|
193
|
+
return self.data_dir.exists() and self.data_dir.is_dir()
|
|
194
|
+
|
|
195
|
+
async def load_bars(self, symbol: str, interval: str, start: str, end: str, *,
|
|
196
|
+
price_adjustment: str | None = None) -> list[Bar]:
|
|
197
|
+
"""从本地文件加载 K 线数据。
|
|
198
|
+
|
|
199
|
+
Args:
|
|
200
|
+
symbol: 品种代码(用于查找文件)
|
|
201
|
+
interval: 周期(当前不用于文件筛选,仅做接口兼容)
|
|
202
|
+
start: 开始日期 "YYYY-MM-DD"
|
|
203
|
+
end: 结束日期 "YYYY-MM-DD"
|
|
204
|
+
price_adjustment: 仅做接口兼容;文件按宿主声明的复权属性原样返回
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
按时间升序排列的 Bar 列表
|
|
208
|
+
"""
|
|
209
|
+
file_path = self._find_file(symbol)
|
|
210
|
+
|
|
211
|
+
if file_path.suffix.lower() == ".csv":
|
|
212
|
+
rows = await self._read_csv_async(file_path)
|
|
213
|
+
elif file_path.suffix.lower() in (".xlsx", ".xls"):
|
|
214
|
+
# Excel 读取依赖 pandas,无原生 async 支持,使用线程池
|
|
215
|
+
rows = await asyncio.to_thread(self._read_excel, file_path)
|
|
216
|
+
else:
|
|
217
|
+
raise DataError(f"Unsupported file format: {file_path.suffix}")
|
|
218
|
+
|
|
219
|
+
bars = self._rows_to_bars(rows, symbol, interval, self._time_contract(symbol))
|
|
220
|
+
bars = self._filter_by_date(bars, start, end)
|
|
221
|
+
bars.sort(key=lambda b: b.datetime)
|
|
222
|
+
return bars
|
|
223
|
+
|
|
224
|
+
def _find_file(self, symbol: str) -> Path:
|
|
225
|
+
"""查找品种对应的数据文件。"""
|
|
226
|
+
# 安全处理:替换文件系统不友好字符
|
|
227
|
+
safe_symbol = symbol.replace("/", "_").replace("\\", "_")
|
|
228
|
+
candidates = [
|
|
229
|
+
self.data_dir / f"{safe_symbol}.csv",
|
|
230
|
+
self.data_dir / f"{safe_symbol}.xlsx",
|
|
231
|
+
self.data_dir / f"{safe_symbol}.xls",
|
|
232
|
+
]
|
|
233
|
+
for c in candidates:
|
|
234
|
+
if c.exists():
|
|
235
|
+
return c
|
|
236
|
+
raise DataError(
|
|
237
|
+
f"No data file found for symbol '{symbol}' in {self.data_dir}. "
|
|
238
|
+
f"Expected: {safe_symbol}.csv / {safe_symbol}.xlsx"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
def _read_csv(self, path: Path) -> list[dict]:
|
|
242
|
+
"""读取 CSV 文件(sync 版本,供线程池回退使用)。"""
|
|
243
|
+
try:
|
|
244
|
+
# 尝试 UTF-8(优先,处理 BOM)
|
|
245
|
+
with open(path, "r", encoding="utf-8-sig", newline="") as f:
|
|
246
|
+
reader = csv.DictReader(f)
|
|
247
|
+
return list(reader)
|
|
248
|
+
except UnicodeDecodeError:
|
|
249
|
+
# fallback 到 GBK(中文 Windows 常见编码)
|
|
250
|
+
with open(path, "r", encoding="gbk", newline="") as f:
|
|
251
|
+
reader = csv.DictReader(f)
|
|
252
|
+
return list(reader)
|
|
253
|
+
|
|
254
|
+
async def _read_csv_async(self, path: Path) -> list[dict]:
|
|
255
|
+
"""读取 CSV 文件(async 版本,使用 aiofiles)。"""
|
|
256
|
+
try:
|
|
257
|
+
import aiofiles
|
|
258
|
+
except ImportError:
|
|
259
|
+
# aiofiles 未安装时回退到 sync 版本
|
|
260
|
+
return await asyncio.to_thread(self._read_csv, path)
|
|
261
|
+
|
|
262
|
+
async def _read_with_encoding(encoding: str) -> list[dict]:
|
|
263
|
+
async with aiofiles.open(path, "r", encoding=encoding, newline="") as f:
|
|
264
|
+
content = await f.read()
|
|
265
|
+
# csv.DictReader 需要文件对象,从内存字符串构建
|
|
266
|
+
from io import StringIO
|
|
267
|
+
reader = csv.DictReader(StringIO(content))
|
|
268
|
+
return list(reader)
|
|
269
|
+
|
|
270
|
+
try:
|
|
271
|
+
return await _read_with_encoding("utf-8-sig")
|
|
272
|
+
except UnicodeDecodeError:
|
|
273
|
+
return await _read_with_encoding("gbk")
|
|
274
|
+
|
|
275
|
+
def _read_excel(self, path: Path) -> list[dict]:
|
|
276
|
+
"""读取 Excel 文件(懒导入 pandas)。"""
|
|
277
|
+
try:
|
|
278
|
+
import pandas as pd
|
|
279
|
+
except ImportError as e:
|
|
280
|
+
raise DataError(
|
|
281
|
+
"pandas is required for Excel support. "
|
|
282
|
+
"Install it with: pip install pandas openpyxl"
|
|
283
|
+
) from e
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
df = pd.read_excel(path)
|
|
287
|
+
return df.to_dict("records")
|
|
288
|
+
except Exception as e:
|
|
289
|
+
raise DataError(f"Failed to read Excel {path}: {e}") from e
|
|
290
|
+
|
|
291
|
+
def _time_contract(self, symbol: str) -> dict:
|
|
292
|
+
"""Load one explicit source declaration from configuration or sidecar."""
|
|
293
|
+
configured = self.time_contracts.get(symbol)
|
|
294
|
+
safe_symbol = symbol.replace("/", "_").replace("\\", "_")
|
|
295
|
+
path = self.data_dir / f"{safe_symbol}.time.json"
|
|
296
|
+
sidecar = None
|
|
297
|
+
if path.exists():
|
|
298
|
+
try:
|
|
299
|
+
sidecar = json.loads(path.read_text(encoding="utf-8"))
|
|
300
|
+
except (ValueError, OSError) as exc:
|
|
301
|
+
raise DataError(f"Cannot read source time contract {path}: {exc}", details={"time_contract": True}) from exc
|
|
302
|
+
if configured is not None and sidecar is not None and configured != sidecar:
|
|
303
|
+
raise DataError("Configured time contract contradicts the symbol sidecar.", details={"time_contract": True})
|
|
304
|
+
contract = configured if configured is not None else sidecar
|
|
305
|
+
if contract is None:
|
|
306
|
+
return {}
|
|
307
|
+
return self._validate_time_contract(contract)
|
|
308
|
+
|
|
309
|
+
@staticmethod
|
|
310
|
+
def _validate_time_contract(contract: dict) -> dict:
|
|
311
|
+
"""Validate source timezone, semantics, and calendar consistency."""
|
|
312
|
+
if not isinstance(contract, dict) or set(contract) - {"timezone", "timestamp_semantics", "calendar_id"}:
|
|
313
|
+
raise DataError("Local time contract accepts timezone, timestamp_semantics, and calendar_id.", details={"time_contract": True})
|
|
314
|
+
zone = contract.get("timezone")
|
|
315
|
+
if zone is not None:
|
|
316
|
+
get_timezone(zone)
|
|
317
|
+
semantics = contract.get("timestamp_semantics")
|
|
318
|
+
if semantics is not None and semantics not in {"open", "close", "trading_date"}:
|
|
319
|
+
raise DataError("Invalid local timestamp_semantics.", details={"time_contract": True})
|
|
320
|
+
calendar_id = contract.get("calendar_id")
|
|
321
|
+
if calendar_id is not None:
|
|
322
|
+
calendar_zone = str(exchange_calendar(calendar_id).tz)
|
|
323
|
+
if zone is not None and timezone_metadata(zone)["tzif_sha256"] != timezone_metadata(calendar_zone)["tzif_sha256"]:
|
|
324
|
+
raise DataError("Local source timezone contradicts the declared session calendar.", details={"time_contract": True})
|
|
325
|
+
return dict(contract)
|
|
326
|
+
|
|
327
|
+
def _rows_to_bars(self, rows: list[dict], symbol: str, interval: str = "1d", contract: dict | None = None) -> list[Bar]:
|
|
328
|
+
"""Resolve source timestamps and reject malformed rows explicitly."""
|
|
329
|
+
if not rows:
|
|
330
|
+
return []
|
|
331
|
+
contract = self._validate_time_contract(contract) if contract is not None else self._time_contract(symbol)
|
|
332
|
+
mapping = _normalize_columns(list(rows[0]))
|
|
333
|
+
missing = {"open", "high", "low", "close", "volume"} - set(mapping)
|
|
334
|
+
if missing:
|
|
335
|
+
raise DataError(f"Missing local OHLCV columns: {sorted(missing)}.")
|
|
336
|
+
explicit_bounds = "open_time" in mapping and "close_time" in mapping
|
|
337
|
+
if ("open_time" in mapping) != ("close_time" in mapping):
|
|
338
|
+
raise DataError("Local data must provide both open_time and close_time.", details={"time_contract": True})
|
|
339
|
+
if not explicit_bounds and "datetime" not in mapping and "session_date" not in mapping:
|
|
340
|
+
raise DataError("Local data requires explicit bounds, a datetime, or a declared trading date.", details={"time_contract": True})
|
|
341
|
+
bars = []
|
|
342
|
+
for number, row in enumerate(rows, 1):
|
|
343
|
+
try:
|
|
344
|
+
zone = contract.get("timezone")
|
|
345
|
+
semantics = contract.get("timestamp_semantics")
|
|
346
|
+
calendar_id = contract.get("calendar_id")
|
|
347
|
+
session = _parse_session_date(row[mapping["session_date"]]) if "session_date" in mapping else None
|
|
348
|
+
original = None
|
|
349
|
+
if explicit_bounds:
|
|
350
|
+
if semantics == "trading_date" and session is None:
|
|
351
|
+
raise DataError("trading_date semantics requires a session_date label.", details={"time_contract": True})
|
|
352
|
+
original = _parse_datetime(row[mapping["close_time"]])
|
|
353
|
+
begin = normalize_utc(_parse_datetime(row[mapping["open_time"]]), source_timezone=zone)
|
|
354
|
+
finish = normalize_utc(original, source_timezone=zone)
|
|
355
|
+
semantics = semantics or "close"
|
|
356
|
+
elif session is not None and "datetime" not in mapping:
|
|
357
|
+
if semantics != "trading_date" or calendar_id is None:
|
|
358
|
+
raise DataError("Date-only data requires timestamp_semantics=trading_date and an explicit calendar_id.", details={"time_contract": True})
|
|
359
|
+
begin, finish = session_bounds(calendar_id, session)
|
|
360
|
+
zone = zone or str(exchange_calendar(calendar_id).tz)
|
|
361
|
+
else:
|
|
362
|
+
raw = row[mapping["datetime"]]
|
|
363
|
+
is_date_only = isinstance(raw, date) and not isinstance(raw, datetime)
|
|
364
|
+
is_date_only |= isinstance(raw, str) and re.fullmatch(r"\d{4}[-/]\d{2}[-/]\d{2}|\d{8}", raw.strip()) is not None
|
|
365
|
+
if is_date_only or semantics == "trading_date":
|
|
366
|
+
if semantics != "trading_date" or calendar_id is None:
|
|
367
|
+
raise DataError("Date-only datetime requires trading_date semantics and an explicit calendar_id.", details={"time_contract": True})
|
|
368
|
+
raw_session = _parse_session_date(raw)
|
|
369
|
+
if session is not None and session != raw_session:
|
|
370
|
+
raise DataError("Trading date columns contradict one another.", details={"time_contract": True})
|
|
371
|
+
session = raw_session
|
|
372
|
+
begin, finish = session_bounds(calendar_id, session)
|
|
373
|
+
zone = zone or str(exchange_calendar(calendar_id).tz)
|
|
374
|
+
else:
|
|
375
|
+
if semantics not in {"open", "close"}:
|
|
376
|
+
raise DataError("One datetime column requires explicit open/close timestamp_semantics.", details={"time_contract": True})
|
|
377
|
+
original = _parse_datetime(raw)
|
|
378
|
+
instant = normalize_utc(original, source_timezone=zone)
|
|
379
|
+
width = _interval_delta(interval)
|
|
380
|
+
begin, finish = (instant, instant + width) if semantics == "open" else (instant - width, instant)
|
|
381
|
+
source_offset = None
|
|
382
|
+
if original is not None and original.tzinfo is not None and original.utcoffset() is not None:
|
|
383
|
+
source_offset = int(original.utcoffset().total_seconds())
|
|
384
|
+
zone = zone or getattr(original.tzinfo, "key", None)
|
|
385
|
+
if zone is None and source_offset == 0:
|
|
386
|
+
zone = "UTC"
|
|
387
|
+
elif zone is not None:
|
|
388
|
+
source_instant = begin if semantics == "open" else finish
|
|
389
|
+
source_offset = int(source_instant.astimezone(get_timezone(zone)).utcoffset().total_seconds())
|
|
390
|
+
bars.append(Bar(
|
|
391
|
+
symbol=symbol, datetime=finish, open_time=begin, session_date=session,
|
|
392
|
+
source_timezone=zone, source_utc_offset_seconds=source_offset,
|
|
393
|
+
source_timestamp_semantics=semantics, calendar_id=calendar_id,
|
|
394
|
+
open=_parse_float(row[mapping["open"]]), high=_parse_float(row[mapping["high"]]),
|
|
395
|
+
low=_parse_float(row[mapping["low"]]), close=_parse_float(row[mapping["close"]]),
|
|
396
|
+
volume=_parse_float(row[mapping["volume"]]),
|
|
397
|
+
open_interest=_parse_float(row.get(mapping.get("open_interest", "")), optional=True),
|
|
398
|
+
))
|
|
399
|
+
except Exception as exc:
|
|
400
|
+
details = dict(exc.details) if isinstance(exc, DataError) else {}
|
|
401
|
+
details["time_contract"] = True
|
|
402
|
+
raise DataError(f"Invalid local bar row {number} for {symbol}: {exc}", details=details) from exc
|
|
403
|
+
return sorted(bars, key=lambda bar: bar.datetime)
|
|
404
|
+
|
|
405
|
+
def _filter_by_date(self, bars: list[Bar], start: str, end: str) -> list[Bar]:
|
|
406
|
+
"""Filter inclusive dates according to the original source contract."""
|
|
407
|
+
return filter_bar_dates(bars, start, end)
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Generate interval-aligned synthetic OHLCV bars for strategy validation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timedelta, timezone
|
|
6
|
+
from decimal import Decimal
|
|
7
|
+
import math
|
|
8
|
+
|
|
9
|
+
from nlalpha.core.contracts.market import Bar
|
|
10
|
+
|
|
11
|
+
from nlalpha_data.base import _interval_delta
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MockDataProvider:
|
|
15
|
+
"""Generate mock bars for each configured universe symbol."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, envelope, instruments=None):
|
|
18
|
+
self._envelope = envelope
|
|
19
|
+
self._bars: list[Bar] | None = None
|
|
20
|
+
universe = envelope.universe if hasattr(envelope, "universe") else {}
|
|
21
|
+
self._symbols = list(universe.get("symbols", []))
|
|
22
|
+
if not self._symbols:
|
|
23
|
+
self._symbols = [leg.get("symbol") for leg in universe.get("legs", [])]
|
|
24
|
+
if (
|
|
25
|
+
not self._symbols
|
|
26
|
+
or any(
|
|
27
|
+
not isinstance(symbol, str) or not symbol or symbol != symbol.strip()
|
|
28
|
+
for symbol in self._symbols
|
|
29
|
+
)
|
|
30
|
+
or len(set(self._symbols)) != len(self._symbols)
|
|
31
|
+
):
|
|
32
|
+
raise ValueError("mock validation requires unique configured universe symbols")
|
|
33
|
+
data = envelope.data if hasattr(envelope, "data") else {}
|
|
34
|
+
execution = envelope.execution if hasattr(envelope, "execution") else {}
|
|
35
|
+
self._interval = _interval_delta(data.get("interval") or execution.get("bar_interval"))
|
|
36
|
+
contract = getattr(envelope, "contract", {}) or {}
|
|
37
|
+
self._instruments = instruments or contract.get("instruments", {})
|
|
38
|
+
if not hasattr(self._instruments, "get"):
|
|
39
|
+
raise ValueError("mock instrument settings must be a symbol mapping")
|
|
40
|
+
|
|
41
|
+
@staticmethod
|
|
42
|
+
def _common_increment(*values: object) -> Decimal:
|
|
43
|
+
"""Return an increment that is a multiple of all configured steps."""
|
|
44
|
+
increments = [Decimal(str(value)) for value in values if value is not None]
|
|
45
|
+
increments = [value for value in increments if value > 0]
|
|
46
|
+
if not increments:
|
|
47
|
+
return Decimal(1)
|
|
48
|
+
scale = max(max(0, -value.as_tuple().exponent) for value in increments)
|
|
49
|
+
scaled = [int(value.scaleb(scale)) for value in increments]
|
|
50
|
+
common = math.lcm(*scaled)
|
|
51
|
+
return Decimal(common).scaleb(-scale)
|
|
52
|
+
|
|
53
|
+
def _instrument_steps(self, symbol: str) -> tuple[Decimal, Decimal]:
|
|
54
|
+
instrument = self._instruments.get(symbol, {})
|
|
55
|
+
if not hasattr(instrument, "get"):
|
|
56
|
+
raise ValueError(f"mock instrument settings for {symbol!r} must be a mapping")
|
|
57
|
+
kind = instrument.get("instrument_type")
|
|
58
|
+
default_price_precision = 5 if kind == "currency_pair" else 2
|
|
59
|
+
price_precision = instrument.get("price_precision", default_price_precision)
|
|
60
|
+
if isinstance(price_precision, bool) or not isinstance(price_precision, int) or not 0 <= price_precision <= 12:
|
|
61
|
+
raise ValueError(f"mock price_precision for {symbol!r} must be an integer in [0, 12]")
|
|
62
|
+
price_step = self._common_increment(
|
|
63
|
+
instrument.get("price_increment"), Decimal(1).scaleb(-price_precision)
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
default_size_precision = 0 if kind in {"equity", "futures", None} else 8
|
|
67
|
+
size_precision = instrument.get("size_precision", default_size_precision)
|
|
68
|
+
if isinstance(size_precision, bool) or not isinstance(size_precision, int) or not 0 <= size_precision <= 12:
|
|
69
|
+
raise ValueError(f"mock size_precision for {symbol!r} must be an integer in [0, 12]")
|
|
70
|
+
configured_size_increment = instrument.get("size_increment")
|
|
71
|
+
if configured_size_increment is None:
|
|
72
|
+
configured_size_increment = Decimal(1).scaleb(-size_precision)
|
|
73
|
+
volume_step = self._common_increment(
|
|
74
|
+
configured_size_increment,
|
|
75
|
+
instrument.get("lot_size"),
|
|
76
|
+
Decimal(1).scaleb(-size_precision),
|
|
77
|
+
)
|
|
78
|
+
return price_step, volume_step
|
|
79
|
+
|
|
80
|
+
def is_available(self) -> bool:
|
|
81
|
+
return True
|
|
82
|
+
|
|
83
|
+
async def load_bars(self, symbol: str, interval: str, start: str, end: str) -> list[Bar]:
|
|
84
|
+
"""Return all generated bars or the requested symbol's bars."""
|
|
85
|
+
if self._bars is None:
|
|
86
|
+
self._bars = self._generate()
|
|
87
|
+
if symbol:
|
|
88
|
+
if symbol not in self._symbols:
|
|
89
|
+
raise ValueError(f"mock data does not include symbol {symbol!r}")
|
|
90
|
+
return [bar for bar in self._bars if bar.symbol == symbol]
|
|
91
|
+
return list(self._bars)
|
|
92
|
+
|
|
93
|
+
def _generate(self) -> list[Bar]:
|
|
94
|
+
"""Generate deterministic bars using each instrument's increments."""
|
|
95
|
+
flat_bars: list[Bar] = []
|
|
96
|
+
base_price = 100.0
|
|
97
|
+
dt = datetime(2023, 1, 1, 9, 30, tzinfo=timezone.utc) + self._interval
|
|
98
|
+
|
|
99
|
+
for seed, symbol in enumerate(self._symbols):
|
|
100
|
+
price_step, volume_step = self._instrument_steps(symbol)
|
|
101
|
+
base_ticks = max(1000, int(Decimal(str(base_price + seed * 10)) / price_step))
|
|
102
|
+
for i in range(200):
|
|
103
|
+
close_ticks = base_ticks + (i % 20)
|
|
104
|
+
close = price_step * close_ticks
|
|
105
|
+
open_p = price_step * (close_ticks - 1)
|
|
106
|
+
high = price_step * (close_ticks + 1)
|
|
107
|
+
low = price_step * (close_ticks - 2)
|
|
108
|
+
flat_bars.append(Bar(
|
|
109
|
+
symbol=symbol,
|
|
110
|
+
datetime=dt + self._interval * i,
|
|
111
|
+
open_time=dt + self._interval * (i - 1),
|
|
112
|
+
source_timezone="UTC",
|
|
113
|
+
source_timestamp_semantics="close",
|
|
114
|
+
open=float(open_p),
|
|
115
|
+
high=float(high),
|
|
116
|
+
low=float(low),
|
|
117
|
+
close=float(close),
|
|
118
|
+
volume=float(volume_step * (1000 + i)),
|
|
119
|
+
))
|
|
120
|
+
|
|
121
|
+
order = {symbol: index for index, symbol in enumerate(self._symbols)}
|
|
122
|
+
flat_bars.sort(key=lambda bar: (bar.datetime, order[bar.symbol]))
|
|
123
|
+
return flat_bars
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Store canonical bars atomically with exact query and source metadata."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import tempfile
|
|
8
|
+
|
|
9
|
+
from nlalpha.core.contracts.market import Bar
|
|
10
|
+
from nlalpha.core.errors import DataError
|
|
11
|
+
from nlalpha_data.store import (
|
|
12
|
+
BaseBarStore, BarOverview, _cache_path, _bars_to_table, _table_to_bars,
|
|
13
|
+
BAR_CACHE_NAMESPACE, BAR_CONTRACT_KEY, BAR_CONTRACT, SOURCE_KEY, QUERY_KEY, read_source_contract,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ParquetBarStore(BaseBarStore):
|
|
18
|
+
"""Persist exact bar queries inside one source-specific cache directory."""
|
|
19
|
+
|
|
20
|
+
def __init__(self, cache_dir: str | Path = "workspace/cache", *, source: dict | None = None):
|
|
21
|
+
self.cache_dir = Path(cache_dir) / BAR_CACHE_NAMESPACE
|
|
22
|
+
self.source = json.loads(json.dumps(source)) if source is not None else None
|
|
23
|
+
|
|
24
|
+
async def save(self, bars: list[Bar], symbol: str = "", interval: str = "", start: str = "", end: str = "") -> int:
|
|
25
|
+
"""Publish a complete Parquet file with an atomic same-directory rename."""
|
|
26
|
+
if not bars:
|
|
27
|
+
return 0
|
|
28
|
+
return await asyncio.to_thread(self._save, bars, symbol, interval, start, end)
|
|
29
|
+
|
|
30
|
+
def _save(self, bars, symbol, interval, start, end):
|
|
31
|
+
import pyarrow.parquet as pq
|
|
32
|
+
if not symbol or not interval:
|
|
33
|
+
raise DataError("Cache writes require explicit symbol and interval.")
|
|
34
|
+
if any(bar.symbol != symbol for bar in bars):
|
|
35
|
+
raise DataError("Cache bars must match the requested symbol.", details={"cache_contract": True})
|
|
36
|
+
query = [symbol, interval, start, end]
|
|
37
|
+
table = _bars_to_table(sorted(bars, key=lambda bar: bar.datetime))
|
|
38
|
+
metadata = {**(table.schema.metadata or {}), BAR_CONTRACT_KEY: BAR_CONTRACT,
|
|
39
|
+
QUERY_KEY: json.dumps(query).encode()}
|
|
40
|
+
if self.source is not None:
|
|
41
|
+
contract = {"schema": "nlalpha.market.source.v1", "assets": {symbol: {**self.source, "cache_hit": False}}}
|
|
42
|
+
metadata[SOURCE_KEY] = json.dumps(contract, sort_keys=True).encode()
|
|
43
|
+
read_source_contract(table.replace_schema_metadata(metadata))
|
|
44
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
target = _cache_path(*query, self.cache_dir)
|
|
46
|
+
descriptor, filename = tempfile.mkstemp(prefix=".bars-", suffix=".tmp", dir=self.cache_dir)
|
|
47
|
+
try:
|
|
48
|
+
with os.fdopen(descriptor, "wb") as output:
|
|
49
|
+
pq.write_table(table.replace_schema_metadata(metadata), output)
|
|
50
|
+
output.flush()
|
|
51
|
+
os.fsync(output.fileno())
|
|
52
|
+
os.replace(filename, target)
|
|
53
|
+
finally:
|
|
54
|
+
Path(filename).unlink(missing_ok=True)
|
|
55
|
+
return len(bars)
|
|
56
|
+
|
|
57
|
+
def _read(self, path: Path, expected_query=None):
|
|
58
|
+
import pyarrow.parquet as pq
|
|
59
|
+
try:
|
|
60
|
+
table = pq.read_table(path)
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
raise
|
|
63
|
+
except Exception as exc:
|
|
64
|
+
raise DataError(f"Cannot read Parquet cache: {exc}", details={"cache_contract": True}) from exc
|
|
65
|
+
bars = _cache_table_to_bars(table)
|
|
66
|
+
try:
|
|
67
|
+
metadata = table.schema.metadata or {}
|
|
68
|
+
query = json.loads(metadata[QUERY_KEY])
|
|
69
|
+
if (not isinstance(query, list) or len(query) != 4
|
|
70
|
+
or any(not isinstance(value, str) for value in query) or not query[0] or not query[1]):
|
|
71
|
+
raise ValueError("Invalid cached query")
|
|
72
|
+
contract = read_source_contract(table)
|
|
73
|
+
entry = None if contract is None else contract["assets"][query[0]]
|
|
74
|
+
source = None if entry is None else {key: value for key, value in entry.items() if key != "cache_hit"}
|
|
75
|
+
if path.name != _cache_path(*query, self.cache_dir).name:
|
|
76
|
+
raise ValueError("Cache filename contradicts query metadata")
|
|
77
|
+
if expected_query is not None and query != list(expected_query):
|
|
78
|
+
raise ValueError("Cache query mismatch")
|
|
79
|
+
if any(bar.symbol != query[0] for bar in bars):
|
|
80
|
+
raise ValueError("Cache bars contradict query symbol")
|
|
81
|
+
if source != self.source:
|
|
82
|
+
raise ValueError("Cache source declaration mismatch")
|
|
83
|
+
except (KeyError, ValueError, TypeError) as exc:
|
|
84
|
+
raise DataError(f"Invalid Parquet cache metadata: {exc}", details={"cache_contract": True}) from exc
|
|
85
|
+
return bars, query
|
|
86
|
+
|
|
87
|
+
async def load(self, symbol: str, interval: str, start: str, end: str) -> list[Bar]:
|
|
88
|
+
"""Read an exact query; only a missing file represents a cache miss."""
|
|
89
|
+
query = (symbol, interval, start, end)
|
|
90
|
+
bars, _ = await asyncio.to_thread(self._read, _cache_path(*query, self.cache_dir), query)
|
|
91
|
+
return bars
|
|
92
|
+
|
|
93
|
+
async def delete(self, symbol: str, interval: str) -> int:
|
|
94
|
+
"""Delete matching exact-query files using their original metadata."""
|
|
95
|
+
return await asyncio.to_thread(self._delete, symbol, interval)
|
|
96
|
+
|
|
97
|
+
def _delete(self, symbol, interval):
|
|
98
|
+
deleted = 0
|
|
99
|
+
for path in self.cache_dir.glob("*.parquet"):
|
|
100
|
+
_, query = self._read(path)
|
|
101
|
+
if query[:2] == [symbol, interval]:
|
|
102
|
+
path.unlink()
|
|
103
|
+
deleted += 1
|
|
104
|
+
return deleted
|
|
105
|
+
|
|
106
|
+
async def get_overview(self) -> list[BarOverview]:
|
|
107
|
+
"""Read all stored series without guessing original symbols from filenames."""
|
|
108
|
+
return await asyncio.to_thread(self._overview)
|
|
109
|
+
|
|
110
|
+
def _overview(self):
|
|
111
|
+
overviews = {}
|
|
112
|
+
for path in self.cache_dir.glob("*.parquet"):
|
|
113
|
+
bars, query = self._read(path)
|
|
114
|
+
if not bars:
|
|
115
|
+
continue
|
|
116
|
+
key = tuple(query[:2])
|
|
117
|
+
previous = overviews.get(key)
|
|
118
|
+
if previous is None:
|
|
119
|
+
overviews[key] = BarOverview(*key, count=len(bars), start=bars[0].datetime, end=bars[-1].datetime)
|
|
120
|
+
else:
|
|
121
|
+
previous.count += len(bars)
|
|
122
|
+
previous.start = min(previous.start, bars[0].datetime)
|
|
123
|
+
previous.end = max(previous.end, bars[-1].datetime)
|
|
124
|
+
return list(overviews.values())
|
|
125
|
+
|
|
126
|
+
async def aclose(self) -> None:
|
|
127
|
+
"""Release the file store's resource scope."""
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _cache_table_to_bars(table, fallback_symbol: str = "") -> list[Bar]:
|
|
131
|
+
"""Validate cached volume semantics before restoring canonical bars."""
|
|
132
|
+
if (table.schema.metadata or {}).get(BAR_CONTRACT_KEY) != BAR_CONTRACT:
|
|
133
|
+
raise DataError("Parquet cache requires the current instrument-quantity bar contract.", details={"cache_contract": True})
|
|
134
|
+
return _table_to_bars(table, fallback_symbol=fallback_symbol)
|