nlalpha-data 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,407 @@
1
+ """本地文件数据提供者(CSV / Excel)。
2
+
3
+ 支持从本地 CSV 和 Excel 文件加载历史 K 线数据。
4
+
5
+ 文件查找规则(按优先级):
6
+ 1. {data_dir}/{symbol}.csv
7
+ 2. {data_dir}/{symbol}.xlsx
8
+ 3. {data_dir}/{symbol}.xls
9
+
10
+ CSV 读取:标准库 csv.DictReader(零额外依赖)
11
+ Excel 读取:懒导入 pandas(如果未安装,Excel 不可用但 CSV 仍可用)
12
+
13
+ 列名自动映射:
14
+ - 日期列:datetime / date / trade_date / 日期 / 交易日期
15
+ - 开盘列:open / 开盘 / 开盘价
16
+ - 最高列:high / 最高 / 最高价
17
+ - 最低列:low / 最低 / 最低价
18
+ - 收盘列:close / 收盘 / 收盘价
19
+ - 成交量列:volume / vol / 成交量
20
+
21
+ 用法:
22
+ provider = LocalFileDataProvider("workspace/data")
23
+ bars = provider.load_bars("000300.SH", "1d", "2023-01-01", "2023-06-30")
24
+ """
25
+
26
+ import asyncio
27
+ import csv
28
+ import json
29
+ import math
30
+ import re
31
+ from datetime import date, datetime
32
+ from pathlib import Path
33
+ from typing import Any
34
+
35
+ from nlalpha.core.contracts.market import Bar
36
+ from nlalpha.core.errors import DataError
37
+ from nlalpha.core.contracts.time import get_timezone, normalize_utc, timezone_metadata
38
+ from nlalpha_data.calendar_base import exchange_calendar, session_bounds
39
+ from nlalpha_data.base import _interval_delta
40
+ from nlalpha_data.store import filter_bar_dates
41
+
42
+
43
+ # 列名映射表:所有可能的列名 -> 标准列名
44
+ _COLUMN_ALIASES = {
45
+ # 日期
46
+ "datetime": "datetime",
47
+ "date": "session_date",
48
+ "trade_date": "session_date",
49
+ "日期": "session_date",
50
+ "交易日期": "session_date",
51
+ "open_time": "open_time",
52
+ "close_time": "close_time",
53
+ "session_date": "session_date",
54
+ "open_interest": "open_interest",
55
+ "oi": "open_interest",
56
+ # 开盘
57
+ "open": "open",
58
+ "开盘": "open",
59
+ "开盘价": "open",
60
+ # 最高
61
+ "high": "high",
62
+ "最高": "high",
63
+ "最高价": "high",
64
+ # 最低
65
+ "low": "low",
66
+ "最低": "low",
67
+ "最低价": "low",
68
+ # 收盘
69
+ "close": "close",
70
+ "收盘": "close",
71
+ "收盘价": "close",
72
+ # 成交量
73
+ "volume": "volume",
74
+ "vol": "volume",
75
+ "成交量": "volume",
76
+ }
77
+
78
+
79
+ def _normalize_columns(headers: list[str]) -> dict[str, str]:
80
+ """将原始列名映射为标准列名。
81
+
82
+ 返回 {标准列名: 原始列名} 的映射字典。
83
+ 如果某个标准列名找不到,对应的值为空字符串。
84
+ """
85
+ # 建立原始列名 -> 标准列名的映射(小写匹配)
86
+ raw_to_std = {}
87
+ for h in headers:
88
+ lower = h.strip().lower()
89
+ if lower in _COLUMN_ALIASES:
90
+ raw_to_std[h] = _COLUMN_ALIASES[lower]
91
+
92
+ # 反向映射:标准列名 -> 原始列名(取第一个匹配的)
93
+ std_to_raw = {}
94
+ for raw, std in raw_to_std.items():
95
+ if std not in std_to_raw:
96
+ std_to_raw[std] = raw
97
+
98
+ return std_to_raw
99
+
100
+
101
+ def _parse_datetime(value: Any) -> datetime:
102
+ """Parse exact microseconds and whole-second source UTC offsets."""
103
+ if not isinstance(value, datetime):
104
+ if not isinstance(value, str):
105
+ raise DataError(f"Invalid local datetime: {value!r}.", details={"time_contract": True})
106
+ value = value.strip()
107
+ if any(any(digit != "0" for digit in match.group(1)[6:]) for match in re.finditer(r"[.,](\d+)", value)):
108
+ raise DataError("Local Bar transport cannot truncate sub-microsecond timestamps.", details={"time_contract": True})
109
+ try:
110
+ parsed = datetime.fromisoformat(value)
111
+ except ValueError:
112
+ parsed = None
113
+ for pattern in ("%Y%m%d", "%Y/%m/%d"):
114
+ try:
115
+ parsed = datetime.strptime(value, pattern)
116
+ break
117
+ except ValueError:
118
+ continue
119
+ if parsed is None:
120
+ raise DataError(f"Unable to parse datetime: {value!r}.", details={"time_contract": True})
121
+ value = parsed
122
+ if getattr(value, "nanosecond", 0):
123
+ raise DataError("Local Bar transport cannot truncate sub-microsecond timestamps.", details={"time_contract": True})
124
+ offset = value.utcoffset()
125
+ if offset is not None and offset.microseconds:
126
+ raise DataError("Local source UTC offset must use whole seconds.", details={"time_contract": True})
127
+ return value.to_pydatetime() if hasattr(value, "to_pydatetime") else value
128
+
129
+
130
+ def _parse_session_date(value: Any) -> date:
131
+ """Parse an actual date label without discarding a clock component."""
132
+ if isinstance(value, datetime):
133
+ if getattr(value, "nanosecond", 0) or value.time() != datetime.min.time() or value.tzinfo is not None:
134
+ raise DataError("Trading date labels must contain only a civil date.", details={"time_contract": True})
135
+ return value.date()
136
+ if isinstance(value, date):
137
+ return value
138
+ if not isinstance(value, str):
139
+ raise DataError(f"Invalid trading date: {value!r}.", details={"time_contract": True})
140
+ for pattern in ("%Y-%m-%d", "%Y%m%d", "%Y/%m/%d"):
141
+ try:
142
+ return datetime.strptime(value.strip(), pattern).date()
143
+ except ValueError:
144
+ continue
145
+ raise DataError(f"Invalid trading date: {value!r}.", details={"time_contract": True})
146
+
147
+
148
+ def _parse_float(value: Any, *, optional: bool = False) -> float | None:
149
+ """Require a finite numeric price, volume, or optional open interest."""
150
+ if value is None or value == "":
151
+ if optional:
152
+ return None
153
+ raise DataError("Missing required numeric local bar value.")
154
+ try:
155
+ result = float(str(value).replace(",", "").strip())
156
+ except (ValueError, TypeError) as exc:
157
+ raise DataError(f"Invalid numeric local bar value: {value!r}.") from exc
158
+ if not math.isfinite(result):
159
+ raise DataError(f"Non-finite numeric local bar value: {value!r}.")
160
+ return result
161
+
162
+
163
+ class LocalFileDataProvider:
164
+ """本地文件数据提供者。"""
165
+
166
+ def __init__(self, data_dir: str | Path = "workspace/data", *, time_contracts: dict[str, dict] | None = None,
167
+ price_adjustments: dict[str, str] | None = None):
168
+ self.data_dir = Path(data_dir).resolve()
169
+ self.time_contracts = dict(time_contracts or {})
170
+ if price_adjustments is not None and (not isinstance(price_adjustments, dict) or any(
171
+ not isinstance(symbol, str) or not symbol or symbol != symbol.strip()
172
+ or not isinstance(value, str) or value not in {"unadjusted", "forward", "backward"}
173
+ for symbol, value in price_adjustments.items()
174
+ )):
175
+ raise DataError("Local price_adjustments must map symbols to unadjusted, forward, or backward.")
176
+ self.price_adjustments = dict(price_adjustments or {})
177
+
178
+ def cache_identity(self) -> dict:
179
+ """Declare local storage and host configuration used to isolate cached bars."""
180
+ return {"data_dir": str(self.data_dir), "time_contracts": self.time_contracts,
181
+ "price_adjustments": self.price_adjustments}
182
+
183
+ async def describe_request(self, symbol: str, interval: str,
184
+ price_adjustment: str | None = None) -> dict:
185
+ """Declare host-configured price adjustment for the requested symbol."""
186
+ if symbol in self.price_adjustments:
187
+ return {"price_adjustment": self.price_adjustments[symbol],
188
+ "evidence": {"type": "host_declaration", "reference": f"local.price_adjustments[{symbol}]"}}
189
+ return {"price_adjustment": "unknown", "evidence": {"type": "unknown", "reference": "No host price-adjustment declaration."}}
190
+
191
+ def is_available(self) -> bool:
192
+ """检查数据目录是否存在。"""
193
+ return self.data_dir.exists() and self.data_dir.is_dir()
194
+
195
+ async def load_bars(self, symbol: str, interval: str, start: str, end: str, *,
196
+ price_adjustment: str | None = None) -> list[Bar]:
197
+ """从本地文件加载 K 线数据。
198
+
199
+ Args:
200
+ symbol: 品种代码(用于查找文件)
201
+ interval: 周期(当前不用于文件筛选,仅做接口兼容)
202
+ start: 开始日期 "YYYY-MM-DD"
203
+ end: 结束日期 "YYYY-MM-DD"
204
+ price_adjustment: 仅做接口兼容;文件按宿主声明的复权属性原样返回
205
+
206
+ Returns:
207
+ 按时间升序排列的 Bar 列表
208
+ """
209
+ file_path = self._find_file(symbol)
210
+
211
+ if file_path.suffix.lower() == ".csv":
212
+ rows = await self._read_csv_async(file_path)
213
+ elif file_path.suffix.lower() in (".xlsx", ".xls"):
214
+ # Excel 读取依赖 pandas,无原生 async 支持,使用线程池
215
+ rows = await asyncio.to_thread(self._read_excel, file_path)
216
+ else:
217
+ raise DataError(f"Unsupported file format: {file_path.suffix}")
218
+
219
+ bars = self._rows_to_bars(rows, symbol, interval, self._time_contract(symbol))
220
+ bars = self._filter_by_date(bars, start, end)
221
+ bars.sort(key=lambda b: b.datetime)
222
+ return bars
223
+
224
+ def _find_file(self, symbol: str) -> Path:
225
+ """查找品种对应的数据文件。"""
226
+ # 安全处理:替换文件系统不友好字符
227
+ safe_symbol = symbol.replace("/", "_").replace("\\", "_")
228
+ candidates = [
229
+ self.data_dir / f"{safe_symbol}.csv",
230
+ self.data_dir / f"{safe_symbol}.xlsx",
231
+ self.data_dir / f"{safe_symbol}.xls",
232
+ ]
233
+ for c in candidates:
234
+ if c.exists():
235
+ return c
236
+ raise DataError(
237
+ f"No data file found for symbol '{symbol}' in {self.data_dir}. "
238
+ f"Expected: {safe_symbol}.csv / {safe_symbol}.xlsx"
239
+ )
240
+
241
+ def _read_csv(self, path: Path) -> list[dict]:
242
+ """读取 CSV 文件(sync 版本,供线程池回退使用)。"""
243
+ try:
244
+ # 尝试 UTF-8(优先,处理 BOM)
245
+ with open(path, "r", encoding="utf-8-sig", newline="") as f:
246
+ reader = csv.DictReader(f)
247
+ return list(reader)
248
+ except UnicodeDecodeError:
249
+ # fallback 到 GBK(中文 Windows 常见编码)
250
+ with open(path, "r", encoding="gbk", newline="") as f:
251
+ reader = csv.DictReader(f)
252
+ return list(reader)
253
+
254
+ async def _read_csv_async(self, path: Path) -> list[dict]:
255
+ """读取 CSV 文件(async 版本,使用 aiofiles)。"""
256
+ try:
257
+ import aiofiles
258
+ except ImportError:
259
+ # aiofiles 未安装时回退到 sync 版本
260
+ return await asyncio.to_thread(self._read_csv, path)
261
+
262
+ async def _read_with_encoding(encoding: str) -> list[dict]:
263
+ async with aiofiles.open(path, "r", encoding=encoding, newline="") as f:
264
+ content = await f.read()
265
+ # csv.DictReader 需要文件对象,从内存字符串构建
266
+ from io import StringIO
267
+ reader = csv.DictReader(StringIO(content))
268
+ return list(reader)
269
+
270
+ try:
271
+ return await _read_with_encoding("utf-8-sig")
272
+ except UnicodeDecodeError:
273
+ return await _read_with_encoding("gbk")
274
+
275
+ def _read_excel(self, path: Path) -> list[dict]:
276
+ """读取 Excel 文件(懒导入 pandas)。"""
277
+ try:
278
+ import pandas as pd
279
+ except ImportError as e:
280
+ raise DataError(
281
+ "pandas is required for Excel support. "
282
+ "Install it with: pip install pandas openpyxl"
283
+ ) from e
284
+
285
+ try:
286
+ df = pd.read_excel(path)
287
+ return df.to_dict("records")
288
+ except Exception as e:
289
+ raise DataError(f"Failed to read Excel {path}: {e}") from e
290
+
291
+ def _time_contract(self, symbol: str) -> dict:
292
+ """Load one explicit source declaration from configuration or sidecar."""
293
+ configured = self.time_contracts.get(symbol)
294
+ safe_symbol = symbol.replace("/", "_").replace("\\", "_")
295
+ path = self.data_dir / f"{safe_symbol}.time.json"
296
+ sidecar = None
297
+ if path.exists():
298
+ try:
299
+ sidecar = json.loads(path.read_text(encoding="utf-8"))
300
+ except (ValueError, OSError) as exc:
301
+ raise DataError(f"Cannot read source time contract {path}: {exc}", details={"time_contract": True}) from exc
302
+ if configured is not None and sidecar is not None and configured != sidecar:
303
+ raise DataError("Configured time contract contradicts the symbol sidecar.", details={"time_contract": True})
304
+ contract = configured if configured is not None else sidecar
305
+ if contract is None:
306
+ return {}
307
+ return self._validate_time_contract(contract)
308
+
309
+ @staticmethod
310
+ def _validate_time_contract(contract: dict) -> dict:
311
+ """Validate source timezone, semantics, and calendar consistency."""
312
+ if not isinstance(contract, dict) or set(contract) - {"timezone", "timestamp_semantics", "calendar_id"}:
313
+ raise DataError("Local time contract accepts timezone, timestamp_semantics, and calendar_id.", details={"time_contract": True})
314
+ zone = contract.get("timezone")
315
+ if zone is not None:
316
+ get_timezone(zone)
317
+ semantics = contract.get("timestamp_semantics")
318
+ if semantics is not None and semantics not in {"open", "close", "trading_date"}:
319
+ raise DataError("Invalid local timestamp_semantics.", details={"time_contract": True})
320
+ calendar_id = contract.get("calendar_id")
321
+ if calendar_id is not None:
322
+ calendar_zone = str(exchange_calendar(calendar_id).tz)
323
+ if zone is not None and timezone_metadata(zone)["tzif_sha256"] != timezone_metadata(calendar_zone)["tzif_sha256"]:
324
+ raise DataError("Local source timezone contradicts the declared session calendar.", details={"time_contract": True})
325
+ return dict(contract)
326
+
327
+ def _rows_to_bars(self, rows: list[dict], symbol: str, interval: str = "1d", contract: dict | None = None) -> list[Bar]:
328
+ """Resolve source timestamps and reject malformed rows explicitly."""
329
+ if not rows:
330
+ return []
331
+ contract = self._validate_time_contract(contract) if contract is not None else self._time_contract(symbol)
332
+ mapping = _normalize_columns(list(rows[0]))
333
+ missing = {"open", "high", "low", "close", "volume"} - set(mapping)
334
+ if missing:
335
+ raise DataError(f"Missing local OHLCV columns: {sorted(missing)}.")
336
+ explicit_bounds = "open_time" in mapping and "close_time" in mapping
337
+ if ("open_time" in mapping) != ("close_time" in mapping):
338
+ raise DataError("Local data must provide both open_time and close_time.", details={"time_contract": True})
339
+ if not explicit_bounds and "datetime" not in mapping and "session_date" not in mapping:
340
+ raise DataError("Local data requires explicit bounds, a datetime, or a declared trading date.", details={"time_contract": True})
341
+ bars = []
342
+ for number, row in enumerate(rows, 1):
343
+ try:
344
+ zone = contract.get("timezone")
345
+ semantics = contract.get("timestamp_semantics")
346
+ calendar_id = contract.get("calendar_id")
347
+ session = _parse_session_date(row[mapping["session_date"]]) if "session_date" in mapping else None
348
+ original = None
349
+ if explicit_bounds:
350
+ if semantics == "trading_date" and session is None:
351
+ raise DataError("trading_date semantics requires a session_date label.", details={"time_contract": True})
352
+ original = _parse_datetime(row[mapping["close_time"]])
353
+ begin = normalize_utc(_parse_datetime(row[mapping["open_time"]]), source_timezone=zone)
354
+ finish = normalize_utc(original, source_timezone=zone)
355
+ semantics = semantics or "close"
356
+ elif session is not None and "datetime" not in mapping:
357
+ if semantics != "trading_date" or calendar_id is None:
358
+ raise DataError("Date-only data requires timestamp_semantics=trading_date and an explicit calendar_id.", details={"time_contract": True})
359
+ begin, finish = session_bounds(calendar_id, session)
360
+ zone = zone or str(exchange_calendar(calendar_id).tz)
361
+ else:
362
+ raw = row[mapping["datetime"]]
363
+ is_date_only = isinstance(raw, date) and not isinstance(raw, datetime)
364
+ is_date_only |= isinstance(raw, str) and re.fullmatch(r"\d{4}[-/]\d{2}[-/]\d{2}|\d{8}", raw.strip()) is not None
365
+ if is_date_only or semantics == "trading_date":
366
+ if semantics != "trading_date" or calendar_id is None:
367
+ raise DataError("Date-only datetime requires trading_date semantics and an explicit calendar_id.", details={"time_contract": True})
368
+ raw_session = _parse_session_date(raw)
369
+ if session is not None and session != raw_session:
370
+ raise DataError("Trading date columns contradict one another.", details={"time_contract": True})
371
+ session = raw_session
372
+ begin, finish = session_bounds(calendar_id, session)
373
+ zone = zone or str(exchange_calendar(calendar_id).tz)
374
+ else:
375
+ if semantics not in {"open", "close"}:
376
+ raise DataError("One datetime column requires explicit open/close timestamp_semantics.", details={"time_contract": True})
377
+ original = _parse_datetime(raw)
378
+ instant = normalize_utc(original, source_timezone=zone)
379
+ width = _interval_delta(interval)
380
+ begin, finish = (instant, instant + width) if semantics == "open" else (instant - width, instant)
381
+ source_offset = None
382
+ if original is not None and original.tzinfo is not None and original.utcoffset() is not None:
383
+ source_offset = int(original.utcoffset().total_seconds())
384
+ zone = zone or getattr(original.tzinfo, "key", None)
385
+ if zone is None and source_offset == 0:
386
+ zone = "UTC"
387
+ elif zone is not None:
388
+ source_instant = begin if semantics == "open" else finish
389
+ source_offset = int(source_instant.astimezone(get_timezone(zone)).utcoffset().total_seconds())
390
+ bars.append(Bar(
391
+ symbol=symbol, datetime=finish, open_time=begin, session_date=session,
392
+ source_timezone=zone, source_utc_offset_seconds=source_offset,
393
+ source_timestamp_semantics=semantics, calendar_id=calendar_id,
394
+ open=_parse_float(row[mapping["open"]]), high=_parse_float(row[mapping["high"]]),
395
+ low=_parse_float(row[mapping["low"]]), close=_parse_float(row[mapping["close"]]),
396
+ volume=_parse_float(row[mapping["volume"]]),
397
+ open_interest=_parse_float(row.get(mapping.get("open_interest", "")), optional=True),
398
+ ))
399
+ except Exception as exc:
400
+ details = dict(exc.details) if isinstance(exc, DataError) else {}
401
+ details["time_contract"] = True
402
+ raise DataError(f"Invalid local bar row {number} for {symbol}: {exc}", details=details) from exc
403
+ return sorted(bars, key=lambda bar: bar.datetime)
404
+
405
+ def _filter_by_date(self, bars: list[Bar], start: str, end: str) -> list[Bar]:
406
+ """Filter inclusive dates according to the original source contract."""
407
+ return filter_bar_dates(bars, start, end)
@@ -0,0 +1,123 @@
1
+ """Generate interval-aligned synthetic OHLCV bars for strategy validation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from datetime import datetime, timedelta, timezone
6
+ from decimal import Decimal
7
+ import math
8
+
9
+ from nlalpha.core.contracts.market import Bar
10
+
11
+ from nlalpha_data.base import _interval_delta
12
+
13
+
14
+ class MockDataProvider:
15
+ """Generate mock bars for each configured universe symbol."""
16
+
17
+ def __init__(self, envelope, instruments=None):
18
+ self._envelope = envelope
19
+ self._bars: list[Bar] | None = None
20
+ universe = envelope.universe if hasattr(envelope, "universe") else {}
21
+ self._symbols = list(universe.get("symbols", []))
22
+ if not self._symbols:
23
+ self._symbols = [leg.get("symbol") for leg in universe.get("legs", [])]
24
+ if (
25
+ not self._symbols
26
+ or any(
27
+ not isinstance(symbol, str) or not symbol or symbol != symbol.strip()
28
+ for symbol in self._symbols
29
+ )
30
+ or len(set(self._symbols)) != len(self._symbols)
31
+ ):
32
+ raise ValueError("mock validation requires unique configured universe symbols")
33
+ data = envelope.data if hasattr(envelope, "data") else {}
34
+ execution = envelope.execution if hasattr(envelope, "execution") else {}
35
+ self._interval = _interval_delta(data.get("interval") or execution.get("bar_interval"))
36
+ contract = getattr(envelope, "contract", {}) or {}
37
+ self._instruments = instruments or contract.get("instruments", {})
38
+ if not hasattr(self._instruments, "get"):
39
+ raise ValueError("mock instrument settings must be a symbol mapping")
40
+
41
+ @staticmethod
42
+ def _common_increment(*values: object) -> Decimal:
43
+ """Return an increment that is a multiple of all configured steps."""
44
+ increments = [Decimal(str(value)) for value in values if value is not None]
45
+ increments = [value for value in increments if value > 0]
46
+ if not increments:
47
+ return Decimal(1)
48
+ scale = max(max(0, -value.as_tuple().exponent) for value in increments)
49
+ scaled = [int(value.scaleb(scale)) for value in increments]
50
+ common = math.lcm(*scaled)
51
+ return Decimal(common).scaleb(-scale)
52
+
53
+ def _instrument_steps(self, symbol: str) -> tuple[Decimal, Decimal]:
54
+ instrument = self._instruments.get(symbol, {})
55
+ if not hasattr(instrument, "get"):
56
+ raise ValueError(f"mock instrument settings for {symbol!r} must be a mapping")
57
+ kind = instrument.get("instrument_type")
58
+ default_price_precision = 5 if kind == "currency_pair" else 2
59
+ price_precision = instrument.get("price_precision", default_price_precision)
60
+ if isinstance(price_precision, bool) or not isinstance(price_precision, int) or not 0 <= price_precision <= 12:
61
+ raise ValueError(f"mock price_precision for {symbol!r} must be an integer in [0, 12]")
62
+ price_step = self._common_increment(
63
+ instrument.get("price_increment"), Decimal(1).scaleb(-price_precision)
64
+ )
65
+
66
+ default_size_precision = 0 if kind in {"equity", "futures", None} else 8
67
+ size_precision = instrument.get("size_precision", default_size_precision)
68
+ if isinstance(size_precision, bool) or not isinstance(size_precision, int) or not 0 <= size_precision <= 12:
69
+ raise ValueError(f"mock size_precision for {symbol!r} must be an integer in [0, 12]")
70
+ configured_size_increment = instrument.get("size_increment")
71
+ if configured_size_increment is None:
72
+ configured_size_increment = Decimal(1).scaleb(-size_precision)
73
+ volume_step = self._common_increment(
74
+ configured_size_increment,
75
+ instrument.get("lot_size"),
76
+ Decimal(1).scaleb(-size_precision),
77
+ )
78
+ return price_step, volume_step
79
+
80
+ def is_available(self) -> bool:
81
+ return True
82
+
83
+ async def load_bars(self, symbol: str, interval: str, start: str, end: str) -> list[Bar]:
84
+ """Return all generated bars or the requested symbol's bars."""
85
+ if self._bars is None:
86
+ self._bars = self._generate()
87
+ if symbol:
88
+ if symbol not in self._symbols:
89
+ raise ValueError(f"mock data does not include symbol {symbol!r}")
90
+ return [bar for bar in self._bars if bar.symbol == symbol]
91
+ return list(self._bars)
92
+
93
+ def _generate(self) -> list[Bar]:
94
+ """Generate deterministic bars using each instrument's increments."""
95
+ flat_bars: list[Bar] = []
96
+ base_price = 100.0
97
+ dt = datetime(2023, 1, 1, 9, 30, tzinfo=timezone.utc) + self._interval
98
+
99
+ for seed, symbol in enumerate(self._symbols):
100
+ price_step, volume_step = self._instrument_steps(symbol)
101
+ base_ticks = max(1000, int(Decimal(str(base_price + seed * 10)) / price_step))
102
+ for i in range(200):
103
+ close_ticks = base_ticks + (i % 20)
104
+ close = price_step * close_ticks
105
+ open_p = price_step * (close_ticks - 1)
106
+ high = price_step * (close_ticks + 1)
107
+ low = price_step * (close_ticks - 2)
108
+ flat_bars.append(Bar(
109
+ symbol=symbol,
110
+ datetime=dt + self._interval * i,
111
+ open_time=dt + self._interval * (i - 1),
112
+ source_timezone="UTC",
113
+ source_timestamp_semantics="close",
114
+ open=float(open_p),
115
+ high=float(high),
116
+ low=float(low),
117
+ close=float(close),
118
+ volume=float(volume_step * (1000 + i)),
119
+ ))
120
+
121
+ order = {symbol: index for index, symbol in enumerate(self._symbols)}
122
+ flat_bars.sort(key=lambda bar: (bar.datetime, order[bar.symbol]))
123
+ return flat_bars
@@ -0,0 +1,134 @@
1
+ """Store canonical bars atomically with exact query and source metadata."""
2
+
3
+ import asyncio
4
+ import json
5
+ import os
6
+ from pathlib import Path
7
+ import tempfile
8
+
9
+ from nlalpha.core.contracts.market import Bar
10
+ from nlalpha.core.errors import DataError
11
+ from nlalpha_data.store import (
12
+ BaseBarStore, BarOverview, _cache_path, _bars_to_table, _table_to_bars,
13
+ BAR_CACHE_NAMESPACE, BAR_CONTRACT_KEY, BAR_CONTRACT, SOURCE_KEY, QUERY_KEY, read_source_contract,
14
+ )
15
+
16
+
17
+ class ParquetBarStore(BaseBarStore):
18
+ """Persist exact bar queries inside one source-specific cache directory."""
19
+
20
+ def __init__(self, cache_dir: str | Path = "workspace/cache", *, source: dict | None = None):
21
+ self.cache_dir = Path(cache_dir) / BAR_CACHE_NAMESPACE
22
+ self.source = json.loads(json.dumps(source)) if source is not None else None
23
+
24
+ async def save(self, bars: list[Bar], symbol: str = "", interval: str = "", start: str = "", end: str = "") -> int:
25
+ """Publish a complete Parquet file with an atomic same-directory rename."""
26
+ if not bars:
27
+ return 0
28
+ return await asyncio.to_thread(self._save, bars, symbol, interval, start, end)
29
+
30
+ def _save(self, bars, symbol, interval, start, end):
31
+ import pyarrow.parquet as pq
32
+ if not symbol or not interval:
33
+ raise DataError("Cache writes require explicit symbol and interval.")
34
+ if any(bar.symbol != symbol for bar in bars):
35
+ raise DataError("Cache bars must match the requested symbol.", details={"cache_contract": True})
36
+ query = [symbol, interval, start, end]
37
+ table = _bars_to_table(sorted(bars, key=lambda bar: bar.datetime))
38
+ metadata = {**(table.schema.metadata or {}), BAR_CONTRACT_KEY: BAR_CONTRACT,
39
+ QUERY_KEY: json.dumps(query).encode()}
40
+ if self.source is not None:
41
+ contract = {"schema": "nlalpha.market.source.v1", "assets": {symbol: {**self.source, "cache_hit": False}}}
42
+ metadata[SOURCE_KEY] = json.dumps(contract, sort_keys=True).encode()
43
+ read_source_contract(table.replace_schema_metadata(metadata))
44
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
45
+ target = _cache_path(*query, self.cache_dir)
46
+ descriptor, filename = tempfile.mkstemp(prefix=".bars-", suffix=".tmp", dir=self.cache_dir)
47
+ try:
48
+ with os.fdopen(descriptor, "wb") as output:
49
+ pq.write_table(table.replace_schema_metadata(metadata), output)
50
+ output.flush()
51
+ os.fsync(output.fileno())
52
+ os.replace(filename, target)
53
+ finally:
54
+ Path(filename).unlink(missing_ok=True)
55
+ return len(bars)
56
+
57
+ def _read(self, path: Path, expected_query=None):
58
+ import pyarrow.parquet as pq
59
+ try:
60
+ table = pq.read_table(path)
61
+ except FileNotFoundError:
62
+ raise
63
+ except Exception as exc:
64
+ raise DataError(f"Cannot read Parquet cache: {exc}", details={"cache_contract": True}) from exc
65
+ bars = _cache_table_to_bars(table)
66
+ try:
67
+ metadata = table.schema.metadata or {}
68
+ query = json.loads(metadata[QUERY_KEY])
69
+ if (not isinstance(query, list) or len(query) != 4
70
+ or any(not isinstance(value, str) for value in query) or not query[0] or not query[1]):
71
+ raise ValueError("Invalid cached query")
72
+ contract = read_source_contract(table)
73
+ entry = None if contract is None else contract["assets"][query[0]]
74
+ source = None if entry is None else {key: value for key, value in entry.items() if key != "cache_hit"}
75
+ if path.name != _cache_path(*query, self.cache_dir).name:
76
+ raise ValueError("Cache filename contradicts query metadata")
77
+ if expected_query is not None and query != list(expected_query):
78
+ raise ValueError("Cache query mismatch")
79
+ if any(bar.symbol != query[0] for bar in bars):
80
+ raise ValueError("Cache bars contradict query symbol")
81
+ if source != self.source:
82
+ raise ValueError("Cache source declaration mismatch")
83
+ except (KeyError, ValueError, TypeError) as exc:
84
+ raise DataError(f"Invalid Parquet cache metadata: {exc}", details={"cache_contract": True}) from exc
85
+ return bars, query
86
+
87
+ async def load(self, symbol: str, interval: str, start: str, end: str) -> list[Bar]:
88
+ """Read an exact query; only a missing file represents a cache miss."""
89
+ query = (symbol, interval, start, end)
90
+ bars, _ = await asyncio.to_thread(self._read, _cache_path(*query, self.cache_dir), query)
91
+ return bars
92
+
93
+ async def delete(self, symbol: str, interval: str) -> int:
94
+ """Delete matching exact-query files using their original metadata."""
95
+ return await asyncio.to_thread(self._delete, symbol, interval)
96
+
97
+ def _delete(self, symbol, interval):
98
+ deleted = 0
99
+ for path in self.cache_dir.glob("*.parquet"):
100
+ _, query = self._read(path)
101
+ if query[:2] == [symbol, interval]:
102
+ path.unlink()
103
+ deleted += 1
104
+ return deleted
105
+
106
+ async def get_overview(self) -> list[BarOverview]:
107
+ """Read all stored series without guessing original symbols from filenames."""
108
+ return await asyncio.to_thread(self._overview)
109
+
110
+ def _overview(self):
111
+ overviews = {}
112
+ for path in self.cache_dir.glob("*.parquet"):
113
+ bars, query = self._read(path)
114
+ if not bars:
115
+ continue
116
+ key = tuple(query[:2])
117
+ previous = overviews.get(key)
118
+ if previous is None:
119
+ overviews[key] = BarOverview(*key, count=len(bars), start=bars[0].datetime, end=bars[-1].datetime)
120
+ else:
121
+ previous.count += len(bars)
122
+ previous.start = min(previous.start, bars[0].datetime)
123
+ previous.end = max(previous.end, bars[-1].datetime)
124
+ return list(overviews.values())
125
+
126
+ async def aclose(self) -> None:
127
+ """Release the file store's resource scope."""
128
+
129
+
130
+ def _cache_table_to_bars(table, fallback_symbol: str = "") -> list[Bar]:
131
+ """Validate cached volume semantics before restoring canonical bars."""
132
+ if (table.schema.metadata or {}).get(BAR_CONTRACT_KEY) != BAR_CONTRACT:
133
+ raise DataError("Parquet cache requires the current instrument-quantity bar contract.", details={"cache_contract": True})
134
+ return _table_to_bars(table, fallback_symbol=fallback_symbol)