powerquant-cli 0.2.6__py3-none-any.whl

quant_cli/app/data/service/bar_cache_service.py

@@ -0,0 +1,287 @@
+"""按已下载范围复用行情，内容校验后发布固定快照；不推断行情质量。"""
+
+import hashlib
+import json
+from collections import defaultdict
+from collections.abc import AsyncIterator
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Protocol
+
+import polars as pl
+from pqdata.schema.bar_content_schema import BarContent, bar_content_digest
+from pqdata.schema.bar_schema import BarPage, BarQuery, BarResponse
+from pydantic import BaseModel, ConfigDict, Field
+
+from quant_cli.app.data.repository.parquet_repository import (
+    atomic_json,
+    cache_writer,
+    read_parquet,
+    write_parquet,
+)
+from quant_cli.app.data.schema.parquet_file_schema import ParquetFile
+
+BAR_COLUMNS = {
+    "instrument_id": pl.String,
+    "timestamp": pl.String,
+    "frequency": pl.String,
+    "open": pl.String,
+    "high": pl.String,
+    "low": pl.String,
+    "close": pl.String,
+    "volume": pl.Int64,
+    "amount": pl.String,
+    "previous_close": pl.String,
+    "suspended": pl.Boolean,
+}
+RANGE_STEP = timedelta(microseconds=1)
+MAX_BLOCK_DAYS = 31
+MAX_BLOCK_INSTRUMENTS = 100
+
+
+class BarSource(Protocol):
+    """缓存只消费 SDK 已提供的查询能力。"""
+
+    @property
+    def source_id(self) -> str: ...
+
+    async def describe_bars(self, query: BarQuery) -> BarContent: ...
+
+    def iter_bar_pages(self, query: BarQuery) -> AsyncIterator[BarPage]: ...
+
+
+class BarBlock(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    query: BarQuery
+    content: BarContent
+    file: ParquetFile
+
+
+class BarCacheQuery(BarQuery):
+    """本地全市场需求；网络请求由准备器切为有界批次。"""
+
+    instrument_ids: list[str] = Field(min_length=1, max_length=10000)
+
+
+class BarCacheSnapshot(BaseModel):
+    """查询条件与文件集合一起固定；完整下载不代表市场事实没有缺口。"""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    format_version: int = 2
+    source: str
+    query: BarCacheQuery
+    blocks: tuple[BarBlock, ...]
+
+
+def bars_frame(rows: list[BarResponse]) -> pl.DataFrame:
+    """Decimal 保存为精确十进制文本，SDK 读回后仍为 Decimal。"""
+    return pl.DataFrame([row.model_dump(mode="json") for row in rows], schema=BAR_COLUMNS)
+
+
+def missing_ranges(
+    start: datetime, end: datetime, covered: list[tuple[datetime, datetime]]
+) -> list[tuple[datetime, datetime]]:
+    """计算闭区间差集；空数据页仍提供已查询范围证据。"""
+    missing = []
+    position = start
+    for left, right in sorted(covered):
+        if right < position or left > end:
+            continue
+        if left > position:
+            missing.append((position, min(end, left - RANGE_STEP)))
+        if right >= end:
+            return missing
+        position = max(position, right + RANGE_STEP)
+    if position <= end:
+        missing.append((position, end))
+    return missing
+
+
+def _overlaps(block: BarBlock, query: BarQuery) -> bool:
+    return (
+        block.query.frequency == query.frequency
+        and bool(set(block.query.instrument_ids) & set(query.instrument_ids))
+        and block.query.start <= query.end
+        and block.query.end >= query.start
+    )
+
+
+async def _download(source: BarSource, query: BarQuery, directory: Path, max_rows: int) -> BarBlock:
+    rows: list[BarResponse] = []
+    revision = query.revision
+    async for page in source.iter_bar_pages(query):
+        if revision is not None and page.revision != revision:
+            raise ValueError("行情下载期间修订变化")
+        revision = page.revision
+        rows.extend(page.items)
+        if len(rows) > max_rows:
+            raise ValueError("行情下载超过配置行数上限")
+    if revision is None:
+        raise ValueError("行情下载没有返回版本")
+    content = BarContent(revision=revision, row_count=len(rows), sha256=bar_content_digest(rows))
+    file = write_parquet(
+        directory,
+        bars_frame(rows),
+        dataset_id="bar.raw",
+        primary_key=("timestamp", "instrument_id", "frequency"),
+    )
+    return BarBlock(
+        query=query.model_copy(update={"revision": revision}), content=content, file=file
+    )
+
+
+async def prepare_bar_cache(
+    source: BarSource, query: BarQuery, cache_dir: Path, max_rows: int
+) -> Path:
+    """在线准备；已有块只检查摘要，新增范围批量下载，最后原子发布索引。"""
+    if query.cursor is not None or max_rows <= 0:
+        raise ValueError("缓存必须从完整范围开始且容量为正数")
+    directory = cache_dir / source.source_id / "bars-v2"
+    with cache_writer(directory):
+        index = directory / "current.json"
+        previous: list[BarBlock] = []
+        if index.exists():
+            previous = list(_read_manifest(index).blocks)
+        selected = [block for block in previous if _overlaps(block, query)]
+        retained = [block for block in previous if not _overlaps(block, query)]
+        probe = BarQuery.model_validate(
+            query.model_dump() | {"instrument_ids": query.instrument_ids[:1], "end": query.start}
+        )
+        baseline = await source.describe_bars(probe)
+        revision = baseline.revision
+        updated: list[BarBlock] = []
+        for block in selected:
+            scope = block.query.model_copy(update={"revision": revision})
+            content = (
+                block.content
+                if block.content.revision == revision
+                else await source.describe_bars(scope)
+            )
+            if content.revision != revision:
+                raise ValueError("缓存更新期间行情版本变化")
+            if (
+                content.sha256 == block.content.sha256
+                and content.row_count == block.content.row_count
+            ):
+                read_parquet(directory, block.file, max_rows)
+                updated.append(block.model_copy(update={"query": scope, "content": content}))
+            else:
+                replacement = await _download(source, scope, directory, max_rows)
+                if replacement.content != content:
+                    raise ValueError("下载内容与范围凭据不一致")
+                updated.append(replacement)
+        requests: dict[tuple[datetime, datetime], list[str]] = defaultdict(list)
+        for instrument_id in query.instrument_ids:
+            covered = [
+                (block.query.start, block.query.end)
+                for block in updated
+                if instrument_id in block.query.instrument_ids
+            ]
+            for left, right in missing_ranges(query.start, query.end, covered):
+                while left <= right:
+                    boundary = min(right, left + timedelta(days=MAX_BLOCK_DAYS) - RANGE_STEP)
+                    requests[(left, boundary)].append(instrument_id)
+                    left = boundary + RANGE_STEP
+        for (left, right), instruments in requests.items():
+            for offset in range(0, len(instruments), MAX_BLOCK_INSTRUMENTS):
+                scope = BarQuery(
+                    instrument_ids=instruments[offset : offset + MAX_BLOCK_INSTRUMENTS],
+                    frequency=query.frequency,
+                    start=left,
+                    end=right,
+                    revision=revision,
+                )
+                block = await _download(source, scope, directory, max_rows)
+                revision = block.content.revision
+                updated.append(block)
+        # 固定版本的最终检查，拒绝将多次请求期间的并发修订发布为完整快照。
+        fixed = query.model_copy(update={"revision": revision})
+        probe = BarQuery.model_validate(
+            fixed.model_dump() | {"instrument_ids": query.instrument_ids[:1], "end": query.start}
+        )
+        final_content = await source.describe_bars(probe)
+        if final_content.revision != revision:
+            raise ValueError("缓存发布前行情版本变化")
+        snapshot = BarCacheSnapshot(
+            source=source.source_id,
+            query=BarCacheQuery.model_validate(fixed.model_dump()),
+            blocks=tuple(updated),
+        )
+        read_bar_snapshot(directory, snapshot, max_rows)
+        payload = snapshot.model_dump(mode="json")
+        digest = hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+        manifest_path = directory / f"{digest}.json"
+        atomic_json(manifest_path, {**payload, "content_digest": digest})
+        catalog = snapshot.model_copy(update={"blocks": tuple(retained + updated)})
+        catalog_payload = catalog.model_dump(mode="json")
+        catalog_digest = hashlib.sha256(
+            json.dumps(catalog_payload, sort_keys=True).encode()
+        ).hexdigest()
+        atomic_json(index, {**catalog_payload, "content_digest": catalog_digest})
+        return manifest_path
+
+
+def _read_manifest(path: Path) -> BarCacheSnapshot:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    digest = payload.pop("content_digest", None)
+    if digest != hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest():
+        raise ValueError("缓存清单摘要不匹配")
+    snapshot = BarCacheSnapshot.model_validate(payload)
+    if snapshot.format_version != 2:
+        raise ValueError("旧缓存需要重新准备")
+    return snapshot
+
+
+def load_bar_snapshot(path: Path, max_rows: int) -> tuple[BarCacheSnapshot, list[BarResponse]]:
+    """显式离线入口，固定清单，完全不访问客户端。"""
+    snapshot = _read_manifest(path)
+    return snapshot, read_bar_snapshot(path.parent, snapshot, max_rows)
+
+
+def read_bar_snapshot(
+    directory: Path, snapshot: BarCacheSnapshot, max_rows: int
+) -> list[BarResponse]:
+    """校验下载覆盖及每个文件，返回请求内行；不把无行当作质量通过。"""
+    query = snapshot.query
+    for instrument in query.instrument_ids:
+        covered = [
+            (block.query.start, block.query.end)
+            for block in snapshot.blocks
+            if instrument in block.query.instrument_ids and block.query.frequency == query.frequency
+        ]
+        if missing_ranges(query.start, query.end, covered):
+            raise ValueError(f"缓存缺少请求范围: {instrument}")
+    rows: dict[tuple[datetime, str], BarResponse] = {}
+    for block in snapshot.blocks:
+        if block.content.revision != query.revision or block.query.revision != query.revision:
+            raise ValueError("缓存文件修订与快照不一致")
+        if block.file.dataset_id != "bar.raw" or block.file.schema_version != 1:
+            raise ValueError("不支持的行情文件类型")
+        frame = read_parquet(directory, block.file, max_rows)
+        all_rows = sorted(
+            [BarResponse.model_validate(row) for row in frame.to_dicts()],
+            key=lambda row: (row.timestamp, row.instrument_id),
+        )
+        if (
+            len(all_rows) != block.content.row_count
+            or bar_content_digest(all_rows) != block.content.sha256
+        ):
+            raise ValueError("行情内容与来源凭据不一致")
+        for row in all_rows:
+            if (
+                row.instrument_id not in block.query.instrument_ids
+                or row.frequency != block.query.frequency
+                or not block.query.start <= row.timestamp <= block.query.end
+            ):
+                raise ValueError("行情行不属于清单查询范围")
+            if (
+                row.instrument_id in query.instrument_ids
+                and query.start <= row.timestamp <= query.end
+            ):
+                key = (row.timestamp, row.instrument_id)
+                if key in rows and rows[key] != row:
+                    raise ValueError("缓存范围重叠且内容冲突")
+                rows[key] = row
+                if len(rows) > max_rows:
+                    raise ValueError("缓存读取超过配置行数上限")
+    return [rows[key] for key in sorted(rows)]

quant_cli/app/data/service/dataset_audit_service.py

@@ -0,0 +1,137 @@
+"""只读检查日线联合快照；缺失事实不会被推断成正常交易或零事件。"""
+
+from datetime import datetime, time
+from zoneinfo import ZoneInfo
+
+from quant_core.app.data.schema.reliability_schema import DatasetHealth
+from quant_core.app.data.service.calendar_coverage_service import build_calendar_coverage
+from quant_core.app.data.service.prepared_data_verification_service import PreparedDataVerification
+from quant_core.app.data.service.prepared_market_service import PreparedMarketData
+
+from quant_cli.app.data.schema.dataset_audit_schema import (
+    DatasetAuditReport,
+    DatasetAuditRequest,
+    DatasetGap,
+)
+
+_CN = ZoneInfo("Asia/Shanghai")
+
+
+async def audit_dataset(
+    market: PreparedMarketData,
+    verification: PreparedDataVerification,
+    request: DatasetAuditRequest,
+) -> DatasetAuditReport:
+    """检查显式股票池与日期范围，预热日期须由调用方纳入请求。"""
+    report = DatasetAuditReport(revision=market.revision, request=request)
+
+    def gap(instrument: str, dataset: str, reason: str, days: tuple = ()) -> None:
+        report.gaps.append(
+            DatasetGap(
+                instrument_id=instrument,
+                dataset=dataset,
+                reason=reason,
+                count=len(days) or 1,
+                dates=days[:100],
+            )
+        )
+
+    if (
+        request.start < market.start.astimezone(_CN).date()
+        or request.end > market.end.astimezone(_CN).date()
+    ):
+        gap("*", "snapshot", "requested_range_outside_snapshot")
+        return report
+    for instrument in request.instrument_ids:
+        facts = market.facts.get(instrument)
+        if facts is None:
+            gap(instrument, "history_facts", "missing_instrument")
+            continue
+        if facts.instrument.listed_date is None:
+            gap(instrument, "instrument", "unknown_listing_date")
+            continue
+        first = max(request.start, facts.instrument.listed_date)
+        last = min(request.end, facts.instrument.delisted_date or request.end)
+        if first > last:
+            gap(instrument, "instrument", "outside_listing_period")
+            continue
+        calendar = build_calendar_coverage(
+            facts.instrument.exchange,
+            first,
+            last,
+            [row for row in facts.calendar if first <= row.trade_date <= last],
+        )
+        if not calendar.complete:
+            gap(instrument, "calendar", "missing_calendar_days", calendar.missing_days)
+        days = calendar.open_day_set
+        if calendar.complete and not days:
+            gap(instrument, "calendar", "no_trading_sessions")
+        states = {row.trade_date: row for row in facts.daily_facts}
+        gap_days = tuple(sorted(days - states.keys()))
+        if gap_days:
+            gap(instrument, "daily_fact", "missing_trading_state", gap_days)
+        unknown_days = tuple(
+            sorted(day for day in days & states.keys() if states[day].suspension_status is None)
+        )
+        if unknown_days:
+            gap(instrument, "daily_fact", "unknown_suspension_status", unknown_days)
+        intraday_days = tuple(
+            sorted(
+                day for day in days & states.keys() if states[day].suspension_status == "intraday"
+            )
+        )
+        if intraday_days:
+            gap(instrument, "daily_fact", "intraday_timing_required", intraday_days)
+        if request.require_special_type:
+            missing_special = tuple(
+                sorted(day for day in days if day not in states or states[day].special_type is None)
+            )
+            if missing_special:
+                gap(instrument, "daily_fact", "unknown_special_type", missing_special)
+        rules_missing = tuple(
+            sorted(
+                day
+                for day in days
+                if sum(
+                    rule.effective_from <= day
+                    and (rule.effective_to is None or day <= rule.effective_to)
+                    for rule in facts.trading_rules
+                )
+                != 1
+            )
+        )
+        if rules_missing:
+            gap(instrument, "trading_rule", "missing_or_ambiguous_rule", rules_missing)
+        bars = market.bars.get((instrument, "1d"), [])
+        bar_days = {row.timestamp.astimezone(_CN).date() for row in bars}
+        missing_bars = tuple(
+            sorted(
+                day
+                for day in days - bar_days
+                if day not in states or states[day].suspension_status != "full_day"
+            )
+        )
+        if missing_bars:
+            gap(instrument, "bar", "unexplained_missing_bar", missing_bars)
+        factors = {row.trade_date for row in facts.adjustment_factors if row.factor > 0}
+        missing_factors = tuple(sorted((bar_days & days) - factors))
+        if missing_factors:
+            gap(instrument, "adjustment_factor", "missing_adjustment_factor", missing_factors)
+        start = datetime.combine(first, time(9, 30), _CN)
+        end = datetime.combine(last, time(15), _CN)
+        for dataset, check in (
+            ("bar", verification.verify_bars(instrument, "1d", start, end)),
+            (
+                "adjustment_factor",
+                verification.verify_adjustment_factors(instrument, "1d", start, end),
+            ),
+            ("corporate_action", verification.verify_corporate_actions(instrument, start, end)),
+        ):
+            try:
+                evidence = await check
+            except ValueError:
+                gap(instrument, dataset, "missing_verification_evidence")
+                continue
+            if evidence.health != DatasetHealth.HEALTHY:
+                gap(instrument, dataset, f"verification_{evidence.health.value.lower()}")
+    return report

quant_cli/app/data/service/dataset_load_service.py

@@ -0,0 +1,115 @@
+"""校验本地数据集后离线重载，禁止路径越界和损坏内容进入回放。"""
+
+import hashlib
+import json
+from datetime import datetime
+from pathlib import Path
+
+from pqdata.schema.bar_schema import BarQuery
+from quant_core.app.data.schema.history_schema import HistoryFacts
+from quant_core.app.data.schema.market_schema import BarResponse, InstrumentResponse
+from quant_core.app.data.schema.reliability_schema import DatasetHealthResult
+from quant_core.app.data.service.prepared_data_verification_service import PreparedDataVerification
+from quant_core.app.data.service.prepared_market_service import PreparedMarketData
+
+from quant_cli.app.data.repository.parquet_repository import read_parquet
+from quant_cli.app.data.schema.parquet_file_schema import ParquetFile
+
+
+def manifest_digest(manifest: dict) -> str:
+    """校验元数据和文件清单的完整性；不用于认证不可信发布者。"""
+    payload = {key: value for key, value in manifest.items() if key != "content_digest"}
+    return hashlib.sha256(
+        json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
+    ).hexdigest()
+
+
+def load_dataset(
+    manifest_path: Path, max_rows: int
+) -> tuple[PreparedMarketData, PreparedDataVerification]:
+    """只使用本地文件；调用者指定容量限制，所有 Bar 文件逐字节校验。"""
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    if any(row.get("exchange") in {"BSE", "XBSE"} for row in manifest.get("instruments", [])):
+        raise ValueError("本地快照包含旧北交所标识，请按 BJSE 重新准备数据集")
+    if manifest.get("format_version") != 2:
+        raise ValueError("旧 JSONL 缓存不再支持，请重新准备数据集")
+    if manifest.get("content_digest") != manifest_digest(manifest):
+        raise ValueError("数据集清单校验失败")
+    if any(
+        "suspension_status" not in row
+        for facts in manifest.get("facts", {}).values()
+        for row in facts.get("daily_facts", [])
+    ):
+        raise ValueError("旧日级状态契约不再支持，请重新准备数据集；原快照保留")
+    directory = manifest_path.parent.resolve()
+    market = PreparedMarketData(
+        datetime.fromisoformat(manifest["start"]),
+        datetime.fromisoformat(manifest["end"]),
+        manifest["revision"],
+    )
+    market.instruments = [InstrumentResponse.model_validate(row) for row in manifest["instruments"]]
+    market.facts = {
+        key: HistoryFacts.model_validate(value) for key, value in manifest["facts"].items()
+    }
+    if any(
+        value.revision != market.revision or key != value.instrument.instrument_id
+        for key, value in market.facts.items()
+    ):
+        raise ValueError("缓存事实版本不一致")
+    total = 0
+    expected = {
+        (instrument_id, frequency)
+        for instrument_id in market.facts
+        for frequency in manifest["frequencies"]
+    }
+    observed: set[tuple[str, str]] = set()
+    for item in manifest["files"]:
+        path = (directory / item["path"]).resolve()
+        if path.parent != directory:
+            raise ValueError("缓存文件路径越界")
+        receipt = ParquetFile.model_validate(
+            {key: value for key, value in item.items() if key != "query"}
+        )
+        query = BarQuery.model_validate(item["query"])
+        if len(query.instrument_ids) != 1:
+            raise ValueError("本地回放文件必须只包含一个标的")
+        key = (query.instrument_ids[0], query.frequency)
+        if key in observed or key not in expected:
+            raise ValueError("缓存文件重复或不属于数据集")
+        if query.start != market.start or query.end != market.end:
+            raise ValueError("缓存文件与清单时间范围不一致")
+        observed.add(key)
+        total += receipt.row_count
+        if total > max_rows:
+            raise ValueError("缓存数据超出内存容量配置")
+        if query.revision != market.revision:
+            raise ValueError("缓存行情版本不一致")
+        if receipt.dataset_id != "bar.raw" or receipt.schema_version != 1:
+            raise ValueError("缓存行情类型不支持")
+        frame = read_parquet(directory, receipt, max_rows)
+        rows = [BarResponse.model_validate(row) for row in frame.to_dicts()]
+        if any(
+            row.instrument_id not in query.instrument_ids
+            or row.frequency != query.frequency
+            or not query.start <= row.timestamp <= query.end
+            for row in rows
+        ) or any(
+            right.timestamp <= left.timestamp for left, right in zip(rows, rows[1:], strict=False)
+        ):
+            raise ValueError("缓存行情内容不符合查询范围或排序")
+        market.add_bars(query.instrument_ids[0], query.frequency, rows)
+    if not expected or observed != expected:
+        raise ValueError("缓存缺少标的或频率对应的文件")
+    verification = PreparedDataVerification()
+    for item in manifest["health"]:
+        result = DatasetHealthResult.model_validate(item)
+        frequency = "1d" if result.dataset == "corporate_action" else result.frequency
+        verification.add(
+            result.dataset,
+            result.instrument_id,
+            frequency,
+            result.requested_start,
+            result.requested_end,
+            result,
+        )
+    return market, verification