powerquant-cli 0.2.6__py3-none-any.whl

quant_cli/app/backtest/service/local_backtest_service.py

@@ -0,0 +1,271 @@
+"""本地回测用例：已准备的数据与显式定义进入公共执行链。"""
+
+import logging
+from collections import Counter
+from zoneinfo import ZoneInfo
+
+from quant_core.app.backtest.service.backtest_execution_service import execute_backtest
+from quant_core.app.data.constants import BarFrequency
+from quant_core.app.factor.interface.public.factor_local import LocalFactorStore
+from quant_core.app.strategy.schema.python_strategy_schema import PythonStrategySpec
+from quant_core.app.strategy.strategy_dependency import python_strategy
+from quant_core.common.local_run import LocalRunRecord, RunStatus, content_digest
+
+from quant_cli.app.backtest.local_backtest_schema import LocalBacktestRequest
+from quant_cli.app.data.data_dependency import load_backtest_market
+from quant_cli.app.data.schema.dataset_audit_schema import DatasetAuditRequest
+from quant_cli.app.data.service.dataset_audit_service import audit_dataset
+from quant_cli.app.factor.local_factor_dependency import (
+    build_local_factor_query,
+    decode_factor_artifact,
+    load_factor_result,
+)
+from quant_cli.common.utils.runtime_environment import runtime_versions
+
+
+async def run_local_backtest(
+    request: LocalBacktestRequest, factor_store: LocalFactorStore | None = None
+) -> LocalRunRecord:
+    """返回可归档记录；不创建本地或远端数据库连接。"""
+    record = LocalRunRecord(
+        request=request.model_dump(mode="json"),
+        provenance={
+            "source_digest": content_digest(request.source),
+            "environment": runtime_versions(),
+        },
+    )
+    try:
+        components = load_backtest_market(request.manifest_path, request.max_rows)
+        record.provenance.update(
+            manifest_digest=content_digest(request.manifest_path.read_text(encoding="utf-8")),
+            market_revision=components.query.revision,
+        )
+        if request.config.frequency == BarFrequency.D1 and (
+            request.config.execution_frequency is None
+            or request.config.execution_frequency == BarFrequency.D1
+        ):
+            audit = await audit_dataset(
+                components.query,
+                components.verification,
+                DatasetAuditRequest(
+                    instrument_ids=request.config.instrument_ids,
+                    start=request.config.start.astimezone(ZoneInfo("Asia/Shanghai")).date(),
+                    end=request.config.end.astimezone(ZoneInfo("Asia/Shanghai")).date(),
+                    require_special_type=request.exclude_st,
+                ),
+            )
+            record.provenance["dataset_audit"] = {
+                **audit.model_dump(mode="json"),
+                "ready": audit.ready,
+            }
+            if not audit.ready:
+                summary = "; ".join(
+                    f"{gap.instrument_id}/{gap.dataset}: {gap.reason}" for gap in audit.gaps[:20]
+                )
+                raise ValueError("回测数据检查未通过；请修复数据并重新准备快照：" + summary)
+        if request.publication is not None:
+            spec = request.publication.definition
+            record.provenance["published_strategy"] = {
+                "strategy_id": spec.strategy_id,
+                "version": spec.version,
+                "content_digest": spec.content_digest,
+            }
+            if (
+                spec.factor_refs
+                and request.factor_run_uuid is None
+                and request.factor_artifact_path is None
+            ):
+                from quant_core.app.factor.schema.local_factor_schema import LocalFactorRequest
+
+                from quant_cli.app.factor.local_factor_dependency import compute_local_factor
+
+                previous_days = []
+                for instrument_id in request.config.instrument_ids:
+                    instrument = await components.query.get_instrument(instrument_id)
+                    calendar = await components.query.list_trading_days(
+                        instrument.exchange,
+                        components.query.start.date(),
+                        request.config.start.date(),
+                    )
+                    before_start = [
+                        day.trade_date
+                        for day in calendar
+                        if day.is_open and day.trade_date < request.config.start.date()
+                    ]
+                    if not before_start:
+                        raise ValueError("发布策略缺少首个决策日前一交易日的因子输入")
+                    previous_days.append(max(before_start))
+                factor_request = LocalFactorRequest(
+                    definitions=request.publication.factors,
+                    manifest_path=request.manifest_path,
+                    instrument_ids=request.config.instrument_ids,
+                    start=min(previous_days),
+                    end=request.config.end.date(),
+                    view=request.factor_view,
+                    exclude_st=request.exclude_st,
+                    max_rows=request.max_rows,
+                )
+                factor_run, factor_values = await compute_local_factor(factor_request)
+                if factor_run.status != RunStatus.COMPLETED:
+                    raise ValueError(f"发布策略因子依赖计算失败: {factor_run.error}")
+                if factor_store is None:
+                    raise ValueError("发布策略因子依赖需显式本地归档能力")
+                factor_store.save(factor_run, factor_values)
+                request = request.model_copy(update={"factor_run_uuid": factor_run.uuid})
+                record.request = request.model_dump(mode="json")
+        factors, aliases, revision = None, {}, None
+        if request.factor_run_uuid is not None or request.factor_artifact_path is not None:
+            if request.factor_artifact_path is not None:
+                factor_run, values = decode_factor_artifact(
+                    request.factor_artifact_path.read_text(encoding="utf-8")
+                )
+                if factor_run.status != RunStatus.COMPLETED:
+                    raise ValueError("独立因子产物未成功")
+            else:
+                if factor_store is None or request.factor_run_uuid is None:
+                    raise ValueError("引用因子运行时必须显式提供读取能力")
+                factor_run, values = load_factor_result(factor_store, request.factor_run_uuid)
+            if factor_run.provenance["market_revision"] != components.query.revision:
+                raise ValueError("因子与回测行情修订不同")
+            if request.publication is not None:
+                if (
+                    factor_run.provenance.get("manifest_digest")
+                    != record.provenance["manifest_digest"]
+                ):
+                    raise ValueError("因子与回测输入快照不同")
+                required = {
+                    item.definition.identity.name: item.content_digest
+                    for item in request.publication.factors
+                }
+                if factor_run.provenance.get("published_digests") != required:
+                    raise ValueError("本地因子结果不匹配发布策略的固定依赖")
+                identities = {
+                    item.definition.identity.name: item.definition.identity.model_dump(mode="json")
+                    for item in request.publication.factors
+                }
+                if factor_run.provenance.get("published_definitions") != identities:
+                    raise ValueError("本地因子结果的完整身份不匹配")
+                for field in ("view", "exclude_st", "instrument_ids"):
+                    expected = (
+                        request.factor_view.model_dump(mode="json")
+                        if field == "view"
+                        else request.exclude_st
+                        if field == "exclude_st"
+                        else list(request.config.instrument_ids)
+                    )
+                    if factor_run.request.get(field) != expected:
+                        raise ValueError("本地因子结果的输入政策不匹配")
+                from datetime import date
+
+                if (
+                    date.fromisoformat(factor_run.request["start"]) >= request.config.start.date()
+                    or date.fromisoformat(factor_run.request["end"]) < request.config.end.date()
+                ):
+                    raise ValueError("本地因子结果未覆盖回测窗口及首日前一交易日")
+                for instrument_id in request.config.instrument_ids:
+                    instrument = await components.query.get_instrument(instrument_id)
+                    calendar = await components.query.list_trading_days(
+                        instrument.exchange,
+                        components.query.start.date(),
+                        request.config.end.date(),
+                    )
+                    before_start = [
+                        day.trade_date
+                        for day in calendar
+                        if day.is_open and day.trade_date < request.config.start.date()
+                    ]
+                    if not before_start:
+                        raise ValueError("首日前一交易日缺少输入")
+                    needed = {max(before_start)} | {
+                        day.trade_date
+                        for day in calendar
+                        if day.is_open
+                        and request.config.start.date()
+                        <= day.trade_date
+                        < request.config.end.date()
+                    }
+                    for name in required:
+                        present = {
+                            ts.date() for ts, key in values.get(name, {}) if key == instrument_id
+                        }
+                        if needed - present:
+                            raise ValueError("本地因子结果缺少必要交易日单元，不能静默跳过")
+            factors, aliases = await build_local_factor_query(
+                factor_run, values, components.query, components.query.end.date()
+            )
+            revision = str(factor_run.uuid)
+            record.provenance["factor_run_uuid"] = revision
+        refs = tuple(aliases.values())
+        if request.publication is not None:
+            spec = request.publication.definition
+            refs = spec.factor_refs
+            aliases = spec.resolved_factors()
+            record.provenance["source_digest"] = spec.source_digest
+        else:
+            spec = PythonStrategySpec(
+                strategy_id=request.config.strategy_id,
+                version="local",
+                name=request.config.strategy_id,
+                python_source=request.source,
+                universe=request.config.instrument_ids,
+                parameters=request.parameters,
+                factor_refs=refs,
+                factor_aliases=aliases,
+            )
+        result = await execute_backtest(
+            request.config,
+            python_strategy(spec, request.config.instrument_ids),
+            components,
+            slippage_bps=request.slippage_bps,
+            factors=factors,
+            factor_refs=refs,
+            factor_aliases=aliases,
+            factor_revision=revision,
+        )
+        record.result = result.model_dump(mode="json")
+        reasons = Counter(
+            attempt.defer_reason.value
+            for attempt in result.execution_attempts
+            if attempt.defer_reason is not None
+        )
+        record.result["execution_diagnostics"] = {
+            "fill_count": len(result.fills),
+            "deferred_reasons": dict(sorted(reasons.items())),
+            "zero_fills_with_deferred": not result.fills and bool(reasons),
+            "message": "无成交且存在延后原因，请检查撮合条件与数据事实"
+            if not result.fills and reasons
+            else None,
+        }
+    except Exception as exc:
+        logging.getLogger(__name__).exception("本地计算失败")
+        record.status, record.error = RunStatus.FAILED, str(exc)
+    return record
+
+
+def compare_backtest_records(records: tuple[LocalRunRecord, ...]) -> dict:
+    """对照固定市场输入与账户配置，展示各次回测结果。"""
+    if not 2 <= len(records) <= 100:
+        raise ValueError("需选择 2 至 100 次回测")
+    first, reasons = records[0], set()
+    for record in records:
+        if record.status != RunStatus.COMPLETED:
+            reasons.add("failed_run")
+        if record.provenance.get("manifest_digest") != first.provenance.get("manifest_digest"):
+            reasons.add("market_snapshot_differs")
+        for key in (
+            "start",
+            "end",
+            "instrument_ids",
+            "frequency",
+            "initial_cash",
+            "execution_frequency",
+            "execution_policy",
+            "strategy_adjustment_type",
+        ):
+            if record.request["config"].get(key) != first.request["config"].get(key):
+                reasons.add(f"{key}_differs")
+    return {
+        "comparable": not reasons,
+        "reasons": sorted(reasons),
+        "runs": [record.model_dump(mode="json") for record in records],
+    }

quant_cli/app/data/cli/data_cli.py

@@ -0,0 +1,78 @@
+"""显式在线数据准备与离线快照校验。"""
+
+import asyncio
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from quant_cli.app.data.data_dependency import build_market_query
+from quant_cli.app.data.schema.dataset_request_schema import DatasetPrepareRequest
+from quant_cli.app.data.service.dataset_load_service import load_dataset
+from quant_cli.app.data.service.dataset_prepare_service import prepare_dataset
+from quant_cli.common.constants import DEFAULT_DATASETS_DIR
+from quant_cli.common.utils.cli_utils import cli_errors, output
+
+app = typer.Typer(help="行情快照准备与校验", no_args_is_help=True)
+
+
+@app.command("prepare")
+@cli_errors
+def prepare(
+    config: Annotated[Path, typer.Option(exists=True)],
+    cache_dir: Annotated[Path, typer.Option()] = DEFAULT_DATASETS_DIR,
+) -> None:
+    """准备显式行情范围与预热窗口，返回实际 Manifest 路径。"""
+    request = DatasetPrepareRequest.model_validate_json(config.read_text(encoding="utf-8"))
+
+    async def execute() -> str:
+        async with build_market_query() as client:
+            result = await prepare_dataset(
+                client,
+                instrument_ids=request.instrument_ids,
+                frequency=request.frequency.value,
+                execution_frequency=request.execution_frequency.value,
+                start=request.start,
+                end=request.end,
+                adjustment_type=request.adjustment_type,
+                cache_dir=cache_dir,
+                lookback_days=request.lookback_days,
+                max_rows=request.max_rows,
+                revision=request.revision,
+            )
+            return str(result.manifest_path.resolve())
+
+    output({"manifest_path": asyncio.run(execute())})
+
+
+@app.command("verify")
+@cli_errors
+def verify(manifest: Annotated[Path, typer.Option(exists=True)], max_rows: int = 1000000) -> None:
+    market, _ = load_dataset(manifest, max_rows)
+    output(
+        {
+            "status": "verified",
+            "revision": market.revision,
+            "start": market.start,
+            "end": market.end,
+        }
+    )
+
+
+@app.command("audit")
+@cli_errors
+def audit(
+    manifest: Annotated[Path, typer.Option(exists=True)],
+    config: Annotated[Path, typer.Option(exists=True)],
+    max_rows: int = 1000000,
+) -> None:
+    """检查日线研究/回测快照的事实与覆盖；缺口以 JSON 返回并非零退出。"""
+    from quant_cli.app.data.schema.dataset_audit_schema import DatasetAuditRequest
+    from quant_cli.app.data.service.dataset_audit_service import audit_dataset
+
+    market, verification = load_dataset(manifest, max_rows)
+    request = DatasetAuditRequest.model_validate_json(config.read_text(encoding="utf-8"))
+    report = asyncio.run(audit_dataset(market, verification, request))
+    output({"ready": report.ready, **report.model_dump(mode="json")})
+    if not report.ready:
+        raise typer.Exit(1)

quant_cli/app/data/data_dependency.py

@@ -0,0 +1,35 @@
+from pqdata import MarketClient
+
+"""量化数据组装；数据库会话不再参与行情读取。"""
+
+from pathlib import Path
+
+from quant_core.app.backtest.backtest_input_schema import BacktestMarketComponents
+from quant_core.app.trading_runtime.replay.exchange_state_resolver import ExchangeStateResolver
+from quant_core.app.trading_runtime.replay.market_runtime_service import ReplayMarketEventSource
+from quant_core.app.trading_runtime.replay.market_state_resolver import MarketStateResolver
+
+
+def load_backtest_market(manifest_path: Path, max_rows: int) -> BacktestMarketComponents:
+    """从已校验本地快照组装回放输入，不访问 SDK 或数据库。"""
+    from quant_cli.app.data.service.dataset_load_service import load_dataset
+
+    market, verification = load_dataset(manifest_path, max_rows)
+    return BacktestMarketComponents(
+        market,
+        verification,
+        ReplayMarketEventSource(market),
+        MarketStateResolver(),
+        ExchangeStateResolver(),
+        manifest_path,
+    )
+
+
+def build_market_query() -> MarketClient:
+    """显式在线准备入口，配置仅为服务地址与令牌。"""
+    import os
+
+    return MarketClient(
+        os.getenv("MARKET_BASE_URL", "http://localhost:8002/api/v1"),
+        token=os.getenv("MARKET_API_TOKEN", ""),
+    )

quant_cli/app/data/repository/parquet_repository.py

@@ -0,0 +1,124 @@
+"""不可变表格文件与原子清单；业务键由数据集提供，不限定行情时间模型。"""
+
+import hashlib
+import json
+import os
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+from quant_cli.app.data.schema.parquet_file_schema import ParquetFile
+
+
+def file_digest(path: Path) -> str:
+    """流式校验文件，避免为了摘要再次读入完整文件。"""
+    with path.open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def atomic_json(path: Path, payload: dict[str, Any]) -> None:
+    """先完整刷盘再替换单个指针；引用文件必须在调用前已完成写入。"""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=path.parent, suffix=".part", delete=False) as stream:
+        temporary = Path(stream.name)
+        try:
+            stream.write(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8"))
+            stream.flush()
+            os.fsync(stream.fileno())
+        except BaseException:
+            stream.close()
+            temporary.unlink(missing_ok=True)
+            raise
+    try:
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+@contextmanager
+def cache_writer(directory: Path) -> Iterator[None]:
+    """进程级非阻塞单写者锁；进程退出自动释放，锁文件不删除。"""
+    directory.mkdir(parents=True, exist_ok=True)
+    with (directory / ".writer.lock").open("a+b") as stream:
+        if stream.tell() == 0:
+            stream.write(b"0")
+            stream.flush()
+        stream.seek(0)
+        if os.name == "nt":
+            import msvcrt
+
+            msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
+        else:
+            import fcntl
+
+            fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        try:
+            yield
+        finally:
+            stream.seek(0)
+            if os.name == "nt":
+                msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
+
+
+def write_parquet(
+    directory: Path,
+    frame: pl.DataFrame,
+    *,
+    dataset_id: str,
+    primary_key: tuple[str, ...],
+    schema_version: int = 1,
+) -> ParquetFile:
+    """拒绝重复或空业务键；按键排序、压缩写入后以内容地址保存。"""
+    if not primary_key or not set(primary_key).issubset(frame.columns):
+        raise ValueError("数据集必须声明存在的业务键")
+    keys = frame.select(primary_key)
+    if any(keys.null_count().row(0)) or keys.is_duplicated().any():
+        raise ValueError("数据集业务键为空或重复")
+    directory.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile(dir=directory, suffix=".part", delete=False) as stream:
+        temporary = Path(stream.name)
+    try:
+        frame.sort(primary_key).write_parquet(temporary, compression="zstd")
+        with temporary.open("rb+") as stream:
+            os.fsync(stream.fileno())
+        digest = file_digest(temporary)
+        path = directory / f"{digest}.parquet"
+        if path.exists():
+            if file_digest(path) != digest:
+                raise ValueError("已有内容地址文件损坏")
+        else:
+            os.replace(temporary, path)
+        return ParquetFile(
+            path=path.name,
+            dataset_id=dataset_id,
+            schema_version=schema_version,
+            primary_key=primary_key,
+            columns=tuple(frame.columns),
+            row_count=frame.height,
+            byte_count=path.stat().st_size,
+            sha256=digest,
+        )
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def read_parquet(directory: Path, receipt: ParquetFile, max_rows: int) -> pl.DataFrame:
+    """校验完整性、容量和键后返回表格；不触发任何网络读取。"""
+    path = (directory / receipt.path).resolve()
+    if path.parent != directory.resolve() or receipt.row_count > max_rows:
+        raise ValueError("缓存路径越界或超出行数上限")
+    if path.stat().st_size != receipt.byte_count or file_digest(path) != receipt.sha256:
+        raise ValueError("缓存文件校验失败")
+    frame = pl.read_parquet(path, n_rows=max_rows + 1)
+    if frame.height != receipt.row_count or tuple(frame.columns) != receipt.columns:
+        raise ValueError("缓存表结构或行数不一致")
+    keys = frame.select(receipt.primary_key)
+    if any(keys.null_count().row(0)) or keys.is_duplicated().any():
+        raise ValueError("缓存业务键非法")
+    return frame

quant_cli/app/data/schema/dataset_audit_schema.py

@@ -0,0 +1,37 @@
+"""日线研究与回测共用快照的就绪检查请求及缺口。"""
+
+from datetime import date
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class DatasetAuditRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    instrument_ids: tuple[str, ...] = Field(min_length=1)
+    start: date
+    end: date
+    require_special_type: bool = False
+
+    @model_validator(mode="after")
+    def validate_range(self) -> "DatasetAuditRequest":
+        if self.start > self.end or len(set(self.instrument_ids)) != len(self.instrument_ids):
+            raise ValueError("检查范围或股票池非法")
+        return self
+
+
+class DatasetGap(BaseModel):
+    instrument_id: str
+    dataset: str
+    reason: str
+    count: int
+    dates: tuple[date, ...] = ()
+
+
+class DatasetAuditReport(BaseModel):
+    revision: str
+    request: DatasetAuditRequest
+    gaps: list[DatasetGap] = Field(default_factory=list)
+
+    @property
+    def ready(self) -> bool:
+        return not self.gaps

quant_cli/app/data/schema/dataset_request_schema.py

@@ -0,0 +1,28 @@
+"""数据准备请求：公开快照范围与容量约束。"""
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from quant_core.app.data.constants import BarFrequency
+from quant_core.app.data.schema.adjustment_schema import AdjustmentType
+
+
+class DatasetPrepareRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    instrument_ids: tuple[str, ...] = Field(min_length=1)
+    frequency: BarFrequency = BarFrequency.D1
+    execution_frequency: BarFrequency = BarFrequency.D1
+    start: datetime
+    end: datetime
+    adjustment_type: AdjustmentType = AdjustmentType.PRE
+    lookback_days: int = Field(default=30, ge=0, le=3660)
+    max_rows: int = Field(default=1000000, ge=1)
+    revision: str | None = None
+
+    @model_validator(mode="after")
+    def validate_input(self) -> "DatasetPrepareRequest":
+        if self.start.utcoffset() is None or self.end.utcoffset() is None or self.start >= self.end:
+            raise ValueError("需指定带时区的合法时间范围")
+        if len(set(self.instrument_ids)) != len(self.instrument_ids):
+            raise ValueError("股票池重复")
+        return self

quant_cli/app/data/schema/parquet_file_schema.py

@@ -0,0 +1,17 @@
+"""不可变 Parquet 文件描述契约。"""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ParquetFile(BaseModel):
+    """文件的精确身份与表结构；覆盖证据由各数据集清单另行记录。"""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+    path: str
+    dataset_id: str
+    schema_version: int = Field(ge=1)
+    primary_key: tuple[str, ...]
+    columns: tuple[str, ...]
+    row_count: int = Field(ge=0)
+    byte_count: int = Field(ge=0)
+    sha256: str = Field(pattern=r"^[a-f0-9]{64}$")