powerquant-cli 0.2.6__py3-none-any.whl

quant_cli/app/data/service/dataset_prepare_service.py

@@ -0,0 +1,187 @@
+from quant_core.app.data.schema.history_schema import HistoryFacts
+
+"""通过 SDK 准备固定版本的本地回放数据；完成后关闭网络客户端。"""
+
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import datetime, time, timedelta
+from pathlib import Path
+
+from pqdata import MarketClient
+from pqdata.schema.bar_schema import BarFrequency, BarQuery
+from pqdata.schema.history_schema import HistoryFactsQuery
+from pydantic import TypeAdapter
+from quant_core.app.data.schema.adjustment_schema import AdjustmentType
+from quant_core.app.data.schema.market_schema import BarResponse, InstrumentResponse
+from quant_core.app.data.service.prepared_data_verification_service import PreparedDataVerification
+from quant_core.app.data.service.prepared_market_service import PreparedMarketData
+
+from quant_cli.app.data.repository.parquet_repository import atomic_json, write_parquet
+from quant_cli.app.data.service.bar_cache_service import (
+    BarCacheQuery,
+    bars_frame,
+    load_bar_snapshot,
+    prepare_bar_cache,
+)
+from quant_cli.app.data.service.dataset_load_service import load_dataset, manifest_digest
+
+
+@dataclass(frozen=True)
+class PreparedDataset:
+    market: PreparedMarketData
+    verification: PreparedDataVerification
+    manifest_path: Path
+
+
+async def prepare_dataset(
+    client: MarketClient,
+    *,
+    instrument_ids: tuple[str, ...],
+    frequency: str,
+    execution_frequency: str,
+    start: datetime,
+    end: datetime,
+    adjustment_type: AdjustmentType,
+    cache_dir: Path,
+    lookback_days: int,
+    max_rows: int,
+    revision: str | None = None,
+) -> PreparedDataset:
+    """先固定事实版本，再校验并下载；版本冲突或超出容量时直接失败。"""
+    history_start = start - timedelta(days=lookback_days)
+    first = await client.get_history_facts(
+        HistoryFactsQuery(
+            instrument_id=instrument_ids[0],
+            start=history_start.date(),
+            end=end.date(),
+            revision=revision,
+        )
+    )
+    market = PreparedMarketData(history_start, end, first.revision)
+    instruments = await client.search_instruments(limit=10000)
+    if len(instruments) >= 10000:
+        raise ValueError("标的目录达到单次读取上限，无法证明目录完整")
+    market.instruments = [InstrumentResponse.model_validate(row) for row in instruments]
+    verification = PreparedDataVerification()
+    cache_key = hashlib.sha256(
+        json.dumps(
+            {
+                "format_version": 2,
+                "source": client.source_id,
+                "revision": first.revision,
+                "instruments": sorted(instrument_ids),
+                "frequency": frequency,
+                "execution_frequency": execution_frequency,
+                "start": start.isoformat(),
+                "end": end.isoformat(),
+                "lookback_days": lookback_days,
+                "adjustment_type": adjustment_type.value,
+            },
+            sort_keys=True,
+        ).encode()
+    ).hexdigest()
+    directory = cache_dir / cache_key
+    manifest_path = directory / "manifest.json"
+    if manifest_path.exists():
+        prepared, evidence = load_dataset(manifest_path, max_rows)
+        return PreparedDataset(prepared, evidence, manifest_path)
+    directory.mkdir(parents=True, exist_ok=True)
+    manifest = {
+        "format_version": 2,
+        "source": client.source_id,
+        "frequencies": sorted({frequency, execution_frequency}),
+        "revision": first.revision,
+        "start": history_start.isoformat(),
+        "end": end.isoformat(),
+        "files": [],
+        "facts": {},
+        "health": [],
+    }
+    total_rows = 0
+    cached_bars = {}
+    for value in sorted({frequency, execution_frequency}):
+        path = await prepare_bar_cache(
+            client,
+            BarCacheQuery(
+                instrument_ids=list(instrument_ids),
+                frequency=TypeAdapter(BarFrequency).validate_python(value),
+                start=history_start,
+                end=end,
+                revision=first.revision,
+            ),
+            cache_dir,
+            max_rows,
+        )
+        _, rows = load_bar_snapshot(path, max_rows)
+        cached_bars[value] = rows
+        total_rows += len(rows)
+        if total_rows > max_rows:
+            raise ValueError("回放数据超过配置行数上限")
+    for instrument_id in instrument_ids:
+        facts = await client.get_history_facts(
+            HistoryFactsQuery(
+                instrument_id=instrument_id,
+                start=history_start.date(),
+                end=end.date(),
+                revision=first.revision,
+            )
+        )
+        market.facts[instrument_id] = HistoryFacts.model_validate(facts.model_dump())
+        manifest["facts"][instrument_id] = facts.model_dump(mode="json")
+        verification_start = start
+        if facts.instrument.listed_date is not None and facts.instrument.listed_date > start.date():
+            verification_start = datetime.combine(
+                facts.instrument.listed_date, time(9, 30), tzinfo=start.tzinfo
+            )
+        if verification_start <= end:
+            for value in sorted({frequency, execution_frequency}):
+                result = await client.verify_bars(instrument_id, value, verification_start, end)
+                verification.add("bar", instrument_id, value, verification_start, end, result)
+                manifest["health"].append(result.model_dump(mode="json"))
+            if adjustment_type is not AdjustmentType.NONE:
+                result = await client.verify_adjustment_factors(
+                    instrument_id, frequency, verification_start, end
+                )
+                verification.add(
+                    "adjustment_factor", instrument_id, frequency, verification_start, end, result
+                )
+                manifest["health"].append(result.model_dump(mode="json"))
+            result = await client.verify_corporate_actions(instrument_id, verification_start, end)
+            verification.add(
+                "corporate_action", instrument_id, "1d", verification_start, end, result
+            )
+            manifest["health"].append(result.model_dump(mode="json"))
+        for value in sorted({frequency, execution_frequency}):
+            query = BarQuery(
+                instrument_ids=[instrument_id],
+                frequency=TypeAdapter(BarFrequency).validate_python(value),
+                start=history_start,
+                end=end,
+                revision=first.revision,
+            )
+            selected = [row for row in cached_bars[value] if row.instrument_id == instrument_id]
+            receipt = write_parquet(
+                directory,
+                bars_frame(selected),
+                dataset_id="bar.raw",
+                primary_key=("timestamp", "instrument_id", "frequency"),
+            )
+            bars = [BarResponse.model_validate(row) for row in selected]
+            market.add_bars(instrument_id, value, bars)
+            manifest["files"].append(
+                {"query": query.model_dump(mode="json"), **receipt.model_dump(mode="json")}
+            )
+    # 校验元数据、覆盖校验和最后一页之间没有并发提交。
+    await client.get_history_facts(
+        HistoryFactsQuery(
+            instrument_id=instrument_ids[0],
+            start=history_start.date(),
+            end=end.date(),
+            revision=first.revision,
+        )
+    )
+    manifest["instruments"] = [row.model_dump(mode="json") for row in market.instruments]
+    manifest["content_digest"] = manifest_digest(manifest)
+    atomic_json(manifest_path, manifest)
+    return PreparedDataset(market, verification, manifest_path)

quant_cli/app/factor/cli/factor_definition_cli.py

@@ -0,0 +1,81 @@
+"""因子定义发布和下载命令；与本地运行记录命令分组。"""
+
+import json
+from pathlib import Path
+from typing import Annotated
+from urllib.parse import quote
+
+import typer
+from quant_core.app.factor.schema.factor_computation_schema import FactorComputationSpec
+from quant_core.app.factor.schema.factor_publication_schema import PublishedFactor
+
+from quant_cli.common.http.definition_client import request_definition, save_definition
+from quant_cli.common.utils.cli_utils import cli_errors, output
+
+app = typer.Typer(help="因子定义发布、查询与获取", no_args_is_help=True)
+Server = Annotated[str, typer.Option(envvar="PQ_QUANT_URL")]
+Token = Annotated[str, typer.Option(envvar="PQ_QUANT_TOKEN", help="建议由环境变量提供令牌")]
+
+
+@app.command("publish")
+@cli_errors
+def publish(
+    config: Annotated[Path, typer.Option(exists=True)],
+    server: Server,
+    token: Token,
+    source: Annotated[Path | None, typer.Option(exists=True)] = None,
+) -> None:
+    """发布 JSON 定义，支持从 Python 文件读取源码，不执行它。"""
+    payload = json.loads(config.read_text(encoding="utf-8"))
+    if source is not None:
+        payload["python_source"] = source.read_text(encoding="utf-8")
+    spec = FactorComputationSpec.model_validate(payload)
+    result = PublishedFactor.model_validate(
+        request_definition(
+            server, token, "POST", "/factor/definitions", payload=spec.model_dump(mode="json")
+        )
+    )
+    output(result.model_dump(mode="json"))
+
+
+@app.command("list")
+@cli_errors
+def list_definitions(
+    server: Server, token: Token, name: str | None = None, limit: int = 100, offset: int = 0
+) -> None:
+    """分页列出发布定义。"""
+    params: dict[str, str | int] = {"limit": limit, "offset": offset}
+    if name is not None:
+        params["name"] = name
+    output(request_definition(server, token, "GET", "/factor/definitions", params=params))
+
+
+@app.command("fetch")
+@cli_errors
+def fetch(
+    name: str,
+    server: Server,
+    token: Token,
+    destination: Annotated[Path, typer.Option()],
+    version: str = "latest",
+    source: str = "local-python",
+    frequency: str = "1d",
+) -> None:
+    """获取并校验完整身份的定义文件，latest 在此固定。"""
+    result = PublishedFactor.model_validate(
+        request_definition(
+            server,
+            token,
+            "GET",
+            f"/factor/definitions/{quote(name, safe='')}/{quote(version, safe='')}",
+            params={"source": source, "frequency": frequency},
+        )
+    )
+    save_definition(destination, result.model_dump(mode="json"))
+    output(
+        {
+            "path": str(destination.resolve()),
+            "identity": result.definition.identity.model_dump(mode="json"),
+            "content_digest": result.content_digest,
+        }
+    )

quant_cli/app/factor/cli/local_factor_cli.py

@@ -0,0 +1,102 @@
+"""本地因子命令，只调用公开执行和存取能力。"""
+
+import asyncio
+from pathlib import Path
+from typing import Annotated
+from uuid import UUID
+
+import typer
+from quant_core.app.factor.schema.local_factor_schema import LocalFactorRequest
+
+from quant_cli.app.factor.cli.factor_definition_cli import app as definitions
+from quant_cli.app.factor.cli.local_selection_cli import register_selection_commands
+from quant_cli.app.factor.local_factor_dependency import build_local_factor_store
+from quant_cli.app.factor.service.local_factor_service import run_local_factor
+from quant_cli.common.constants import DEFAULT_DATABASE_PATH
+from quant_cli.common.utils.cli_utils import cli_errors, output
+
+app = typer.Typer(help="本地因子计算与归档", no_args_is_help=True)
+
+app.add_typer(definitions, name="definition")
+register_selection_commands(app)
+
+
+@app.command("compute")
+@cli_errors
+def compute(
+    config: Annotated[Path, typer.Option(exists=True)],
+    source: Annotated[Path | None, typer.Option(exists=True)] = None,
+    definition: Annotated[list[Path] | None, typer.Option(exists=True)] = None,
+    database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+) -> None:
+    """从 JSON 请求和源码或下载定义执行本地因子，并默认归档。"""
+    import json
+
+    payload = json.loads(config.read_text(encoding="utf-8"))
+    if source is not None:
+        payload["source"] = source.read_text(encoding="utf-8")
+    if definition:
+        from quant_core.app.factor.schema.factor_publication_schema import PublishedFactor
+
+        if payload.get("source") or payload.get("definitions"):
+            raise ValueError("发布定义与源码/内嵌定义不能同时提供")
+        payload["definitions"] = [
+            PublishedFactor.model_validate_json(path.read_text(encoding="utf-8")).model_dump(
+                mode="json"
+            )
+            for path in definition
+        ]
+    request = LocalFactorRequest.model_validate(payload)
+    output(asyncio.run(run_local_factor(request, build_local_factor_store(database))))
+
+
+@app.command("get")
+@cli_errors
+def get(
+    run: Annotated[UUID, typer.Option()],
+    database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+) -> None:
+    output(build_local_factor_store(database).get(run))
+
+
+@app.command("list")
+@cli_errors
+def list_runs(
+    limit: int = 20,
+    offset: int = 0,
+    database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+) -> None:
+    output(
+        [
+            row.model_dump(mode="json")
+            for row in build_local_factor_store(database).list_runs(limit, offset)
+        ]
+    )
+
+
+@app.command("import")
+@cli_errors
+def import_artifact(
+    artifact: Annotated[Path, typer.Option(exists=True)],
+    database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+) -> None:
+    """由单一归档入口导入独立 Worker 产物。"""
+    from quant_cli.app.factor.service.local_factor_service import import_factor_artifact
+
+    output(
+        import_factor_artifact(
+            artifact.read_text(encoding="utf-8"), build_local_factor_store(database)
+        )
+    )
+
+
+@app.command("values")
+@cli_errors
+def values(
+    run: Annotated[UUID, typer.Option()],
+    limit: int = 100,
+    offset: int = 0,
+    database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+) -> None:
+    """分页查询归档因子值。"""
+    output(build_local_factor_store(database).list_values(run, limit, offset))

quant_cli/app/factor/cli/local_selection_cli.py

@@ -0,0 +1,58 @@
+"""选股命令只负责输入输出与依赖组装。"""
+
+import asyncio
+from pathlib import Path
+from typing import Annotated
+from uuid import UUID
+
+import typer
+
+from quant_cli.app.factor.local_factor_dependency import (
+    build_local_factor_store,
+    build_local_selection_store,
+)
+from quant_cli.app.factor.schema.local_selection_schema import LocalSelectionRequest
+from quant_cli.app.factor.service.local_selection_service import run_local_selection
+from quant_cli.common.constants import DEFAULT_DATABASE_PATH
+from quant_cli.common.utils.cli_utils import cli_errors, output
+
+
+def register_selection_commands(app: typer.Typer) -> None:
+    @app.command("select")
+    @cli_errors
+    def select(
+        config: Annotated[Path, typer.Option(exists=True)],
+        database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+    ) -> None:
+        request = LocalSelectionRequest.model_validate_json(config.read_text(encoding="utf-8"))
+        output(
+            asyncio.run(
+                run_local_selection(
+                    request,
+                    build_local_factor_store(database),
+                    build_local_selection_store(database),
+                )
+            )
+        )
+
+    @app.command("selection-get")
+    @cli_errors
+    def get(
+        run: Annotated[UUID, typer.Option()],
+        database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+    ) -> None:
+        output(build_local_selection_store(database).get(run))
+
+    @app.command("selection-list")
+    @cli_errors
+    def list_runs(
+        limit: int = 20,
+        offset: int = 0,
+        database: Annotated[Path, typer.Option(envvar="PQ_LOCAL_DATABASE")] = DEFAULT_DATABASE_PATH,
+    ) -> None:
+        output(
+            [
+                row.model_dump(mode="json")
+                for row in build_local_selection_store(database).list_runs(limit, offset)
+            ]
+        )

quant_cli/app/factor/interface/local_selection_store.py

@@ -0,0 +1,12 @@
+"""本地选股运行存取协议。"""
+
+from typing import Protocol
+from uuid import UUID
+
+from quant_core.common.local_run import LocalRunRecord
+
+
+class LocalSelectionStore(Protocol):
+    def save(self, record: LocalRunRecord) -> None: ...
+    def get(self, run_uuid: UUID) -> LocalRunRecord: ...
+    def list_runs(self, limit: int = 20, offset: int = 0) -> list[LocalRunRecord]: ...

quant_cli/app/factor/local_factor_dependency.py

@@ -0,0 +1,38 @@
+"""本地因子组装，不加载远端存储实现。"""
+
+from pathlib import Path
+
+from quant_core.app.factor.interface.public.factor_local import LocalFactorStore
+
+from quant_cli.app.factor.interface.local_selection_store import LocalSelectionStore
+from quant_cli.app.factor.repository.factor_local_repository import FactorLocalRepository
+
+
+def build_local_factor_store(path: Path) -> LocalFactorStore:
+    """返回本地因子结果存取。"""
+    return FactorLocalRepository(path)
+
+
+# 公开函数入口不包一层重复转发；内部实现只由本域组装模块暴露。
+from quant_cli.app.factor.service.local_factor_query_service import (
+    build_local_factor_query as build_local_factor_query,
+)
+from quant_cli.app.factor.service.local_factor_service import (
+    compute_local_factor as compute_local_factor,
+)
+from quant_cli.app.factor.service.local_factor_service import (
+    decode_factor_artifact as decode_factor_artifact,
+)
+from quant_cli.app.factor.service.local_factor_service import (
+    import_factor_artifact as import_factor_artifact,
+)
+from quant_cli.app.factor.service.local_factor_service import (
+    load_factor_result as load_factor_result,
+)
+
+
+def build_local_selection_store(path: Path) -> LocalSelectionStore:
+    """按显式路径组装选股归档能力。"""
+    from quant_cli.app.factor.repository.selection_local_repository import SelectionLocalRepository
+
+    return SelectionLocalRepository(path)

quant_cli/app/factor/repository/factor_local_repository.py

@@ -0,0 +1,154 @@
+"""factor 本地运行存取；短事务由本地连接上下文管理。"""
+
+from datetime import UTC
+from pathlib import Path
+from uuid import UUID
+
+from quant_core.common.local_run import LocalRunRecord
+
+from quant_cli.common.database.duckdb_connection import local_transaction
+
+
+class FactorLocalRepository:
+    """只维护本领域的运行记录，查询使用数据库分页。"""
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+
+    def save(self, record: LocalRunRecord, values: dict | None = None) -> None:
+        with local_transaction(self.path) as connection:
+            connection.execute("""CREATE TABLE IF NOT EXISTS factor_local_run (
+                id BIGINT PRIMARY KEY, uuid UUID UNIQUE NOT NULL,
+                created_at TIMESTAMPTZ NOT NULL, document JSON NOT NULL)""")
+            existing = connection.execute(
+                "SELECT document FROM factor_local_run WHERE uuid = ?", [str(record.uuid)]
+            ).fetchone()
+            if existing is not None:
+                if LocalRunRecord.model_validate_json(existing[0]) != record:
+                    raise ValueError("同一运行 UUID 的记录不可覆盖")
+                from decimal import Decimal
+
+                stored = connection.execute(
+                    """SELECT name, as_of AT TIME ZONE 'UTC', instrument_id, value
+                    FROM factor_local_value AS v JOIN factor_local_run AS r ON r.id = v.run_id
+                    WHERE r.uuid = ?""",
+                    [str(record.uuid)],
+                ).fetchall()
+                actual = {
+                    (name, timestamp.replace(tzinfo=UTC), instrument): Decimal(value)
+                    if value is not None
+                    else None
+                    for name, timestamp, instrument, value in stored
+                }
+                expected = {
+                    (name, timestamp, instrument): value
+                    for name, cells in (values or {}).items()
+                    for (timestamp, instrument), value in cells.items()
+                }
+                if actual != expected:
+                    raise ValueError("同一运行 UUID 的因子值不可覆盖")
+                return
+            connection.execute(
+                """INSERT INTO factor_local_run
+                SELECT COALESCE(MAX(id), 0) + 1, ?, ?, ? FROM factor_local_run""",
+                [str(record.uuid), record.created_at, record.model_dump_json()],
+            )
+
+            connection.execute("""CREATE TABLE IF NOT EXISTS factor_local_value (
+                run_id BIGINT NOT NULL REFERENCES factor_local_run(id),
+                name VARCHAR NOT NULL, as_of TIMESTAMPTZ NOT NULL,
+                instrument_id VARCHAR NOT NULL, value VARCHAR,
+                PRIMARY KEY (run_id, name, as_of, instrument_id))""")
+            identity = connection.execute(
+                "SELECT id FROM factor_local_run WHERE uuid = ?", [str(record.uuid)]
+            ).fetchone()
+            assert identity is not None
+            if values:
+                connection.executemany(
+                    "INSERT INTO factor_local_value VALUES (?, ?, ?, ?, ?)",
+                    [
+                        (
+                            identity[0],
+                            name,
+                            timestamp,
+                            instrument,
+                            None if value is None else str(value),
+                        )
+                        for name, cells in values.items()
+                        for (timestamp, instrument), value in cells.items()
+                    ],
+                )
+
+    def get(self, run_uuid: UUID) -> LocalRunRecord:
+        if not self.path.is_file():
+            raise ValueError("运行记录不存在")
+        with local_transaction(self.path, read_only=True) as connection:
+            if not connection.execute(
+                "SELECT 1 FROM information_schema.tables WHERE table_name = ?",
+                ["factor_local_run"],
+            ).fetchall():
+                raise ValueError("运行记录不存在")
+            row = connection.execute(
+                "SELECT document FROM factor_local_run WHERE uuid = ?", [str(run_uuid)]
+            ).fetchone()
+        if row is None:
+            raise ValueError("运行记录不存在")
+        return LocalRunRecord.model_validate_json(row[0])
+
+    def list_runs(self, limit: int = 20, offset: int = 0) -> list[LocalRunRecord]:
+        if not 1 <= limit <= 1000 or offset < 0:
+            raise ValueError("分页参数非法")
+        if not self.path.is_file():
+            return []
+        with local_transaction(self.path, read_only=True) as connection:
+            if not connection.execute(
+                "SELECT 1 FROM information_schema.tables WHERE table_name = ?",
+                ["factor_local_run"],
+            ).fetchall():
+                return []
+            rows = connection.execute(
+                """SELECT document FROM factor_local_run
+                ORDER BY id DESC LIMIT ? OFFSET ?""",
+                [limit, offset],
+            ).fetchall()
+        return [LocalRunRecord.model_validate_json(row[0]) for row in rows]
+
+    def load_values(self, run_uuid: UUID) -> dict:
+        from decimal import Decimal
+
+        values: dict = {}
+        with local_transaction(self.path, read_only=True) as connection:
+            rows = connection.execute(
+                """SELECT name, as_of AT TIME ZONE 'UTC', instrument_id, value
+                FROM factor_local_value AS v JOIN factor_local_run AS r ON r.id = v.run_id
+                    WHERE r.uuid = ? ORDER BY as_of, instrument_id""",
+                [str(run_uuid)],
+            ).fetchall()
+        for name, timestamp, instrument, value in rows:
+            values.setdefault(name, {})[(timestamp.replace(tzinfo=UTC), instrument)] = (
+                Decimal(value) if value is not None else None
+            )
+        return values
+
+    def list_values(self, run_uuid: UUID, limit: int = 100, offset: int = 0) -> list[dict]:
+        """分页返回因子值；不在 CLI 中一次展开全市场结果。"""
+        if not 1 <= limit <= 10000 or offset < 0:
+            raise ValueError("因子值分页参数非法")
+        self.get(run_uuid)
+        with local_transaction(self.path, read_only=True) as connection:
+            rows = connection.execute(
+                """SELECT name, as_of AT TIME ZONE 'UTC', instrument_id, value
+                FROM factor_local_value AS v JOIN factor_local_run AS r ON r.id = v.run_id
+                    WHERE r.uuid = ? ORDER BY name, as_of, instrument_id
+                LIMIT ? OFFSET ?""",
+                [str(run_uuid), limit, offset],
+            ).fetchall()
+        return [
+            {
+                "name": name,
+                "as_of": timestamp.replace(tzinfo=UTC).isoformat(),
+                "instrument_id": instrument,
+                "value": value,
+            }
+            for name, timestamp, instrument, value in rows
+        ]