quantnodes 3.0.0__py3-none-any.whl

QuantNodes/cache_node/base.py

@@ -0,0 +1,244 @@
+# coding=utf-8
+"""
+MarketDataCacheNode - 行情数据缓存节点
+
+继承 BaseNode, 用 Parquet 文件缓存行情数据。
+支持透明代理模式和独立 Pipeline 节点两种集成方式。
+
+工作流程:
+1. 生成 cache_key (hash of source+table+columns+filter)
+2. 检查缓存是否存在且未过期
+3. 命中 → 直接读缓存返回
+4. 未命中或过期 → 查询数据源 → 写入缓存 → 返回
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import pandas as pd
+
+from QuantNodes.core.node import BaseNode, register_node
+from QuantNodes.cache_node.cache_store import ParquetCacheStore
+from QuantNodes.cache_node.metadata import CacheMetadata
+
+
+def make_cache_key(source: str, table: str, columns: List[str], query_filter: str) -> str:
+    """生成缓存 key (MD5 前 12 位)"""
+    key_parts = [
+        source,
+        table,
+        ",".join(sorted(columns or [])),
+        query_filter or "",
+    ]
+    key_str = "|".join(key_parts)
+    return hashlib.md5(key_str.encode()).hexdigest()[:12]
+
+
+@register_node
+class MarketDataCacheNode(BaseNode[Dict[str, Any], pd.DataFrame]):
+    """行情数据缓存节点
+
+    透明代理模式:
+        在 ConfigBacktestTool._load_from_db() 中自动调用,
+        对上层透明, 只需在 DataConfig 中设置 cache_enabled=True。
+
+    独立节点模式:
+        在 YAML Pipeline 中作为独立节点使用:
+        market_data_cache >> config_executor >> backtest
+
+    输入 (input_data dict):
+        source: 数据源类型 (clickhouse/mysql)
+        table: 表名
+        columns: 列名列表
+        query_filter: WHERE 子句
+        node: BaseDBNode 实例 (用于回源查询)
+        date_column: 日期列名 (用于增量查询)
+
+    输出:
+        pd.DataFrame 缓存数据
+    """
+
+    def __init__(self, name: str = None, config: Dict[str, Any] = None, **kwargs):
+        super().__init__(name=name or "MarketDataCache", config=config, **kwargs)
+        self._store = ParquetCacheStore(
+            cache_dir=self.config.get("cache_dir", "~/.quantnodes/cache")
+        )
+        self._meta = CacheMetadata()
+        self._ttl_days = self.config.get("ttl_days", 7)
+        self._force_refresh = self.config.get("force_refresh", False)
+
+    def _execute(self, input_data: Dict[str, Any] = None, **kwargs) -> pd.DataFrame:
+        """执行缓存查询
+
+        1. 生成 cache_key
+        2. 检查缓存 → 命中则直接返回
+        3. 未命中 → 查询数据源 → 写缓存 → 返回
+        """
+        if input_data is None:
+            raise ValueError("input_data is required")
+
+        source = input_data["source"]
+        table = input_data["table"]
+        columns = input_data.get("columns", [])
+        query_filter = input_data.get("query_filter", "")
+        db_node = input_data.get("node")
+        date_column = input_data.get("date_column", "")
+
+        cache_key = make_cache_key(source, table, columns, query_filter)
+        table_dir = self._store._get_table_dir(table)
+
+        # 1. 检查缓存
+        if not self._force_refresh and self._store.exists(table):
+            cached_meta = self._meta.load(table_dir)
+            if cached_meta is not None and not self._meta.is_expired(cached_meta):
+                # 缓存命中
+                self._meta.touch(cached_meta)
+                self._meta.save(table_dir, cached_meta)
+                df = self._store.read(table)
+                if df is not None:
+                    return df
+
+        # 2. 缓存未命中或过期 → 查询数据源
+        if db_node is None:
+            raise ValueError("input_data['node'] (BaseDBNode) is required for cache miss")
+
+        # 3. 尝试增量查询
+        if self._store.exists(table) and date_column:
+            cached_meta = self._meta.load(table_dir)
+            if cached_meta is not None and cached_meta.date_range:
+                last_date = cached_meta.date_range[1]
+                if last_date:
+                    new_df = self._incremental_query(
+                        db_node, source, table, columns,
+                        query_filter, date_column, last_date
+                    )
+                    if new_df is not None and len(new_df) > 0:
+                        self._store.append(table, new_df)
+                        self._update_meta_after_write(
+                            table, cache_key, source, query_filter,
+                            columns, date_column, table_dir
+                        )
+                        return self._store.read(table)
+
+        # 4. 全量查询
+        df = self._full_query(db_node, source, table, columns, query_filter)
+
+        # 5. 写入缓存
+        if df is not None and not df.empty:
+            self._store.write(table, df)
+            self._update_meta_after_write(
+                table, cache_key, source, query_filter,
+                columns, date_column, table_dir, df
+            )
+
+        return df if df is not None else pd.DataFrame()
+
+    def _full_query(
+        self, db_node, source: str, table: str,
+        columns: List[str], query_filter: str
+    ) -> Optional[pd.DataFrame]:
+        """全量查询"""
+        cols_str = ", ".join(columns) if columns else "*"
+        sql = f"SELECT {cols_str} FROM {table}"
+        if query_filter:
+            sql += " " + query_filter.lstrip("WHERE ").lstrip("where ")
+
+        db_node.connect()
+        try:
+            return db_node.query(sql)
+        finally:
+            db_node.disconnect()
+
+    def _incremental_query(
+        self, db_node, source: str, table: str,
+        columns: List[str], query_filter: str,
+        date_column: str, last_date: str
+    ) -> Optional[pd.DataFrame]:
+        """增量查询: 只查新增数据"""
+        cols_str = ", ".join(columns) if columns else "*"
+        sql = f"SELECT {cols_str} FROM {table}"
+
+        # 构建 WHERE 子句
+        where_parts = []
+        if query_filter:
+            where_parts.append(query_filter.lstrip("WHERE ").lstrip("where "))
+
+        # 使用 db_date_column (数据库原始列名) 来做增量查询
+        # 这里 date_column 可能是映射后的名字, 需要从 query_filter 中推断原始列名
+        # 简单方案: 直接用日期比较
+        where_parts.append(f"{date_column} > '{last_date}'")
+
+        sql += " WHERE " + " AND ".join(where_parts)
+
+        db_node.connect()
+        try:
+            return db_node.query(sql)
+        finally:
+            db_node.disconnect()
+
+    def _update_meta_after_write(
+        self, table: str, cache_key: str, source: str,
+        query_filter: str, columns: List[str],
+        date_column: str, table_dir: Path,
+        df: pd.DataFrame = None,
+    ) -> None:
+        """写入后更新元数据"""
+        if df is None:
+            df = self._store.read(table)
+        if df is None:
+            return
+
+        date_range = []
+        if date_column and date_column in df.columns:
+            date_range = [
+                str(df[date_column].min()),
+                str(df[date_column].max()),
+            ]
+
+        meta = self._meta.create(
+            table=table,
+            cache_key=cache_key,
+            source=source,
+            query_filter=query_filter,
+            ttl_days=self._ttl_days,
+            row_count=len(df),
+            columns=list(df.columns),
+            date_range=date_range,
+        )
+        self._meta.save(table_dir, meta)
+
+    def invalidate(self, table: str = None) -> None:
+        """手动失效缓存
+
+        Args:
+            table: 指定表名, 为 None 时清除所有缓存
+        """
+        if table:
+            self._store.delete(table)
+        else:
+            import shutil
+            if self._store.cache_dir.exists():
+                shutil.rmtree(self._store.cache_dir)
+
+    def get_info(self) -> Dict[str, Any]:
+        """获取缓存状态信息"""
+        tables = self._store.list_tables()
+        info = {
+            "cache_dir": str(self._store.cache_dir),
+            "tables": {},
+        }
+        for t in tables:
+            size = self._store.get_size(t)
+            table_dir = self._store._get_table_dir(t)
+            meta = self._meta.load(table_dir)
+            info["tables"][t] = {
+                "size_bytes": size,
+                "size_mb": round(size / 1024 / 1024, 2),
+                "expired": self._meta.is_expired(meta) if meta else True,
+                "row_count": meta.row_count if meta else 0,
+                "created_at": meta.created_at if meta else None,
+            }
+        return info

QuantNodes/cache_node/cache_store.py

@@ -0,0 +1,99 @@
+# coding=utf-8
+"""
+Parquet 缓存存储引擎
+
+用 Parquet 文件缓存行情数据, 支持读写/追加/删除/存在检查。
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+from QuantNodes.core.path_utils import ensure_dir
+
+
+class ParquetCacheStore:
+    """Parquet 文件缓存存储
+
+    目录结构:
+        cache_dir/
+        └── {table}/
+            └── data.parquet
+
+    table 中的 '.' 自动替换为 '__' (quote.cn_stock → quote__cn_stock)
+    """
+
+    def __init__(self, cache_dir: str = "~/.quantnodes/cache"):
+        self.cache_dir = Path(cache_dir).expanduser()
+
+    def _get_table_dir(self, table: str) -> Path:
+        """表名 → 缓存目录"""
+        safe_name = table.replace(".", "__")
+        return self.cache_dir / safe_name
+
+    def _get_data_path(self, table: str) -> Path:
+        """表名 → data.parquet 路径"""
+        return self._get_table_dir(table) / "data.parquet"
+
+    def exists(self, table: str) -> bool:
+        """检查缓存是否存在"""
+        return self._get_data_path(table).exists()
+
+    def read(self, table: str) -> Optional[pd.DataFrame]:
+        """读取缓存, 不存在则返回 None"""
+        path = self._get_data_path(table)
+        if not path.exists():
+            return None
+        try:
+            return pd.read_parquet(path)
+        except Exception:
+            return None
+
+    def write(self, table: str, df: pd.DataFrame) -> None:
+        """写入缓存 (覆盖模式)"""
+        table_dir = self._get_table_dir(table)
+        ensure_dir(table_dir)
+        df.to_parquet(table_dir / "data.parquet", index=False)
+
+    def append(self, table: str, df_new: pd.DataFrame) -> int:
+        """追加数据到缓存
+
+        自动去重: 以 df_new 的行追加到已有数据。
+        返回追加后的总行数。
+        """
+        existing = self.read(table)
+        if existing is not None and not existing.empty:
+            combined = pd.concat([existing, df_new], ignore_index=True)
+        else:
+            combined = df_new
+
+        self.write(table, combined)
+        return len(combined)
+
+    def delete(self, table: str) -> bool:
+        """删除缓存, 返回是否成功"""
+        table_dir = self._get_table_dir(table)
+        if not table_dir.exists():
+            return False
+        import shutil
+        shutil.rmtree(table_dir)
+        return True
+
+    def get_size(self, table: str) -> int:
+        """获取缓存文件大小 (bytes), 不存在返回 0"""
+        path = self._get_data_path(table)
+        if not path.exists():
+            return 0
+        return path.stat().st_size
+
+    def list_tables(self):
+        """列出所有缓存的表"""
+        if not self.cache_dir.exists():
+            return []
+        tables = []
+        for d in self.cache_dir.iterdir():
+            if d.is_dir() and (d / "data.parquet").exists():
+                tables.append(d.name.replace("__", "."))
+        return tables

QuantNodes/cache_node/metadata.py

@@ -0,0 +1,100 @@
+# coding=utf-8
+"""
+缓存元数据管理
+
+管理缓存文件的元数据 (创建时间、过期时间、行数、日期范围等)。
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional, List
+from QuantNodes.core.path_utils import ensure_dir
+
+
+@dataclass
+class CacheMeta:
+    """缓存元数据"""
+    table: str = ""
+    cache_key: str = ""
+    created_at: str = ""          # ISO 格式
+    last_accessed: str = ""       # ISO 格式
+    ttl_days: int = 7
+    row_count: int = 0
+    columns: List[str] = field(default_factory=list)
+    date_range: List[str] = field(default_factory=list)  # [start, end]
+    source: str = ""
+    query_filter: str = ""
+
+
+class CacheMetadata:
+    """缓存元数据管理器"""
+
+    META_FILENAME = "metadata.json"
+
+    def _get_meta_path(self, table_dir: Path) -> Path:
+        return table_dir / self.META_FILENAME
+
+    def load(self, table_dir: Path) -> Optional[CacheMeta]:
+        """加载元数据, 不存在或损坏返回 None"""
+        path = self._get_meta_path(table_dir)
+        if not path.exists():
+            return None
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+            valid_keys = CacheMeta.__dataclass_fields__
+            filtered = {k: v for k, v in data.items() if k in valid_keys}
+            return CacheMeta(**filtered)
+        except Exception:
+            return None
+
+    def save(self, table_dir: Path, meta: CacheMeta) -> None:
+        """保存元数据"""
+        ensure_dir(table_dir)
+        path = self._get_meta_path(table_dir)
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(asdict(meta), f, indent=2, ensure_ascii=False)
+
+    def is_expired(self, meta: CacheMeta) -> bool:
+        """检查缓存是否过期"""
+        if not meta.created_at:
+            return True
+        try:
+            created = datetime.fromisoformat(meta.created_at)
+            return datetime.now() > created + timedelta(days=meta.ttl_days)
+        except Exception:
+            return True
+
+    def touch(self, meta: CacheMeta) -> None:
+        """更新最后访问时间"""
+        meta.last_accessed = datetime.now().isoformat()
+
+    def create(
+        self,
+        table: str,
+        cache_key: str,
+        source: str,
+        query_filter: str,
+        ttl_days: int,
+        row_count: int,
+        columns: List[str],
+        date_range: List[str],
+    ) -> CacheMeta:
+        """创建新的元数据"""
+        now = datetime.now().isoformat()
+        return CacheMeta(
+            table=table,
+            cache_key=cache_key,
+            created_at=now,
+            last_accessed=now,
+            ttl_days=ttl_days,
+            row_count=row_count,
+            columns=columns,
+            date_range=date_range,
+            source=source,
+            query_filter=query_filter,
+        )

QuantNodes/cli/__init__.py

@@ -0,0 +1,109 @@
+# coding=utf-8
+"""
+QuantNodes CLI - Command Line Interface.
+
+This package is split for readability:
+    _helpers.py          shared helpers + constants
+    commands/init.py     ``quantnodes init``
+    commands/run.py      ``quantnodes run`` (api/frontend start helpers)
+    commands/chat.py     ``quantnodes chat``
+    commands/evolve.py   ``quantnodes evolve`` (Week 5)
+    commands/factor.py   ``quantnodes factor-*`` (Week 5~13)
+    commands/version.py  ``quantnodes version`` / ``quantnodes help``
+
+Public re-exports below keep ``from QuantNodes.cli import main`` and the
+individual ``cmd_*`` symbols importable exactly as before the split.
+
+Commands:
+    init    - Initialize current directory
+    run     - Start services
+    version - Show version
+    help    - Show this help message
+
+Cache TTL alignment (H18): the .env template generated by
+``write_env_file`` (in ``_helpers.py``) writes
+``QUANTNODES__CACHE_TTL=604800``, kept in sync with
+``IFindFetcher.DEFAULT_CACHE_TTL_S`` (7 * 86400).
+"""
+
+import sys
+import argparse
+
+from ._helpers import *  # noqa: F401,F403  (helpers + constants re-exported)
+# PROG_NAME 在本模块 _build_parser 中使用; 其余 add_* builders 现由各 Command
+# 子类自行调用, 这里保留 re-export 仅为 backward compat (redundant-alias 显式标注).
+from ._helpers import (
+    PROG_NAME,
+    add_cli_overrides as add_cli_overrides,
+    add_lineage_depth_args as add_lineage_depth_args,
+    add_metric_arg as add_metric_arg,
+    add_output_arg as add_output_arg,
+    add_pool_dir_arg as add_pool_dir_arg,
+    add_title_arg as add_title_arg,
+    add_top_arg as add_top_arg,
+)
+# cmd_* 函数 re-export: backward compat, ``from QuantNodes.cli import cmd_init`` 不变.
+# Phase 3.1 起 main() 已改走 COMMAND_REGISTRY dispatch, 这些函数仅 cmd_help
+# 在缺省分支直接调用, 其余靠各 Command.run() 间接调用.
+from .commands.init import cmd_init as cmd_init
+from .commands.run import (
+    cmd_run as cmd_run,
+    start_api_server as start_api_server,
+    start_frontend_server as start_frontend_server,
+)
+from .commands.version import cmd_version as cmd_version, cmd_help
+from .commands.chat import cmd_chat as cmd_chat
+from .commands.evolve import (
+    cmd_evolve as cmd_evolve,
+    _load_runner_from_config as _load_runner_from_config,
+)
+from .commands.factor import (
+    cmd_factor_info as cmd_factor_info,
+    cmd_factor_best as cmd_factor_best,
+    cmd_factor_visual as cmd_factor_visual,
+    cmd_factor_dashboard as cmd_factor_dashboard,
+    cmd_factor_data_fetch as cmd_factor_data_fetch,
+    cmd_factor_rag_eval as cmd_factor_rag_eval,
+    cmd_factor_rag_show as cmd_factor_rag_show,
+)
+from .commands import COMMAND_REGISTRY as COMMAND_REGISTRY
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    """Build the top-level argparse parser (with all subcommands).
+
+    Phase 3.1 (2026-06-22): 改为遍历 COMMAND_REGISTRY, 每个 Command 自己
+    负责 add_arguments(subparsers). 替代原 90 行手写 parser 构造. 新增子
+    命令只需写 Command 子类 + 在 commands/__init__.py 注册.
+
+    Phase I1 (2026-06-20): repeated argument groups consolidated via
+    builders in _helpers.py (add_pool_dir_arg 等), 现由各 Command 调用.
+    """
+    parser = argparse.ArgumentParser(
+        prog=PROG_NAME,
+        description="QuantNodes CLI - 量化研究节点架构命令行工具",
+        add_help=False
+    )
+    subparsers = parser.add_subparsers(dest="command", help="可用命令")
+    for cmd in COMMAND_REGISTRY.all():
+        cmd.add_arguments(subparsers)
+    return parser
+
+
+def main():
+    """Main CLI entry point.
+
+    Phase 3.1 (2026-06-22): 改为 registry dispatch. 未知/缺省 command 回退
+    到 help (与旧行为一致).
+    """
+    parser = _build_parser()
+    args = parser.parse_args()
+
+    cmd = COMMAND_REGISTRY.get(args.command)
+    if cmd is None:
+        return cmd_help(args)
+    return cmd.run(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())