quantnodes 3.0.0__py3-none-any.whl

QuantNodes/research/quant_alpha/evaluation/baselines/g2_llm_only.py

@@ -0,0 +1,269 @@
+# coding=utf-8
+"""
+g2_llm_only.py - G2 baseline：mock LLM 直接生成 50 个公式字符串
+
+G2 = "LLM-Only"，代表「直接问 LLM 生成 alpha 公式」的最简 baseline。
+无 5 智能体编排、无反思循环、无 evaluator 反馈 — 仅 LLM 一次输出。
+
+Stage 1 用 mock 模拟 LLM 输出（valid + invalid 混合）；
+Stage 2 用真实 LLM（MiniMax）替换。
+
+模拟策略（mock 阶段）：
+- 60% 公式：valid（LLM 已知基础语法）
+- 25% 公式：复杂 valid（LLM 写出更复杂的算子组合）
+- 15% 公式：invalid（模拟 LLM 错误 / 不支持的算子）
+
+复用：
+- contracts.Baseline：generate_factors() 接口
+- g1_handcrafted._gen_formula：共享公式生成逻辑（DRY）
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+from typing import List, Optional
+
+from ..contracts import Baseline, FactorSpec
+from .g1_handcrafted import _gen_formula
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["G2LlmOnly"]
+
+
+# 15% 模拟 LLM 错误：使用 parser 不支持的算子
+INVALID_LLM_TOKENS = [
+    "rank(close)",  # 跨截面 rank，tool 不支持
+    "IndNeutralize(close, industry)",  # tool 不支持
+    "ts_zscore(close, 20)",  # rolling 映射缺失
+    "log(vol)",  # 需要先除以均值
+    "close - ts_mean(close, 5)",  # 中缀语法
+    "correlation(close, vol, 20)",  # tool 不支持
+    "quantile(close, 0.5)",  # tool 不支持
+]
+
+
+class G2LlmOnly(Baseline):
+    """G2 LLM-Only baseline
+
+    模拟「直接问 LLM 生成 alpha 公式」的最简 baseline：
+    - 不调用 AlphaGptWorkflow
+    - 不做反思 / critic / evaluator 反馈
+    - 一次性输出 N 个公式（含 valid + invalid 混合）
+
+    Stage 1：使用 mock 公式生成器（valid + invalid 混合）
+    Stage 2：注入真实 LLM client（LLMGateway → MiniMax）
+    """
+
+    def __init__(self, n: int = 50, seed: int = 7, llm_client=None) -> None:
+        self.n = n
+        self.seed = seed
+        self._llm_client = llm_client
+
+    @property
+    def group_name(self) -> str:
+        return "G2_LlmOnly"
+
+    def generate_factors(self, n: Optional[int] = None) -> List[FactorSpec]:
+        """生成 n 个因子
+
+        Stage 2 (有 llm_client): 用真实 LLM 生成公式
+        Stage 1 (无 llm_client): 用 mock 公式生成器
+        """
+        n = n or self.n
+
+        # Stage 2: 真实 LLM 生成
+        if self._llm_client is not None:
+            return self._generate_with_llm(n)
+
+        # Stage 1: mock 公式生成
+        return self._generate_mock(n)
+
+    def _generate_with_llm(self, n: int) -> List[FactorSpec]:
+        """Stage 2: 用真实 LLM + operator_lookup 工具生成公式。"""
+        import asyncio
+
+        prompt = (
+            f"你是一个量化研究员，负责生成 alpha 因子公式。\n\n"
+            f"任务：生成 {n} 个唯一的 alpha 因子公式。\n\n"
+            f"步骤：\n"
+            f"1. 先调用 operator_lookup(action='list_operators') 获取可用算子列表\n"
+            f"2. 调用 operator_lookup(action='get_operator_info', name='xxx') 获取算子详情\n"
+            f"3. 生成公式，确保使用可用算子\n"
+            f"4. 调用 operator_lookup(action='validate_formula', formula='xxx') 验证每个公式\n\n"
+            f"基础特征：open, high, low, close, vol, amount\n"
+            f"公式语法：Python 函数调用，如 rank(ts_mean(close, 20))\n\n"
+            f"最终输出：仅返回 JSON 数组，如 [\"rank(ts_mean(close, 20))\", \"ts_std(close, 10)\"]\n"
+            f"不要解释，不要 markdown，只要 JSON 数组。"
+        )
+
+        try:
+            # 使用 nanobot agent (有工具访问权限)
+            from QuantNodes.agent.nanobot_bridge import Agent
+            agent = Agent(workspace=".agent")
+            response = asyncio.run(agent.run(prompt, session_id="g2-formula-gen"))
+            formulas = self._parse_llm_formulas(response, n)
+        except Exception as e:
+            logger.warning("[G2] LLM generation failed: %s, falling back to mock", e)
+            return self._generate_mock(n)
+
+        # 公式校验 + 重试
+        valid_formulas = []
+        for formula in formulas:
+            if self._validate_formula(formula):
+                valid_formulas.append(formula)
+            if len(valid_formulas) >= n:
+                break
+
+        # 如果 valid 不足，用 mock 补充
+        if len(valid_formulas) < n:
+            logger.warning(
+                "[G2] Only %d/%d valid formulas, supplementing with mock",
+                len(valid_formulas), n,
+            )
+            mock_factors = self._generate_mock(n - len(valid_formulas))
+            for f in mock_factors:
+                f.formula_id = f"G2_{len(valid_formulas):03d}"
+                valid_formulas.append(f.formula)
+
+        factors: List[FactorSpec] = []
+        for i, formula in enumerate(valid_formulas[:n]):
+            if isinstance(formula, str):
+                factors.append(
+                    FactorSpec(
+                        formula_id=f"G2_{i:03d}",
+                        formula=formula,
+                        source="g2_llm_only",
+                        category=self._infer_category(formula),
+                        complexity=formula.count("("),
+                        meta={"llm_generated": True},
+                    )
+                )
+            else:
+                # FactorSpec from mock supplement
+                formula.formula_id = f"G2_{i:03d}"
+                factors.append(formula)
+
+        logger.info("[G2] LLM generated %d factors (%d valid)", len(factors), len(valid_formulas))
+        return factors
+
+    @staticmethod
+    def _validate_formula(formula: str) -> bool:
+        """校验公式是否有效。"""
+        from QuantNodes.research.quant_alpha.operator_vocab import OperatorVocab
+        import polars as pl
+
+        test_data = pl.DataFrame({
+            "date": ["2020-01-01", "2020-01-02", "2020-01-03"] * 3,
+            "code": ["A"] * 3 + ["B"] * 3 + ["C"] * 3,
+            "open": [10.0] * 9,
+            "high": [10.5] * 9,
+            "low": [9.5] * 9,
+            "close": [10.0, 10.1, 10.2, 11.0, 11.1, 11.2, 12.0, 12.1, 12.2],
+            "vol": [1000.0] * 9,
+            "amount": [10000.0] * 9,
+        })
+
+        try:
+            vocab = OperatorVocab.default()
+            result = vocab.evaluate(
+                formula=formula,
+                data=test_data,
+                date_column="date",
+                code_column="code",
+            )
+            return result is not None and len(result) == len(test_data)
+        except Exception:
+            return False
+
+    @staticmethod
+    def _parse_llm_formulas(response: str, expected: int) -> List[str]:
+        """从 LLM 响应中解析公式列表。"""
+        import json
+        import re
+
+        # 尝试直接 JSON 解析
+        try:
+            result = json.loads(response)
+            if isinstance(result, list):
+                return [str(f) for f in result]
+        except json.JSONDecodeError:
+            pass
+
+        # 尝试从 markdown 代码块提取
+        match = re.search(r'```(?:json)?\s*(\[.*?\])\s*```', response, re.DOTALL)
+        if match:
+            try:
+                result = json.loads(match.group(1))
+                if isinstance(result, list):
+                    return [str(f) for f in result]
+            except json.JSONDecodeError:
+                pass
+
+        # 尝试提取所有看起来像公式的内容
+        formulas = re.findall(r'"([^"]+\([^"]+\)[^"]*)"', response)
+        if formulas:
+            return formulas
+
+        logger.warning("[G2] Failed to parse LLM response, using fallback")
+        return []
+
+    def _generate_mock(self, n: int) -> List[FactorSpec]:
+        """Stage 1: mock 公式生成。"""
+        rng = random.Random(self.seed)
+
+        factors: List[FactorSpec] = []
+        seen: set = set()
+
+        n_invalid = max(1, int(n * 0.15))
+        n_complex = max(1, int(n * 0.25))
+        n_simple = n - n_invalid - n_complex
+
+        attempts = 0
+        while len(factors) < n and attempts < n * 10:
+            attempts += 1
+            if len(factors) < n_simple:
+                formula = _gen_formula(rng, max_depth=1)
+            elif len(factors) < n_simple + n_complex:
+                formula = _gen_formula(rng, max_depth=2)
+            else:
+                formula = rng.choice(INVALID_LLM_TOKENS)
+
+            if formula in seen:
+                continue
+            seen.add(formula)
+
+            factors.append(
+                FactorSpec(
+                    formula_id=f"G2_{len(factors):03d}",
+                    formula=formula,
+                    source="g2_llm_only",
+                    category=self._infer_category(formula),
+                    complexity=formula.count("("),
+                    meta={"seed": self.seed, "valid": "rank" not in formula},
+                )
+            )
+
+        logger.info(
+            "[G2] generated %d factors (n_simple=%d, n_complex=%d, n_invalid=%d)",
+            len(factors),
+            min(n_simple, len(factors)),
+            max(0, min(n_complex, len(factors) - n_simple)),
+            max(0, len(factors) - n_simple - n_complex),
+        )
+        return factors
+
+    @staticmethod
+    def _infer_category(formula: str) -> str:
+        if "ts_mean" in formula and "delta" not in formula:
+            return "momentum"
+        if "delta" in formula:
+            return "momentum"
+        if "ts_std" in formula:
+            return "volatility"
+        if "vol" in formula:
+            return "volume"
+        if "abs" in formula:
+            return "reversal"
+        return "value"

QuantNodes/research/quant_alpha/evaluation/baselines/g3_alpha_gpt.py

@@ -0,0 +1,152 @@
+# coding=utf-8
+"""
+g3_alpha_gpt.py - G3 baseline：包 AlphaGptWorkflow（M5）
+
+G3 = "Alpha-GPT"，代表完整 5 智能体编排：idea-generator → formula-translator →
+evaluator → reflector → critic，5 轮迭代。
+
+Stage 1：用 mock LLM（_mock_llm_response）跑通 workflow
+Stage 2：注入 NanobotLLMWrapper(MiniMaxClient) 替换 mock
+
+复用：
+- QuantNodes.research.quant_alpha.workflow.AlphaGptWorkflow：完整 5 智能体编排
+- QuantNodes.research.quant_alpha.workflow.AlphaGptConfig：config dataclass
+- contracts.Baseline：generate_factors() 接口
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import List, Optional
+
+from ..contracts import Baseline, FactorSpec
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["G3AlphaGpt"]
+
+
+class G3AlphaGpt(Baseline):
+    """G3 Alpha-GPT baseline
+
+    包装 AlphaGptWorkflow（M5），跑完整 5 智能体编排。
+    Stage 1：用 mock LLM 跑通；Stage 2：注入真实 LLM client。
+    """
+
+    def __init__(
+        self,
+        n: int = 30,
+        objective: str = "maximize IC and IR for 1-day forward return",
+        iterations: int = 3,
+        pool_size: int = 10,
+        seed: int = 11,
+        llm_client=None,
+    ) -> None:
+        self.n = n
+        self.objective = objective
+        self.iterations = iterations
+        self.pool_size = pool_size
+        self.seed = seed
+        self._llm_client = llm_client
+        self._last_workflow_result = None  # 暴露给 runner 调试
+
+    @property
+    def group_name(self) -> str:
+        return "G3_AlphaGpt"
+
+    def generate_factors(self, n: Optional[int] = None) -> List[FactorSpec]:
+        """运行 AlphaGptWorkflow 并提取 final_pool 公式
+
+        Args:
+            n: 截取前 n 个因子（默认 self.n）
+
+        Returns:
+            FactorSpec 列表（公式 + 元信息）
+        """
+        n = n or self.n
+
+        from QuantNodes.research.quant_alpha.workflow.alpha_gpt import (
+            AlphaGptConfig,
+            AlphaGptWorkflow,
+        )
+
+        config = AlphaGptConfig(
+            objective=self.objective,
+            iterations=self.iterations,
+            pool_size=self.pool_size,
+            top_k=n,
+        )
+
+        logger.info(
+            "[G3] starting AlphaGptWorkflow (iterations=%d, pool_size=%d)",
+            self.iterations,
+            self.pool_size,
+        )
+
+        try:
+            # Stage 2: 注入真实 LLM client
+            # Stage 1: llm_client=None → AlphaGptWorkflow 使用 mock
+            workflow = AlphaGptWorkflow(config=config, llm_client=self._llm_client)
+            result = workflow.run()
+        except Exception as e:
+            logger.error("[G3] AlphaGptWorkflow failed: %s, using fallback", e)
+            # workflow 失败时, 用 mock fallback（与 workflow 返回空时的逻辑一致）
+            result = None
+
+        if result is not None:
+            self._last_workflow_result = result
+
+        factors: List[FactorSpec] = []
+        if result is not None:
+            for i, formula_rec in enumerate(result.final_pool[:n]):
+                factors.append(
+                    FactorSpec(
+                        formula_id=f"G3_{i:03d}",
+                        formula=formula_rec.formula,
+                        source="g3_alpha_gpt",
+                        category=formula_rec.category or "unknown",
+                        complexity=formula_rec.formula.count("("),
+                        meta={
+                            "rank": formula_rec.rank,
+                            "selection_reason": formula_rec.selection_reason,
+                            "round_discovered": formula_rec.round_discovered,
+                        },
+                    )
+                )
+
+        # Stage 1 mock 兼容：若 workflow 失败/返回空/final_pool 不足，
+        # 用 G1 风格的简单 valid 公式兜底（保证 baseline 数量稳定）
+        if len(factors) < n:
+            logger.warning(
+                "[G3] workflow returned %d factors (期望 %d), mock 兜底补充",
+                len(factors),
+                n,
+            )
+            from .g1_handcrafted import _gen_formula
+            import random
+
+            rng = random.Random(self.seed + 100)
+            while len(factors) < n:
+                formula = _gen_formula(rng)
+                if any(f.formula == formula for f in factors):
+                    continue
+                factors.append(
+                    FactorSpec(
+                        formula_id=f"G3_{len(factors):03d}",
+                        formula=formula,
+                        source="g3_alpha_gpt",
+                        category="g3_fallback_mock",
+                        complexity=formula.count("("),
+                        meta={"fallback": True, "seed": self.seed},
+                    )
+                )
+
+        if result is not None:
+            logger.info(
+                "[G3] AlphaGptWorkflow returned %d factors (total=%d, elapsed=%.2fs)",
+                len(factors),
+                result.total_formulas,
+                result.elapsed_seconds,
+            )
+
+        return factors

QuantNodes/research/quant_alpha/evaluation/clickhouse_data_loader.py

@@ -0,0 +1,227 @@
+# coding=utf-8
+"""
+clickhouse_data_loader.py - Stage 2 ClickHouse 数据加载器
+
+从 ClickHouse 加载全 A 股日线数据，支持本地 parquet 缓存。
+复用 database_node/clickhouse_node.py 的 CHBase 客户端。
+
+ClickHouse 表 schema (quote.stock_quote):
+    ts_code     LowCardinality(String)  → code
+    trade_date  DateTime                → date (cast Date)
+    open        Float64                 → open
+    high        Float64                 → high
+    low         Float64                 → low
+    close       Float64                 → close
+    vol         Float64                 → vol
+    amount      Float64                 → amount
+
+Stage 2 替代 MockDataLoader，接口完全兼容 DataLoader ABC。
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional
+
+import polars as pl
+
+from .contracts import DataLoader
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["ClickHouseDataLoader"]
+
+
+class ClickHouseDataLoader(DataLoader):
+    """Stage 2 ClickHouse 数据加载器
+
+    从 ClickHouse 加载全 A 股日线数据。
+    支持本地 parquet 缓存（首次查询后自动缓存）。
+
+    用法::
+
+        loader = ClickHouseDataLoader(
+            table="quote.stock_quote",
+            start_date="2019-01-01",
+            end_date="2024-12-31",
+        )
+        df = loader.load()  # polars.DataFrame
+    """
+
+    # ClickHouse → polars 字段映射
+    FIELD_MAP = {
+        "ts_code": "code",
+        "open": "open",
+        "high": "high",
+        "low": "low",
+        "close": "close",
+        "vol": "vol",
+        "amount": "amount",
+    }
+
+    def __init__(
+        self,
+        table: str = "quote.stock_quote",
+        host: str = "localhost",
+        port: int = 8123,
+        user: str = "data",
+        password: str = "123456",
+        database: str = "quote",
+        start_date: str = "2019-01-01",
+        end_date: str = "2024-12-31",
+        min_amount_percentile: float = 0.0,
+        cache_parquet: Optional[str] = "data/cache/full_a_2019_2024.parquet",
+    ) -> None:
+        self.table = table
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.database = database
+        self.start_date = start_date
+        self.end_date = end_date
+        self.min_amount_percentile = min_amount_percentile
+        self.cache_parquet = Path(cache_parquet) if cache_parquet else None
+
+    def load(self) -> pl.DataFrame:
+        """加载数据：优先读缓存 parquet，否则从 ClickHouse 查询并缓存。"""
+        if self.cache_parquet and self.cache_parquet.exists():
+            logger.info("[ClickHouseDataLoader] 读取缓存: %s", self.cache_parquet)
+            df = pl.read_parquet(self.cache_parquet)
+            logger.info("[ClickHouseDataLoader] 缓存加载完成: %s rows", df.height)
+            return df
+
+        logger.info(
+            "[ClickHouseDataLoader] 从 ClickHouse 查询: %s (%s ~ %s)",
+            self.table, self.start_date, self.end_date,
+        )
+        df = self._query_clickhouse()
+        df = self._clean(df)
+
+        if self.cache_parquet:
+            self.cache_parquet.parent.mkdir(parents=True, exist_ok=True)
+            df.write_parquet(self.cache_parquet)
+            logger.info(
+                "[ClickHouseDataLoader] 缓存已保存: %s (%s rows)",
+                self.cache_parquet, df.height,
+            )
+
+        return df
+
+    def _query_clickhouse(self) -> pl.DataFrame:
+        """从 ClickHouse 查询数据，返回 polars DataFrame。
+
+        P2.12c.3 (v3.0+): 重构 — 委托 ClickHouseNode (database_node) 而非 raw HTTP。
+        原实现使用 http.client.HTTPConnection 直接调用，绕过了 production-tested
+        CHBase 客户端和 ClickHouseNode 的连接池/错误处理。
+        """
+        from QuantNodes.database_node.clickhouse_node import ClickHouseNode
+
+        fields = ", ".join(
+            f"{ch_name} AS {pl_name}" for ch_name, pl_name in self.FIELD_MAP.items()
+        )
+        sql = (
+            f"SELECT {fields}, CAST(trade_date AS Date) AS date "
+            f"FROM {self.table} "
+            f"WHERE trade_date >= '{self.start_date}' "
+            f"AND trade_date <= '{self.end_date}' "
+            f"ORDER BY date, code"
+        )
+
+        logger.info("[ClickHouseDataLoader] SQL: %s", sql[:200])
+
+        node = ClickHouseNode(
+            host=self.host,
+            port=self.port,
+            user=self.user,
+            passwd=self.password,
+            database="default",
+        )
+
+        try:
+            pd_df = node.query(sql)
+        except Exception as e:
+            raise RuntimeError(f"ClickHouse query failed: {e}") from e
+
+        if pd_df is None or len(pd_df) == 0:
+            raise RuntimeError("ClickHouse returned empty result")
+
+        # 转 polars (Table 4 pipeline 是 polars-first)
+        df = pl.from_pandas(pd_df)
+        logger.info("[ClickHouseDataLoader] 查询完成: %s rows", df.height)
+        return df
+
+    def _clean(self, df: pl.DataFrame) -> pl.DataFrame:
+        """数据清洗：类型转换 + 过滤。"""
+        # 确保 date 列为 Date 类型
+        if df["date"].dtype == pl.Utf8:
+            df = df.with_columns(pl.col("date").str.to_date())
+        elif df["date"].dtype == pl.Datetime:
+            df = df.with_columns(pl.col("date").cast(pl.Date))
+
+        # 确保数值列为 Float64
+        for col in ["open", "high", "low", "close", "vol", "amount"]:
+            if col in df.columns:
+                df = df.with_columns(pl.col(col).cast(pl.Float64))
+
+        # 过滤停牌（vol == 0）
+        before = df.height
+        df = df.filter(pl.col("vol") > 0)
+        if df.height < before:
+            logger.info(
+                "[ClickHouseDataLoader] 过滤停牌: %d → %d rows",
+                before, df.height,
+            )
+
+        # 过滤低流动性（可选）
+        if self.min_amount_percentile > 0:
+            threshold = df["amount"].quantile(self.min_amount_percentile)
+            before = df.height
+            df = df.filter(pl.col("amount") >= threshold)
+            logger.info(
+                "[ClickHouseDataLoader] 过滤低流动性 (< %.0f): %d → %d rows",
+                threshold, before, df.height,
+            )
+
+        return df
+
+    def load_summary(self) -> dict:
+        """返回数据摘要（不加载全量数据）。
+
+        P2.12c.4: 重构 — 委托 ClickHouseNode.query() 而非 raw HTTP。
+        """
+        from QuantNodes.database_node.clickhouse_node import ClickHouseNode
+
+        sql = (
+            f"SELECT min(trade_date) as min_date, max(trade_date) as max_date, "
+            f"count() as total_rows, count(distinct ts_code) as n_stocks "
+            f"FROM {self.table} "
+            f"WHERE trade_date >= '{self.start_date}' "
+            f"AND trade_date <= '{self.end_date}'"
+        )
+
+        node = ClickHouseNode(
+            host=self.host,
+            port=self.port,
+            user=self.user,
+            passwd=self.password,
+            database="default",
+        )
+
+        try:
+            pd_df = node.query(sql)
+        except Exception as e:
+            logger.warning("[ClickHouseDataLoader] summary query failed: %s", e)
+            return {"error": "query failed"}
+
+        if pd_df is None or len(pd_df) == 0:
+            return {"error": "query failed"}
+
+        # pandas → dict (与原行为兼容: 返回第一行)
+        row = pd_df.iloc[0].to_dict()
+        # 处理 timestamp → str 转换 (避免 JSON 序列化失败)
+        for k, v in row.items():
+            if hasattr(v, "isoformat"):
+                row[k] = v.isoformat()
+        return row