quantnodes 3.0.0__py3-none-any.whl

QuantNodes/research/factor_test/nodes/preprocess_strategies.py

@@ -0,0 +1,277 @@
+# coding=utf-8
+"""Preprocess strategies (Phase 2.2, Strategy pattern).
+
+将原 factor_preprocess_node.py::_preprocess_vectorized 中 3 类硬编码 if
+分支 (missing fill / de-extreme / normalise) 抽象为独立的 Strategy 类:
+
+  MissingFillStrategy (ABC)
+    ├── PassThroughMissing        # 默认无操作
+    └── IndustryAverageMissing     # ind_avg: 行业内均值填充
+
+  DeExtremeStrategy (ABC)
+    ├── PassThroughExtreme         # 默认无操作
+    ├── MedianAbsoluteDeviationExtreme  # median ± n * MAD
+    └── PercentileShrinkExtreme    # quantile clip
+
+  NormStrategy (ABC)
+    ├── PassThroughNorm            # 默认无操作
+    ├── ZScoreNorm                 # (x - mean) / std
+    └── RankToNormalNorm           # rank → scipy.stats.norm.ppf
+
+  build_preprocess_strategies(missing, extreme, norm) -> (Missing, Extreme, Norm)
+
+每个 strategy 的 apply() 接受 in-place 输入 + 必要 kwargs, 返回处理后的
+DataFrame. _preprocess_vectorized 退化为简单的 3 行 dispatch.
+
+新增预处理类型 (如 winsorize 自定义 quantile / rank_to_uniform) 只需新增
+一个 Strategy 子类, _preprocess_vectorized 无需修改.
+"""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+from scipy.stats import norm as scipy_norm
+
+
+# ============================================================================
+# MissingFillStrategy
+# ============================================================================
+
+class MissingFillStrategy(ABC):
+    """缺失值填充策略. apply() 返回处理后的 DataFrame."""
+
+    name: str = ""
+
+    @abstractmethod
+    def apply(
+        self, result: pd.DataFrame, industry: Optional[pd.DataFrame] = None,
+    ) -> pd.DataFrame:
+        raise NotImplementedError
+
+
+class PassThroughMissing(MissingFillStrategy):
+    """默认: 不做缺失值填充."""
+
+    name = "passthrough"
+
+    def apply(
+        self, result: pd.DataFrame, industry: Optional[pd.DataFrame] = None,
+    ) -> pd.DataFrame:
+        return result
+
+
+class IndustryAverageMissing(MissingFillStrategy):
+    """ind_avg: 用 (date, industry) 组内的均值填充缺失值.
+
+    行为与原 _preprocess_vectorized line 120-141 一致.
+    """
+
+    name = "ind_avg"
+
+    def apply(
+        self, result: pd.DataFrame, industry: Optional[pd.DataFrame] = None,
+    ) -> pd.DataFrame:
+        if industry is None:
+            return result
+        industry_aligned = industry.reindex(
+            index=result.index, columns=result.columns,
+        )
+        # Stack to long format for groupby transform on (date, industry).
+        long = pd.DataFrame({
+            "factor": result.stack(),
+            "ind": industry_aligned.stack(),
+        }).dropna(subset=["ind"])
+        long = long[long["ind"] > 0]
+        if long.empty:
+            return result
+        date_level = long.index.get_level_values(0)
+        group_mean = long.groupby([date_level, "ind"])["factor"].transform("mean")
+        filled = long["factor"].fillna(group_mean)
+        if isinstance(filled.index, pd.MultiIndex):
+            filled_wide = filled.unstack(level=1)
+        else:
+            filled_wide = filled.unstack(level=-1)
+        result.loc[filled_wide.index, filled_wide.columns] = filled_wide.values
+        return result
+
+
+# ============================================================================
+# DeExtremeStrategy
+# ============================================================================
+
+class DeExtremeStrategy(ABC):
+    """去极值策略. apply() 返回处理后的 DataFrame."""
+
+    name: str = ""
+
+    @abstractmethod
+    def apply(
+        self, result: pd.DataFrame, *,
+        mad_n: float = 3.0, pct_low: float = 0.01, pct_high: float = 0.99,
+    ) -> pd.DataFrame:
+        raise NotImplementedError
+
+
+class PassThroughExtreme(DeExtremeStrategy):
+    """默认: 不去极值."""
+
+    name = "passthrough"
+
+    def apply(
+        self, result: pd.DataFrame, *,
+        mad_n: float = 3.0, pct_low: float = 0.01, pct_high: float = 0.99,
+    ) -> pd.DataFrame:
+        return result
+
+
+class MedianAbsoluteDeviationExtreme(DeExtremeStrategy):
+    """median: 用 median ± n * MAD 截断.
+
+    行为与原 _preprocess_vectorized line 144-151 一致.
+    """
+
+    name = "median"
+
+    def apply(
+        self, result: pd.DataFrame, *,
+        mad_n: float = 3.0, pct_low: float = 0.01, pct_high: float = 0.99,
+    ) -> pd.DataFrame:
+        median_per_row = result.median(axis=1)
+        mad_per_row = (result.sub(median_per_row, axis=0)).abs().median(axis=1)
+        lower = median_per_row - mad_n * mad_per_row
+        upper = median_per_row + mad_n * mad_per_row
+        return result.clip(lower=lower, upper=upper, axis=0)
+
+
+class PercentileShrinkExtreme(DeExtremeStrategy):
+    """pct_shrink: 用 [pct_low, pct_high] 分位数截断.
+
+    行为与原 _preprocess_vectorized line 152-155 一致.
+    """
+
+    name = "pct_shrink"
+
+    def apply(
+        self, result: pd.DataFrame, *,
+        mad_n: float = 3.0, pct_low: float = 0.01, pct_high: float = 0.99,
+    ) -> pd.DataFrame:
+        q1 = result.quantile(pct_low, axis=1)
+        q2 = result.quantile(pct_high, axis=1)
+        return result.clip(lower=q1, upper=q2, axis=0)
+
+
+# ============================================================================
+# NormStrategy
+# ============================================================================
+
+class NormStrategy(ABC):
+    """标准化策略. apply() 返回处理后的 DataFrame."""
+
+    name: str = ""
+
+    @abstractmethod
+    def apply(self, result: pd.DataFrame) -> pd.DataFrame:
+        raise NotImplementedError
+
+
+class PassThroughNorm(NormStrategy):
+    """默认: 不标准化."""
+
+    name = "passthrough"
+
+    def apply(self, result: pd.DataFrame) -> pd.DataFrame:
+        return result
+
+
+class ZScoreNorm(NormStrategy):
+    """zscore: (x - mean) / std. ddof=1 与原代码一致 (line 160)."""
+
+    name = "zscore"
+
+    def apply(self, result: pd.DataFrame) -> pd.DataFrame:
+        mean_per_row = result.mean(axis=1)
+        std_per_row = result.std(axis=1, ddof=1)
+        std_per_row = std_per_row.replace(0, np.nan)
+        return result.sub(mean_per_row, axis=0).div(std_per_row, axis=0)
+
+
+class RankToNormalNorm(NormStrategy):
+    """norm: 排名 → scipy.stats.norm.ppf. 边界 rank=0/1 clip 到 (min/2, (max+1)/2).
+
+    行为与原 _preprocess_vectorized line 163-184 一致.
+    """
+
+    name = "norm"
+
+    def apply(self, result: pd.DataFrame) -> pd.DataFrame:
+        ranks = result.rank(axis=1, pct=True)
+        # Clip ranks away from 0 and 1 (avoid -inf/+inf from ppf).
+        min_rank = ranks.where(ranks > 0).min(axis=1) * 0.5
+        max_rank = (ranks.where(ranks < 1).max(axis=1) + 1) * 0.5
+        # Default fallback for rows where min_rank/max_rank is NaN.
+        min_rank = min_rank.fillna(0.01)
+        max_rank = max_rank.fillna(0.99)
+        lower = pd.DataFrame(
+            np.broadcast_to(min_rank.values[:, None], ranks.shape),
+            index=ranks.index, columns=ranks.columns,
+        )
+        upper = pd.DataFrame(
+            np.broadcast_to(max_rank.values[:, None], ranks.shape),
+            index=ranks.index, columns=ranks.columns,
+        )
+        ranks = ranks.where(ranks.notna(), np.nan)  # preserve NaN
+        ranks = ranks.clip(lower=lower, upper=upper, axis=1)
+        return pd.DataFrame(
+            scipy_norm.ppf(ranks.values, 0, 1),
+            index=ranks.index, columns=ranks.columns,
+        )
+
+
+# ============================================================================
+# Factory
+# ============================================================================
+
+def build_missing_strategy(missing: str) -> MissingFillStrategy:
+    """根据配置名构造 MissingFillStrategy. 未知值 → PassThroughMissing."""
+    table = {
+        "ind_avg": IndustryAverageMissing,
+    }
+    cls = table.get(missing, PassThroughMissing)
+    return cls()
+
+
+def build_extreme_strategy(extreme: str) -> DeExtremeStrategy:
+    """根据配置名构造 DeExtremeStrategy. 未知值 → PassThroughExtreme."""
+    table = {
+        "median": MedianAbsoluteDeviationExtreme,
+        "pct_shrink": PercentileShrinkExtreme,
+    }
+    cls = table.get(extreme, PassThroughExtreme)
+    return cls()
+
+
+def build_norm_strategy(norm: str) -> NormStrategy:
+    """根据配置名构造 NormStrategy. 未知值 → PassThroughNorm."""
+    table = {
+        "zscore": ZScoreNorm,
+        "norm": RankToNormalNorm,
+    }
+    cls = table.get(norm, PassThroughNorm)
+    return cls()
+
+
+def build_preprocess_strategies(
+    missing: str, extreme: str, norm: str,
+) -> Tuple[MissingFillStrategy, DeExtremeStrategy, NormStrategy]:
+    """一次性构造 3 个 strategy.
+
+    Phase 2.2: 替代 _preprocess_vectorized 内的硬编码 if 链.
+    """
+    return (
+        build_missing_strategy(missing),
+        build_extreme_strategy(extreme),
+        build_norm_strategy(norm),
+    )

QuantNodes/research/factor_test/nodes/risk_correlation_node.py

@@ -0,0 +1,112 @@
+# coding: utf-8
+"""Node 11: 风险因子相关性 / Risk Correlation Node
+
+Migrated from factor_performance.py:879-937 corr_riskfactor()
+Security: exec()/eval() eliminated.
+"""
+
+import logging
+
+import numpy as np
+import pandas as pd
+
+from QuantNodes.research.factor_test.nodes._base import PydanticConfigNode
+from QuantNodes.research.factor_test.nodes.configs import RiskCorrelationNodeConfig
+
+logger = logging.getLogger(__name__)
+
+
+class RiskCorrelationNode(PydanticConfigNode):
+    """与风险因子的 Spearman 秩相关 + 稳定系数
+
+    输入: factor_neutral, risk_factors
+    输出: {mean: DataFrame, stability: DataFrame}
+    """
+
+    ConfigSchema = RiskCorrelationNodeConfig
+    _ALIASES = {"_factors": "factors"}
+
+    def _execute(self, input_data=None, **kwargs) -> dict:
+        context = kwargs.get('context', {})
+        factor_data = self._factor_data(context)
+        if factor_data is None:
+            raise ValueError("因子数据缺失")
+
+        loader = self._ctx_load(context, '_loader')
+        if loader is None:
+            raise ValueError("数据加载器缺失")
+
+        return self._calc_risk_correlation(factor_data, loader, self._factors)
+
+    def _calc_risk_correlation(self, factor_data, loader, risk_factors_config):
+        """计算与风险因子的相关性"""
+        adj_dates = factor_data.index.tolist()
+
+        # 确定风险因子列表
+        if risk_factors_config == 'all':
+            try:
+                risk_keys = loader.get_apikeys('risk_factor.h5')
+                risk_keys = [k[1:] for k in risk_keys]  # 去掉前导 /
+                risk_factors = [('risk_factor.h5', k) for k in risk_keys]
+            except Exception:
+                risk_factors = []
+        else:
+            risk_factors = risk_factors_config
+
+        if not risk_factors:
+            return {'mean': pd.DataFrame(), 'stability': pd.DataFrame()}
+
+        # 加载风险因子 (安全版本: 字典查找替代 exec)
+        risk_data_dict = {}
+        for file_key, factor_key in risk_factors:
+            try:
+                if file_key == 'risk_factor.h5':
+                    rf = loader.load_h5(file_key, factor_key)
+                else:
+                    rf = loader.load_custom((file_key, factor_key))
+                if loader.valid_shape(rf):
+                    rf = loader.add_index(rf)
+                risk_data_dict[factor_key] = rf
+            except Exception as e:
+                logger.warning(f"加载风险因子 {factor_key} 失败: {e}")
+
+        if not risk_data_dict:
+            return {'mean': pd.DataFrame(), 'stability': pd.DataFrame()}
+
+        risk_names = list(risk_data_dict.keys())
+
+        # 计算每期相关系数
+        corr_all_list = []
+        for date_i in adj_dates:
+            risk_i = pd.DataFrame()
+            for name in risk_names:
+                rf = risk_data_dict[name]
+                if date_i in rf.index:
+                    risk_i[name] = rf.loc[date_i]
+
+            factor_i = factor_data.loc[date_i]
+            merged = pd.merge(
+                factor_i.to_frame('factor'), risk_i,
+                left_index=True, right_index=True
+            )
+            if len(merged) > 2:
+                corr_i = merged.corr(method='spearman')
+                corr_i['date'] = date_i
+                corr_all_list.append(corr_i)
+
+        if not corr_all_list:
+            return {'mean': pd.DataFrame(), 'stability': pd.DataFrame()}
+
+        corr_all = pd.concat(corr_all_list)
+        corr_all['group'] = corr_all['date']
+
+        corr_mean = corr_all.groupby('group').mean()
+        corr_std = corr_all.groupby('group').std(ddof=1)
+        corr_stab = corr_mean / corr_std
+        corr_stab = corr_stab.reindex(index=corr_mean.columns.tolist())
+        corr_stab.replace(np.inf, np.nan, inplace=True)
+
+        return {
+            'mean': corr_mean,
+            'stability': corr_stab,
+        }

QuantNodes/research/factor_test/nodes/sample_pool_filter_node.py

@@ -0,0 +1,110 @@
+# coding: utf-8
+"""Node 2: 样本池筛选 / Sample Pool Filter Node
+
+Migrated from factor_utils.py:155-234 select_range()
+"""
+
+import numpy as np
+import pandas as pd
+
+from QuantNodes.research.factor_test.nodes._base import PydanticConfigNode
+from QuantNodes.research.factor_test.nodes.configs import SamplePoolNodeConfig
+from QuantNodes.research.factor_test.utils.constants import (
+    resolve_index_mapping, resolve_industry_map,
+)
+
+
+class SamplePoolFilterNode(PydanticConfigNode):
+    """根据指数范围和行业筛选样本池
+
+    输入: context["LoadData"] 的输出
+    输出: stock_sample (1=选中, nan=剔除)
+
+    M9: index_mapping 可自定义，合并全局默认 + 节点自定义覆盖
+    M12: i18n_name_map 可自定义行业代码→名称映射，合并全局默认 + 节点自定义覆盖
+    """
+
+    ConfigSchema = SamplePoolNodeConfig
+    _ALIASES = {
+        "_sample_index": "sample_index",
+        "_sample_industry": "sample_industry",
+        "_sample_customdir": "sample_index_customdir",
+    }
+
+    def __init__(self, name="SamplePoolFilter", config=None, **kwargs):
+        super().__init__(name, config, **kwargs)
+        # M9: 合并全局默认 INDEX_MAPPING + 节点自定义覆盖
+        self._index_mapping = resolve_index_mapping({
+            "INDEX_MAPPING": self.cfg.index_mapping
+        } if self.cfg.index_mapping else None)
+        # M12: 合并全局默认 INDUSTRY_MAPPING + 节点自定义覆盖
+        self._i18n_name_map = resolve_industry_map({
+            "INDUSTRY_MAP": self.cfg.i18n_name_map
+        } if self.cfg.i18n_name_map else None)
+
+    def _execute(self, input_data=None, **kwargs) -> pd.DataFrame:
+        context = kwargs.get('context', {})
+        load_data = context.get('LoadData') or input_data or {}
+
+        stklist = load_data['stklist']
+        trade_dt = load_data['trade_dt']
+        loader = load_data['_loader']
+
+        n_dt = len(trade_dt)
+        n_stk = len(stklist)
+
+        # 指数筛选
+        index_filt = np.ones((n_dt, n_stk))
+        if self._sample_index != 'all':
+            if self._sample_index in self._index_mapping:
+                h5_file, key = self._index_mapping[self._sample_index]
+                if_index = loader.load_h5(h5_file, key)
+                if_index = if_index.replace(np.nan, 0)
+                index_filt = index_filt * if_index.values
+            elif self._sample_index == 'ZZ800':
+                if_300 = loader.load_h5('stk_daily.h5', 'id_300').replace(np.nan, 0)
+                if_500 = loader.load_h5('stk_daily.h5', 'id_500').replace(np.nan, 0)
+                index_filt = index_filt * (if_300 + if_500).values
+            elif self._sample_index == 'custom' and self._sample_customdir:
+                if_index = loader.load_custom(self._sample_customdir)
+                if_index = if_index.replace(np.nan, 0)
+                index_filt = index_filt * if_index.values
+            else:
+                raise ValueError(f"不支持的指数: {self._sample_index}")
+
+         # 行业筛选
+        industry_filt = np.ones((n_dt, n_stk))
+        if self._sample_industry != 'all':
+            if isinstance(self._sample_industry, tuple):
+                ind_key, ind_name = self._sample_industry
+            else:
+                ind_key = 'id_citic1'
+                ind_name = self._sample_industry
+
+            id_industry = loader.load_h5('stk_daily.h5', ind_key)
+            # M12: i18n 映射: self._i18n_name_map
+            if self._i18n_name_map and ind_key in self._i18n_name_map:
+                ind_name = self._i18n_name_map[ind_key]
+            id_name_data = loader.load_h5(
+                'stk_daily.h5', f'ind_name_{ind_key.replace("id_", "").upper()}'
+            )
+
+            # 找到行业名称对应的编号
+            ind_idx = np.where(id_name_data.values == ind_name)[0]
+            if len(ind_idx) > 0:
+                ind_num = ind_idx[0] + 1  # 行业编号从 1 开始
+                industry_filt[id_industry.values != ind_num] = 0
+            else:
+                raise ValueError(f"行业 '{ind_name}' 未找到")
+
+        # 合并筛选
+        stock_sample = index_filt * industry_filt
+        stock_sample[stock_sample > 0] = 1
+        stock_sample = pd.DataFrame(
+            stock_sample,
+            index=trade_dt.iloc[:, 0].values,
+            columns=stklist.iloc[:, 0].values,
+        )
+        stock_sample = stock_sample.replace(0, np.nan)
+
+        return stock_sample

QuantNodes/research/factor_test/nodes/tradability_filter_node.py

@@ -0,0 +1,92 @@
+# coding: utf-8
+"""Node 3: 可交易性筛选 / Tradability Filter Node
+
+Migrated from factor_utils.py:250-308 valid_tradable()
+Security: exec() replaced with dict lookup.
+"""
+
+import numpy as np
+import pandas as pd
+
+from QuantNodes.research.factor_test.nodes._base import PydanticConfigNode
+from QuantNodes.research.factor_test.nodes.configs import TradabilityNodeConfig
+
+
+class TradabilityFilterNode(PydanticConfigNode):
+    """验证股票可交易性 (ST/停牌/涨跌停/新股/自定义追踪)
+
+    输入: context["LoadData"] 的输出 + context["SamplePoolFilter"] 的输出
+    输出: tradable (1=可交易, nan=不可)
+    """
+
+    ConfigSchema = TradabilityNodeConfig
+    _ALIASES = {"_tradable_setting": "tradable"}
+
+    def _execute(self, input_data=None, **kwargs) -> pd.DataFrame:
+        context = kwargs.get('context', {})
+        load_data = self._ctx(context)
+        sample = context.get('SamplePoolFilter')
+
+        loader = load_data['_loader']
+        stklist = load_data['stklist']
+        trade_dt = load_data['trade_dt']
+
+        n_dt = len(trade_dt)
+        n_stk = len(stklist)
+
+        # 加载可交易性数据 (优先从 context, 回退到 loader)
+        def _get(key):
+            if key in load_data:
+                return load_data[key]
+            return loader.load_h5('stk_daily.h5', key) if loader else None
+
+        st = _get('st')
+        suspend = _get('suspend')
+        ud_limit = _get('ud_limit')
+        if ud_limit is not None:
+            ud_limit = ud_limit.abs()
+        ipo_days = _get('ipo_days')
+
+        # 初始化: 全部可交易
+        if_tradable = pd.DataFrame(
+            np.ones((n_dt, n_stk)),
+            index=trade_dt.iloc[:, 0].values,
+            columns=stklist.iloc[:, 0].values
+        )
+
+        s = self._tradable_setting
+
+        if s.no_st:
+            if_tradable[st == 1] = np.nan
+
+        if s.no_suspended:
+            if_tradable[suspend == 1] = np.nan
+
+        if s.no_up_down_limit:
+            if_tradable[ud_limit == 1.0] = np.nan
+
+        if s.min_ipo_days:
+            if_tradable[ipo_days < s.min_ipo_days] = np.nan
+
+        # 自定义追踪条件 (安全版本: 字典查找替代 exec)
+        if s.trace:
+            trace_data_map = {
+                'suspend': suspend,
+                'st': st,
+                'ud_limit': ud_limit,
+            }
+            for trace_key, (m, n) in s.trace.items():
+                if trace_key not in trace_data_map:
+                    raise ValueError(f"不支持的追踪条件: {trace_key}")
+                trace_data = trace_data_map[trace_key]
+                if m > 0:
+                    trace_sum = trace_data.rolling(m).sum().shift(1)
+                else:
+                    trace_sum = trace_data.rolling(-m).sum().shift(m)
+                if_tradable[trace_sum >= n] = np.nan
+
+        # 合并样本池筛选
+        if sample is not None:
+            if_tradable = if_tradable * sample
+
+        return if_tradable