quantnodes 3.0.0__py3-none-any.whl

QuantNodes/core/data_preprocessing.py

@@ -0,0 +1,453 @@
+# coding=utf-8
+"""
+数据预处理工具类
+
+替代 QuantStudio.FactorDataBase.FactorOperation.DataPreprocessingFun
+提供因子数据预处理的各种操作：标准化、去极值、缺失值处理、正交化等
+"""
+
+import numpy as np
+import pandas as pd
+from scipy import stats
+from typing import Optional, Callable
+
+
+class DataPreprocessingFun:
+    """
+    数据预处理函数集合
+
+    提供因子数据预处理的各种静态方法，包括：
+    - 标准化：Z-Score、Rank、分位数
+    - 去极值：Winsorize
+    - 缺失值填充：常量、函数、回归
+    - 正交化：Gram-Schmidt
+    """
+
+    @staticmethod
+    def regressChangeRate(x: np.ndarray) -> np.ndarray:
+        """
+        回归变化率
+
+        对输入数据进行回归分析，返回回归系数作为变化率度量
+        """
+        if not isinstance(x, np.ndarray):
+            x = np.array(x)
+
+        if x.ndim < 2:
+            return x
+
+        n_factor = x.shape[0] if x.ndim > 2 else 1
+        result = np.zeros(x.shape[1:]) if x.ndim > 2 else np.zeros(x.shape)
+
+        if x.ndim > 2:
+            for i in range(x.shape[1]):
+                for j in range(x.shape[2]):
+                    y = x[0, i, j] if x.shape[0] > 0 else x[i, j]
+                    if n_factor > 1:
+                        X = np.vstack([x[k, i, j] for k in range(1, n_factor)]).T
+                        X = np.column_stack([np.ones(len(X)), X])
+                        try:
+                            beta = np.linalg.lstsq(X, y, rcond=None)[0]
+                            result[i, j] = beta[1] if len(beta) > 1 else beta[0]
+                        except Exception:
+                            result[i, j] = np.nan
+                    else:
+                        result[i, j] = y[-1] - y[0] if len(y) > 1 else y[0]
+        else:
+            for i in range(x.shape[1]):
+                y = x[:, i]
+                t = np.arange(len(y))
+                try:
+                    beta = np.polyfit(t, y, 1)[0]
+                    result[i] = beta
+                except Exception:
+                    result[i] = np.nan
+
+        return result
+
+    @staticmethod
+    def standardizeZScore(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        avg_weight: Optional[np.ndarray] = None,
+        dispersion_weight: Optional[np.ndarray] = None,
+        avg_statistics: str = "平均值",
+        dispersion_statistics: str = "标准差",
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """Z-Score 标准化"""
+        result = data.copy().astype(float)
+        valid_mask = ~np.isnan(data)
+
+        if mask is not None:
+            valid_mask = valid_mask & (mask == 1)
+
+        if cat_data is None:
+            if avg_weight is not None and dispersion_weight is not None:
+                mean_val = np.nansum(data * avg_weight) / np.nansum(avg_weight * valid_mask)
+                weighted_diff_sq = ((data - mean_val) * avg_weight) ** 2
+                std_val = np.sqrt(
+                    np.nansum(weighted_diff_sq) / np.nansum(avg_weight * valid_mask)
+                )
+            else:
+                mean_val = np.nanmean(data[valid_mask]) if valid_mask.any() else 0
+                std_val = np.nanstd(data[valid_mask]) if valid_mask.any() else 1
+
+            if std_val == 0 or np.isnan(std_val):
+                std_val = 1
+
+            result[valid_mask] = (data[valid_mask] - mean_val) / std_val
+            result[~valid_mask] = np.nan
+        else:
+            unique_cats = np.unique(cat_data[~np.isnan(cat_data)])
+            for cat in unique_cats:
+                cat_mask = valid_mask & (cat_data == cat)
+                if cat_mask.any():
+                    cat_data_arr = data[cat_mask]
+                    if avg_weight is not None:
+                        w = avg_weight[cat_mask]
+                        cat_weight = w * ~np.isnan(cat_data_arr)
+                        mean_val = np.nansum(cat_data_arr * w) / np.nansum(cat_weight)
+                        cat_diff_sq = ((cat_data_arr - mean_val) * w) ** 2
+                        std_val = np.sqrt(np.nansum(cat_diff_sq) / np.nansum(cat_weight))
+                    else:
+                        mean_val = np.nanmean(cat_data_arr)
+                        std_val = np.nanstd(cat_data_arr)
+
+                    if std_val == 0 or np.isnan(std_val):
+                        std_val = 1
+
+                    result[cat_mask] = (data[cat_mask] - mean_val) / std_val
+
+            uncategorized_mask = valid_mask & np.isnan(cat_data)
+            if uncategorized_mask.any():
+                result[uncategorized_mask] = np.nan
+
+        return result
+
+    @staticmethod
+    def standardizeRank(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        ascending: bool = True,
+        uniformization: bool = True,
+        perturbation: bool = False,
+        offset: float = 0.5,
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """Rank 标准化"""
+        result = np.zeros_like(data, dtype=float)
+        valid_mask = ~np.isnan(data)
+
+        if mask is not None:
+            valid_mask = valid_mask & (mask == 1)
+
+        if cat_data is None:
+            valid_data = data[valid_mask]
+            if len(valid_data) > 0:
+                ranks = stats.rankdata(valid_data, ascending=ascending)
+                if uniformization:
+                    ranks = ranks / (len(ranks) + 1)
+                if perturbation:
+                    ranks = ranks + np.random.uniform(-offset, offset, len(ranks))
+                result[valid_mask] = ranks
+            result[~valid_mask] = np.nan
+        else:
+            unique_cats = np.unique(cat_data[~np.isnan(cat_data)])
+            for cat in unique_cats:
+                cat_mask = valid_mask & (cat_data == cat)
+                valid_cat_data = data[cat_mask]
+                if len(valid_cat_data) > 0:
+                    ranks = stats.rankdata(valid_cat_data, ascending=ascending)
+                    if uniformization:
+                        ranks = ranks / (len(ranks) + 1)
+                    if perturbation:
+                        ranks = ranks + np.random.uniform(-offset, offset, len(ranks))
+                    result[cat_mask] = ranks
+
+            uncategorized_mask = valid_mask & np.isnan(cat_data)
+            if uncategorized_mask.any():
+                result[uncategorized_mask] = np.nan
+
+        return result
+
+    @staticmethod
+    def fillNaNByValue(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        fill_value: float = 0,
+        fill_method: str = "常数",
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """按值填充 NaN"""
+        result = data.copy()
+        nan_mask = np.isnan(data)
+
+        if mask is not None:
+            nan_mask = nan_mask & (mask == 1)
+
+        if fill_method == "常数":
+            result[nan_mask] = fill_value
+        elif fill_method == "均值":
+            valid_data = data[~nan_mask]
+            fill_val = np.nanmean(valid_data) if len(valid_data) > 0 else fill_value
+            result[nan_mask] = fill_val
+        elif fill_method == "中位数":
+            valid_data = data[~nan_mask]
+            fill_val = np.nanmedian(valid_data) if len(valid_data) > 0 else fill_value
+            result[nan_mask] = fill_val
+        elif fill_method == "前向填充":
+            df = pd.DataFrame(result)
+            result = df.ffill().values
+            result[nan_mask] = np.nan
+        elif fill_method == "后向填充":
+            df = pd.DataFrame(result)
+            result = df.bfill().values
+            result[nan_mask] = np.nan
+
+        return result
+
+    @staticmethod
+    def winsorize(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        winsorize_lower: float = 0.01,
+        winsorize_upper: float = 0.01,
+        fill_value: Optional[float] = None,
+        fill_method: str = "均值方差",
+        boundary_method: str = "边界值",
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """Winsorize 去极值处理"""
+        if not (0 <= winsorize_lower <= 1) or not (0 <= winsorize_upper <= 1):
+            raise ValueError(
+                f"winsorize bounds must be in [0, 1], "
+                f"got lower={winsorize_lower}, upper={winsorize_upper}"
+            )
+        if winsorize_lower + winsorize_upper > 1:
+            total = winsorize_lower + winsorize_upper
+            raise ValueError(
+                f"winsorize_lower + winsorize_upper must be <= 1, got {total}"
+            )
+
+        result = data.copy().astype(float)
+        valid_mask = ~np.isnan(data)
+
+        if mask is not None:
+            valid_mask = valid_mask & (mask == 1)
+
+        if cat_data is None:
+            valid_data = data[valid_mask]
+            if len(valid_data) > 0:
+                lower_bound = np.nanpercentile(valid_data, winsorize_lower * 100)
+                upper_bound = np.nanpercentile(valid_data, (1 - winsorize_upper) * 100)
+
+                if fill_method == "均值方差":
+                    mean_val = np.nanmean(valid_data)
+                    std_val = np.nanstd(valid_data)
+                    result[valid_mask & (data < lower_bound)] = mean_val - 2 * std_val
+                    result[valid_mask & (data > upper_bound)] = mean_val + 2 * std_val
+                else:
+                    result[valid_mask & (data < lower_bound)] = lower_bound
+                    result[valid_mask & (data > upper_bound)] = upper_bound
+        else:
+            unique_cats = np.unique(cat_data[~np.isnan(cat_data)])
+            for cat in unique_cats:
+                cat_mask = valid_mask & (cat_data == cat)
+                valid_cat_data = data[cat_mask]
+                if len(valid_cat_data) > 0:
+                    lower_bound = np.nanpercentile(valid_cat_data, winsorize_lower * 100)
+                    upper_bound = np.nanpercentile(valid_cat_data, (1 - winsorize_upper) * 100)
+
+                    if fill_method == "均值方差":
+                        mean_val = np.nanmean(valid_cat_data)
+                        std_val = np.nanstd(valid_cat_data)
+                        result[cat_mask & (data < lower_bound)] = mean_val - 2 * std_val
+                        result[cat_mask & (data > upper_bound)] = mean_val + 2 * std_val
+                    else:
+                        result[cat_mask & (data < lower_bound)] = lower_bound
+                        result[cat_mask & (data > upper_bound)] = upper_bound
+
+        return result
+
+    @staticmethod
+    def standardizeQuantile(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        ascending: bool = True,
+        perturbation: bool = False,
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """分位数标准化"""
+        return DataPreprocessingFun.standardizeRank(
+            data=data,
+            mask=mask,
+            cat_data=cat_data,
+            ascending=ascending,
+            uniformization=True,
+            perturbation=perturbation,
+            offset=0.5,
+            other_handle=other_handle,
+            **kwargs
+        )
+
+    @staticmethod
+    def fillNaNByFun(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        val_fun: Optional[Callable] = None,
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """按函数值填充 NaN"""
+        result = data.copy().astype(float)
+        nan_mask = np.isnan(data)
+
+        if mask is not None:
+            nan_mask = nan_mask & (mask == 1)
+
+        if val_fun is None:
+            val_fun = lambda x, n: np.zeros(n) + np.nanmean(x)
+
+        n = np.sum(nan_mask)
+        if n > 0:
+            non_nan_data = data[~nan_mask]
+            if len(non_nan_data) > 0:
+                fill_values = val_fun(non_nan_data, n)
+            else:
+                fill_values = np.zeros(n) + np.nan
+            result[nan_mask] = fill_values[:n]
+
+        return result
+
+    @staticmethod
+    def fillNaNByRegress(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        X: Optional[np.ndarray] = None,
+        intercept: bool = True,
+        weight_data: Optional[np.ndarray] = None,
+        dummy_data: Optional[np.ndarray] = None,
+        other_handle: str = "填充None",
+        **kwargs
+    ) -> np.ndarray:
+        """按回归预测值填充 NaN"""
+        result = data.copy().astype(float)
+        nan_mask = np.isnan(data)
+
+        if mask is not None:
+            nan_mask = nan_mask & (mask == 1)
+
+        if X is None or X.size == 0:
+            return result
+
+        valid_rows = ~nan_mask
+        nan_rows = nan_mask
+
+        if np.sum(valid_rows) < 2 or np.sum(nan_rows) < 1:
+            return result
+
+        X_valid = X[valid_rows] if X.ndim > 1 else X[valid_rows].reshape(-1, 1)
+        y_valid = data[valid_rows]
+
+        if intercept:
+            X_valid = np.column_stack([np.ones(len(X_valid)), X_valid])
+
+        try:
+            beta = np.linalg.lstsq(X_valid, y_valid, rcond=None)[0]
+
+            X_nan = X[nan_rows] if X.ndim > 1 else X[nan_rows].reshape(-1, 1)
+            if intercept:
+                X_nan = np.column_stack([np.ones(len(X_nan)), X_nan])
+
+            y_pred = X_nan @ beta
+            result[nan_rows] = y_pred
+        except Exception:
+            pass
+
+        return result
+
+    @staticmethod
+    def orthogonalize(
+        data: np.ndarray,
+        mask: Optional[np.ndarray] = None,
+        cat_data: Optional[np.ndarray] = None,
+        X: Optional[np.ndarray] = None,
+        method: str = "gram_schmidt",
+        weight_data: Optional[np.ndarray] = None,
+        dummy_data: Optional[np.ndarray] = None,
+        other_handle: str = "填充None",
+        intercept: bool = True,
+        **kwargs
+    ) -> np.ndarray:
+        """正交化处理"""
+        result = data.copy().astype(float)
+        valid_mask = ~np.isnan(data)
+
+        if mask is not None:
+            valid_mask = valid_mask & (mask == 1)
+
+        if X is None or X.size == 0:
+            return result
+
+        if method == "gram_schmidt":
+            for i in range(result.shape[0]):
+                row_mask = valid_mask[i]
+                if not row_mask.any():
+                    continue
+
+                x_valid = X[row_mask]
+                y_valid = result[i, row_mask]
+
+                if len(x_valid) < 2:
+                    continue
+
+                try:
+                    x_mean = np.nanmean(x_valid, axis=0)
+                    x_centered = x_valid - x_mean
+                    norm = np.linalg.norm(x_centered, axis=0)
+                    norm[norm == 0] = 1
+                    x_orthogonal = x_centered / norm
+
+                    beta = np.linalg.lstsq(x_orthogonal, y_valid, rcond=None)[0]
+                    y_pred = x_orthogonal @ beta
+                    result[i, row_mask] = y_valid - y_pred + np.nanmean(y_valid)
+                except Exception:
+                    continue
+        else:
+            for i in range(result.shape[0]):
+                row_mask = valid_mask[i]
+                if not row_mask.any():
+                    continue
+
+                x_valid = X[row_mask]
+                y_valid = result[i, row_mask]
+
+                if len(x_valid) < 2:
+                    continue
+
+                try:
+                    if intercept:
+                        X_design = np.column_stack([np.ones(len(x_valid)), x_valid])
+                    else:
+                        X_design = x_valid
+
+                    beta = np.linalg.lstsq(X_design, y_valid, rcond=None)[0]
+                    y_pred = X_design @ beta
+                    result[i, row_mask] = y_valid - y_pred + np.nanmean(y_valid)
+                except Exception:
+                    continue
+
+        return result

QuantNodes/core/data_source.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""DataSource - 顶层数据源标记基类 (Phase 3.3)
+
+QuantNodes 有两个独立的数据接入子树:
+
+  - ``FileFormatLoader`` (文件): 按 path/key 产出 pd.DataFrame
+    (H5 / CSV / NPY / Parquet)
+  - ``BaseDBNode`` (数据库): 按 SQL 产出 pd.DataFrame
+    (SQLite / DuckDB / MySQL / ClickHouse / CSV / Parquet)
+
+两者接口差异大 (文件是面板矩阵语义, 数据库是 SQL 语义), 强行合并会
+泄漏抽象。``DataSource`` 因此被设计成**最小化标记基类**, 只统一:
+
+  1. ``close()`` 生命周期 (释放文件句柄 / 数据库连接)
+  2. 上下文管理器协议 (``with`` 自动 close)
+  3. "产出 pd.DataFrame" 的语义约定 (不强制具体读取签名)
+
+子树各自保留专用接口 (``load`` vs ``query``)。
+"""
+from abc import ABC, abstractmethod
+
+
+class DataSource(ABC):
+    """所有数据源的顶层标记 + 生命周期协议。
+
+    Subclasses:
+        FileFormatLoader: 文件格式适配器 (H5/CSV/NPY/Parquet)
+        BaseDBNode: 数据库节点 (SQLite/DuckDB/MySQL/ClickHouse/CSV/Parquet)
+
+    共同契约:
+        - 产出 pd.DataFrame (具体读取方法由子树定义)
+        - close() 释放底层资源
+        - 支持 ``with`` 上下文管理器
+    """
+
+    @abstractmethod
+    def close(self) -> None:
+        """释放底层资源 (文件句柄 / 数据库连接)。"""
+        raise NotImplementedError
+
+    def __enter__(self) -> "DataSource":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> bool:
+        self.close()
+        return False

QuantNodes/core/events.py

@@ -0,0 +1,178 @@
+# coding=utf-8
+"""
+core/events.py — 轻量级进程内事件总线 (Tier 0: Foundation)
+
+替代 v2.x 中已删除的 agent/bus/ 设计 (2026-06-23 nanobot 重构遗留)。
+
+设计原则:
+- 同步语义: handler 在 publish 线程同步执行
+- 单进程: 不支持跨进程/分布式
+- 解耦: publisher 与 subscriber 互不感知
+- 失败隔离: handler 异常不影响 publisher 和其他 handler
+
+使用示例:
+    from QuantNodes.core.events import get_event_bus, Events
+
+    bus = get_event_bus()
+
+    def on_factor_mined(event):
+        logger.info("New factor: %s", event.payload["formula"])
+
+    bus.subscribe(Events.FACTOR_MINED, on_factor_mined)
+    bus.publish_sync(Events.FACTOR_MINED, formula="rank(close)", ir=0.5)
+
+Event 命名规范:
+    `<domain>.<action>` — 例如 factor.mined / backtest.completed
+    所有事件常量定义在 Events 类中 (避免散落的字符串字面量)
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Event:
+    """事件对象
+
+    Args:
+        name: 事件名 (建议从 Events 类选取)
+        payload: 事件数据 (字典形式)
+        source: 触发源标识 (可选, 用于调试)
+    """
+
+    name: str
+    payload: Dict[str, Any] = field(default_factory=dict)
+    source: str = ""
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.name, str) or not self.name:
+            raise ValueError("Event.name must be a non-empty string")
+
+
+HandlerType = Callable[[Event], None]
+
+
+class EventBus:
+    """进程内事件总线 (同步)
+
+    线程安全: 本类内部无锁，订阅/发布应在同一线程或加外部锁。
+    """
+
+    def __init__(self) -> None:
+        self._subscribers: Dict[str, List[HandlerType]] = {}
+
+    def subscribe(self, event_name: str, handler: HandlerType) -> None:
+        """订阅事件。
+
+        同一 handler 可多次订阅同一事件, 触发时执行多次。
+        """
+        self._subscribers.setdefault(event_name, []).append(handler)
+
+    def unsubscribe(self, event_name: str, handler: HandlerType) -> None:
+        """取消订阅。handler 不存在时静默忽略。"""
+        if event_name in self._subscribers:
+            try:
+                self._subscribers[event_name].remove(handler)
+            except ValueError:
+                pass
+
+    def publish(self, event: Event) -> None:
+        """发布事件。同步调用所有订阅者。
+
+        异常隔离: 单个 handler 抛异常不影响其他 handler 和 publisher。
+        """
+        handlers = self._subscribers.get(event.name, [])
+        for handler in handlers:
+            try:
+                handler(event)
+            except Exception as e:
+                logger.warning(
+                    "Event handler %r failed for %s: %s",
+                    handler,
+                    event.name,
+                    e,
+                )
+
+    def publish_sync(self, name: str, source: str = "", **payload: Any) -> None:
+        """便捷发布接口。"""
+        self.publish(Event(name=name, payload=payload, source=source))
+
+    def clear(self) -> None:
+        """清空所有订阅 (主要用于测试)。"""
+        self._subscribers.clear()
+
+    def subscriber_count(self, event_name: str) -> int:
+        """返回某事件的订阅者数量 (主要用于测试)。"""
+        return len(self._subscribers.get(event_name, []))
+
+    def event_names(self) -> List[str]:
+        """返回所有已注册的事件名 (主要用于测试)。"""
+        return list(self._subscribers.keys())
+
+
+# 全局单例
+_global_bus: Optional[EventBus] = None
+
+
+def get_event_bus() -> EventBus:
+    """获取全局 EventBus 单例。
+
+    延迟创建: 首次调用时实例化, 后续返回同一对象。
+    测试中可用 reset_event_bus() 重置。
+    """
+    global _global_bus
+    if _global_bus is None:
+        _global_bus = EventBus()
+    return _global_bus
+
+
+def reset_event_bus() -> None:
+    """重置全局 EventBus (主要用于测试隔离)。"""
+    global _global_bus
+    _global_bus = None
+
+
+class Events:
+    """事件名常量集中定义
+
+    命名规范: `<domain>.<action>`
+    - factor.*    因子挖掘相关
+    - backtest.*  回测相关
+    - agent.*     Agent 交互相关
+    - monitor.*   监控相关
+    - workflow.*  工作流相关
+    """
+
+    # factor 事件
+    FACTOR_MINED = "factor.mined"
+    FACTOR_EVALUATED = "factor.evaluated"
+    FACTOR_REJECTED = "factor.rejected"
+
+    # backtest 事件
+    BACKTEST_COMPLETED = "backtest.completed"
+    BACKTEST_FAILED = "backtest.failed"
+
+    # agent 事件
+    AGENT_MESSAGE = "agent.message"
+    AGENT_TOOL_CALLED = "agent.tool_called"
+
+    # monitor 事件
+    DRIFT_DETECTED = "monitor.drift_detected"
+
+    # workflow 事件
+    WORKFLOW_ROUND_START = "workflow.round_start"
+    WORKFLOW_ROUND_END = "workflow.round_end"
+
+
+__all__ = [
+    "Event",
+    "EventBus",
+    "Events",
+    "get_event_bus",
+    "reset_event_bus",
+]

QuantNodes/core/evolution/__init__.py

@@ -0,0 +1,22 @@
+"""EvolutionLoop — 多轮演化主循环。
+
+公开 API:
+    - EvolutionSetting: Pydantic 配置
+    - FactorCandidate: 候选因子 dataclass
+    - Hypothesizer / Mutator / Crosser: 3 个 LLM-based operators
+    - EvolutionLoop: 主控循环
+    - EvolutionResult: 结果 dataclass
+"""
+from .settings import EvolutionSetting
+from .operators import Crosser, FactorCandidate, Hypothesizer, Mutator
+from .loop import EvolutionLoop, EvolutionResult
+
+__all__ = [
+    "EvolutionSetting",
+    "FactorCandidate",
+    "Hypothesizer",
+    "Mutator",
+    "Crosser",
+    "EvolutionLoop",
+    "EvolutionResult",
+]