rust-pyfunc 0.77.0__tar.gz → 0.80.0__tar.gz

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/.gitignore

@@ -239,3 +239,4 @@ docs/theme_feature_expansion_explanation.md
 .mindfs/sessions/session-list.db
 .maturin_zig/*
 dist_versions/*
+python/rust_pyfunc/rust_pyfunc_worker

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/Cargo.lock

@@ -550,6 +550,17 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "core_affinity"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342"
+dependencies = [
+ "libc",
+ "num_cpus",
+ "winapi",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.17"
@@ -2181,12 +2192,13 @@ checksum = "082f11ffa03bbef6c2c6ea6bea1acafaade2fd9050ae0234ab44a2153742b058"
 
 [[package]]
 name = "rust_pyfunc"
-version = "0.77.0"
+version = "0.80.0"
 dependencies = [
  "arrow",
  "base64 0.21.7",
  "bincode",
  "chrono",
+ "core_affinity",
  "criterion",
  "crossbeam",
  "csv",
@@ -2671,6 +2683,22 @@ dependencies = [
  "safe_arch",
 ]
 
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
 [[package]]
 name = "winapi-util"
 version = "0.1.9"
@@ -2680,6 +2708,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
 [[package]]
 name = "windows-core"
 version = "0.61.2"

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rust_pyfunc"
-version = "0.77.0"
+version = "0.80.0"
 edition = "2021"
 description = "A collection of high-performance Python functions implemented in Rust"
 readme = "README.md"
@@ -46,9 +46,15 @@ log = "0.4"
 once_cell = "1.19"
 csv = "1.3"
 ndarray-npy = "0.8"
+core_affinity = "0.8"
 hdf5-metno = { version = "0.12", features = ["static", "zlib"], optional = true }
 zstd = "0.13"
 
+# Worker 二进制（多进程因子流水线）
+[[bin]]
+name = "rust_pyfunc_worker"
+path = "src/bin/worker_pipeline.rs"
+
 [features]
 default = ["hdf5"]
 hdf5 = ["hdf5-metno"]
@@ -65,8 +71,9 @@ criterion = "0.5"
 [profile.release]
 lto = true
 codegen-units = 1
-panic = "abort"
 opt-level = 3
+# 注意：不用 panic = "abort"，因为 worker bin 需要 catch_unwind 捕获计算 panic，
+# 避免整个 worker 进程崩溃重启。pyo3 0.18 在 unwind 下正常工作。
 
 [build-dependencies]
 pkg-config = "0.3"

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust_pyfunc
-Version: 0.77.0
+Version: 0.80.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/alter.sh

@@ -26,5 +26,10 @@ conda activate chenzongwei311
 export PATH="/home/chenzongwei/.local/bin:$PATH"
 /home/chenzongwei/.local/bin/mold -run maturin develop --release
 
+# 5. 编译并部署 worker 二进制（多进程因子流水线 mode="multiprocess" 需要）
+/home/chenzongwei/.local/bin/mold -run cargo build --release --bin rust_pyfunc_worker
+cp target/release/rust_pyfunc_worker python/rust_pyfunc/rust_pyfunc_worker
+echo "✅ worker 二进制已部署到 python/rust_pyfunc/rust_pyfunc_worker"
+
 # 如果需要，你也可以在脚本结束时停用环境
 # conda deactivate

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/python/rust_pyfunc/__init__.pyi

@@ -76,6 +76,24 @@ def compute_corr_diff_features(
 ) -> Tuple[List[float], object]:
     ...
 
+def compute_agent_validation_factors_py(
+    market_timestamps: object,
+    market_prices: object,
+    market_volumes: object,
+    market_flags: object,
+    ob_timestamps: object,
+    ob_bid_prc1: object,
+    ob_ask_prc1: object,
+    ob_bid_vol1: object,
+    ob_ask_vol1: object,
+    per_agent_market_indices: List[object],
+    per_agent_directions: List[object],
+    per_agent_volumes: List[object],
+    fwd_horizons_sec: List[float],
+    pt_horizons_sec: List[float],
+) -> List[float]:
+    ...
+
 
 # 版本信息
 __version__ = "0.18.0"
@@ -181,6 +199,7 @@ __all__ = [
     "compute_agent_trading_daily_feature_table_multi",
     "compute_agent_trading_daily_feature_table_for_get_features_factors",
     "compute_agent_trading_daily_feature_table_multi_for_get_features_factors",
+    "compute_agent_validation_factors_py",
     # 统计分析函数
     "calculate_base_entropy",
     "calculate_shannon_entropy_change",

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/python/rust_pyfunc/core_functions.pyi

@@ -2288,3 +2288,104 @@ def compute_ts_stats(series: List[float]) -> List[float]:
         数据不足5个时返回空列表
     """
     ...
+
+
+def read_trade_fast(
+    code: str,
+    date: int,
+    with_retreat: int = 0,
+    with_afternoon_adjust: bool = False,
+) -> NDArray[np.float64]:
+    """高速读取逐笔成交数据（Rust 多线程实现，替代 read_trade 的加速版）。
+
+    用 memmap2 内存映射文件，大文件按字节块切分并行解析（rayon），
+    读取时即完成全部预处理：flag!=32 过滤、exchtime 微秒整数→epoch 秒、
+    可选的下午时段平移（adjust_afternoon）。
+
+    与 read_trade 的差异：
+    - 直接返回 (n, 7) float64 numpy 数组，跳过 DataFrame 构造与 Timestamp 往返转换
+    - with_afternoon_adjust=True 时在读时一步完成下午时段平移并过滤非盘中数据
+    - time_sec 列为 epoch 秒（与 prepare() 中 exchtime.astype(int64)//1e9 一致）
+
+    列顺序（n×7 矩阵）：
+        [0] time_sec    成交时间，epoch 秒
+        [1] price       成交价
+        [2] volume      成交量
+        [3] turnover    成交额
+        [4] flag        成交标志（66=主买, 83=主卖；32=撤单，已被过滤）
+        [5] bid_order   买方订单号
+        [6] ask_order   卖方订单号
+
+    Parameters
+    ----------
+    code : str
+        股票代码，如 "000001"
+    date : int
+        交易日，如 20260605
+    with_retreat : int, 默认 0
+        1 时保留 flag==32 的撤单记录，0 时过滤（与 read_trade 默认一致）
+    with_afternoon_adjust : bool, 默认 False
+        True 时做 adjust_afternoon 平移：保留 [09:30,11:30]，下午 [13:00,14:57]
+        整体前移 90 分钟，过滤集合竞价前/收盘后的数据
+
+    Returns
+    -------
+    numpy.ndarray
+        (n, 7) float64 数组
+    """
+    ...
+
+
+def run_factor_pipeline(
+    pipeline: str,
+    tasks: List,
+    n_jobs: int,
+    backup_file: str,
+    expected_result_length: int,
+    trading_days: List[int],
+    params: object = None,
+    update_mode: bool = False,
+    bind_cores: bool = True,
+    backup_batch_size: int = 2000,
+    progress_log: bool = False,
+) -> None:
+    """纯 Rust 因子流水线引擎（run_factor_pipeline 优化方案 Phase 3）。
+
+    把 go 函数（如 hm90.go）的整条链路在 Rust 内部一气呵成，消除 Python worker、
+    pandas、pyo3 往返、msgpack 等所有流转浪费。并行控制采用自适应嵌套并行 + 核绑定，
+    总线程恒定 = n_jobs。
+
+    backup 格式与现有 run_pools_queue 完全兼容（复用 backup_writer），
+    后续 query_backup / query_backup_factor_only_ultra_fast 无需改动。
+
+    Parameters
+    ----------
+    pipeline : str
+        流水线标识，目前支持 "order_pair_hm90"（hm90.go 的 Rust 翻译）
+    tasks : List[[int, str]]
+        任务列表，每个元素 [date, code]
+    n_jobs : int
+        并行线程数（总 CPU 占用 ≈ n_jobs × 100%，默认生产用 200）
+    backup_file : str
+        备份文件路径（与 run_pools_queue 的 backup_file 格式兼容）
+    expected_result_length : int
+        每个任务的预期结果长度（因子数）
+    trading_days : List[int]
+        交易日历数组（由 rp.td.trading_days.tolist() 提供，用于 last_trading_day 查找）
+    params : dict, 可选
+        流水线参数，如 {"tolerance_v1": 0.001, "tolerance_v2": 0.00001}
+    update_mode : bool, 默认 False
+        断点续算：True 时跳过 backup 中已完成的任务
+    bind_cores : bool, 默认 True
+        核绑定：把 n_jobs 个线程各绑定到一个物理核，CPU 占用精确稳定
+    backup_batch_size : int, 默认 2000
+        每攒满 N 个结果写一次 backup（批量 zstd 压缩）
+    progress_log : bool, 默认 False
+        每 500 个任务打印一次进度
+
+    Notes
+    -----
+    "order_pair_hm90" 流水线关闭了 lyapunov 特征（原 get_features_factors 默认开启，
+    但花 69% 时间只产生 2.3% 特征）。如需 lyapunov，仍可用原 run_pools_queue。
+    """
+    ...

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/python/rust_pyfunc/factor_taskd.py

@@ -343,7 +343,7 @@ def cancel_task(task_id: int):
 
         # 快照当前进度后标记为取消
         progress = None
-        if row.get("backup_prefix"):
+        if row["backup_prefix"]:
             progress = _read_progress(row["backup_prefix"], task_id)
         db.execute(
             "UPDATE tasks SET status='cancelled', finished_at=?, progress_snapshot=? WHERE id=?",

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/python/rust_pyfunc/statistical_analysis.pyi

@@ -1797,3 +1797,50 @@ def mutual_information_2d_knn_final(
     >>> # print(f"成交量与金额的平均互信息: {np.mean(mi):.4f}")
     """
     ...
+
+def cross_stock_autoreg_38(
+    data_prev: NDArray[np.float64],
+    data_today: NDArray[np.float64],
+    n_lags: int,
+) -> NDArray[np.float64]:
+    """用前一日数据拟合自回归系数，在今日数据上做跨股票迁移预测，提取38个统计量。
+
+    data_prev: (T, N) 前一日分钟数据（已标准化）
+    data_today: (T, N) 今日分钟数据（已标准化）
+    n_lags: 自回归滞后阶数（5或15）
+
+    返回: (38, N) 每只股票38个统计量
+
+    统计量顺序：
+    0: corr_in_out    1: r2_in_uprate    2: r2_out_uprate    3: trans_out_uprate
+    4: in_mean        5: out_mean        6: in_ab_mean       7: out_ab_mean
+    8: trans_mean     9: in_std         10: out_std         11: in_ab_std
+   12: out_ab_std    13: trans_std      14: in_skew         15: out_skew
+   16: in_ab_skew    17: out_ab_skew    18: trans_skew      19: in_kurt
+   20: out_kurt      21: in_ab_kurt     22: out_ab_kurt     23: trans_kurt
+   24: in_median     25: out_median     26: in_ab_median    27: out_ab_median
+   28: trans_median  29: in_max         30: in_ab_max       31: out_ab_max
+   32: trans_max     33: in_min         34: out_min         35: in_ab_min
+   36: out_ab_min    37: trans_min
+    """
+    ...
+
+
+def cross_stock_crossvar_38(
+    y_prev: NDArray[np.float64],
+    x_prev: NDArray[np.float64],
+    y_today: NDArray[np.float64],
+    x_today: NDArray[np.float64],
+    n_lags: int,
+) -> NDArray[np.float64]:
+    """用前一日数据拟合交叉变量回归系数，在今日数据上做跨股票迁移预测，提取38个统计量。
+
+    模型: y ~ y_lag1..n_lags + x_lag1..n_lags
+
+    y_prev/x_prev: (T, N) 前一日因变量/自变量（已标准化）
+    y_today/x_today: (T, N) 今日因变量/自变量（已标准化）
+    n_lags: 每变量滞后阶数
+
+    返回: (38, N) 统计量矩阵，顺序同 cross_stock_autoreg_38
+    """
+    ...

{rust_pyfunc-0.77.0 → rust_pyfunc-0.80.0}/python/rust_pyfunc/trading_data_utils.py

@@ -1,3 +1,4 @@
+import mmap
 import os
 
 
@@ -48,6 +49,21 @@ def _resolve_stock_path(date: int, subdir: str, filename: str) -> str:
     )
 
 
+def _read_csv_nocache(file_path: str, **kwargs):
+    """mmap + MADV_SEQUENTIAL：避免一次性读出的 CSV 数据占据页缓存"""
+    fd = os.open(file_path, os.O_RDONLY)
+    mm = None
+    try:
+        mm = mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
+        mm.madvise(mmap.MADV_SEQUENTIAL)
+        df = pd.read_csv(mm, **kwargs)
+        return df
+    finally:
+        if mm is not None:
+            mm.close()
+        os.close(fd)
+
+
 def _to_exchange_timestamp(series: pd.Series) -> pd.Series:
     return pd.to_timedelta(series / 1e6, unit="s") + _TRADE_EPOCH_OFFSET
 
@@ -68,43 +84,38 @@ def adjust_afternoon(df: pd.DataFrame, only_inday: int = 1) -> pd.DataFrame:
     return df
 
 
-def read_trade(symbol: str, date: int, with_retreat: int = 0) -> pd.DataFrame:
+def read_trade(symbol: str, date: int, with_retreat: int = 0, no_cache: bool = False) -> pd.DataFrame:
     file_name = f"{symbol}_{date}_transaction.csv"
     file_path = _resolve_stock_path(date, "transaction", file_name)
-    df = pd.read_csv(
-        file_path,
+    csv_kwargs = dict(
         dtype={"symbol": str},
         usecols=[
-            "exchtime",
-            "price",
-            "volume",
-            "turnover",
-            "flag",
-            "index",
-            "localtime",
-            "ask_order",
-            "bid_order",
+            "exchtime", "price", "volume", "turnover", "flag",
+            "index", "localtime", "ask_order", "bid_order",
         ],
-        memory_map=True,
-        engine="c",
         low_memory=False,
     )
+    if no_cache:
+        df = _read_csv_nocache(file_path, **csv_kwargs)
+    else:
+        df = pd.read_csv(file_path, memory_map=True, engine="c", **csv_kwargs)
     if not with_retreat:
         df = df[df.flag != 32]
     df.exchtime = _to_exchange_timestamp(df.exchtime)
     return df
 
 
-def read_market(symbol: str, date: int, with_high_low_limited: int = 0) -> pd.DataFrame:
+def read_market(symbol: str, date: int, with_high_low_limited: int = 0, no_cache: bool = False) -> pd.DataFrame:
     file_name = f"{symbol}_{date}_market_data.csv"
     file_path = _resolve_stock_path(date, "market_data", file_name)
-    df = pd.read_csv(
-        file_path,
+    csv_kwargs = dict(
         dtype={"symbol": str},
-        memory_map=True,
-        engine="c",
         low_memory=False,
     )
+    if no_cache:
+        df = _read_csv_nocache(file_path, **csv_kwargs)
+    else:
+        df = pd.read_csv(file_path, memory_map=True, engine="c", **csv_kwargs)
     df.exchtime = _to_exchange_timestamp(df.exchtime)
     if not with_high_low_limited:
         df = df[(df.ask_prc1 != 0) & (df.bid_prc1 != 0)]