tidequant 0.1.0__tar.gz

tidequant-0.1.0/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.4
+Name: tidequant
+Version: 0.1.0
+Summary: TideQuant因子层: 算子库、表达式因子求值与历史因子生成引擎
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: jsonargparse
+Requires-Dist: loguru
+Requires-Dist: numba
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pyarrow
+Requires-Dist: setproctitle
+Requires-Dist: tqdm
+
+# TideQuant 潮汐量化
+顺势而生,逆势有度
+
+本包发布的是 TideQuant 的**因子层**: 算子库、表达式因子求值,
+以及从历史逐笔数据生成因子库的引擎。交易执行、交易所对接与看板
+不在这个包里。
+
+## 安装
+
+```bash
+pip install tidequant
+```
+
+## 因子计算
+
+```python
+from tidequant.factor import OPS, evaluate, parse
+
+node = parse("cs_zscore(ts_mean(amount / volume, 20))")
+value = evaluate(node, lambda leaf: load(leaf))
+```
+
+`OPS` 是算子表,`register` 可以往里加自定义算子。算子分三族:
+`ts_` 时序、`cs_` 截面、`ew_` 指数加权。
+
+## 生成历史因子库
+
+读取逐笔 trade 数据,按 interval 聚合字段因子,再对 `factors.json`
+里的表达式因子沿整个时间轴求值:
+
+```bash
+python -m tidequant.engine.history_factor_generator \
+    --tbt_folder data/binance/trades --out_folder data/factors_15m \
+    --start_dt 2022-07-01 --end_dt 2026-07-01 --interval 15m
+```
+
+已生成的因子自动跳过,中断后重跑自动续写。用 `--factors_path`
+指向自己的因子名单文件即可替换内置的 `factors.json`。
+
+因子库读写接口在 `tidequant.data.FactorBase`。

tidequant-0.1.0/README.md

@@ -0,0 +1,40 @@
+# TideQuant 潮汐量化
+顺势而生,逆势有度
+
+本包发布的是 TideQuant 的**因子层**: 算子库、表达式因子求值,
+以及从历史逐笔数据生成因子库的引擎。交易执行、交易所对接与看板
+不在这个包里。
+
+## 安装
+
+```bash
+pip install tidequant
+```
+
+## 因子计算
+
+```python
+from tidequant.factor import OPS, evaluate, parse
+
+node = parse("cs_zscore(ts_mean(amount / volume, 20))")
+value = evaluate(node, lambda leaf: load(leaf))
+```
+
+`OPS` 是算子表,`register` 可以往里加自定义算子。算子分三族:
+`ts_` 时序、`cs_` 截面、`ew_` 指数加权。
+
+## 生成历史因子库
+
+读取逐笔 trade 数据,按 interval 聚合字段因子,再对 `factors.json`
+里的表达式因子沿整个时间轴求值:
+
+```bash
+python -m tidequant.engine.history_factor_generator \
+    --tbt_folder data/binance/trades --out_folder data/factors_15m \
+    --start_dt 2022-07-01 --end_dt 2026-07-01 --interval 15m
+```
+
+已生成的因子自动跳过,中断后重跑自动续写。用 `--factors_path`
+指向自己的因子名单文件即可替换内置的 `factors.json`。
+
+因子库读写接口在 `tidequant.data.FactorBase`。

tidequant-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tidequant"
+dynamic = ["version"]
+description = "TideQuant因子层: 算子库、表达式因子求值与历史因子生成引擎"
+readme = "README.md"
+requires-python = ">=3.11"
+# 只列因子层真用得上的; 主仓pyproject里那些(ccxt/aiohttp/xarray/dash等)
+# 是交易与看板侧的, 装因子层的人不该被它们连坐
+dependencies = [
+    "jsonargparse",
+    "loguru",
+    "numba",
+    "numpy",
+    "pandas",
+    "pyarrow",
+    "setproctitle",
+    "tqdm",
+]
+
+classifiers = [
+    "Intended Audience :: Financial and Insurance Industry",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Office/Business :: Financial :: Investment",
+]
+
+# 构建根是publish/build.py搭出来的暂存树, 里头只有瘦包那几个文件,
+# 所以find直接全收即可 —— 该收哪些的决定在build.py的PATHS
+[tool.setuptools.packages.find]
+include = ["tidequant*"]
+
+[tool.setuptools.package-data]
+# factors.json是包内数据文件, 不显式列出的话wheel里不会有它, 装上就
+# 读不到因子配方
+"tidequant.factor" = ["*.json"]
+
+[tool.setuptools.dynamic]
+version = { attr = "tidequant.__version__" }

tidequant-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tidequant-0.1.0/tidequant/__init__.py

@@ -0,0 +1,24 @@
+__version__ = "0.1.0"
+
+from .enums import *
+
+from .types import *
+
+
+def __getattr__(name: str):
+    """
+    Executor惰性导入(PEP 562)
+
+    顶层直接import它会连带拖进exchange与ccxt, 而PyPI上发布的是只含
+    因子层的瘦包(见publish/), 那份wheel里没有executor, 照旧写法
+    import tidequant.factor就会在这一行直接炸
+
+    from tidequant import Executor与tidequant.Executor照旧可用;
+    只有from tidequant import *不再带上它
+    """
+    if name == "Executor":
+        from .executor import Executor
+
+        return Executor
+
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

tidequant-0.1.0/tidequant/data.py

@@ -0,0 +1,326 @@
+"""
+数据库相关类
+"""
+
+import os
+from concurrent.futures import ThreadPoolExecutor
+from typing import List, Literal
+
+import numpy as np
+import pandas as pd
+
+
+class TradeBase:
+    """
+    历史逐笔数据库
+
+    布局(folder/): <symbol>/<YYYY-MM-DD>.feather
+    每文件一个symbol一个UTC自然日的逐笔交易
+    """
+
+    def __init__(self, folder: str) -> None:
+        self.folder: str = folder
+
+    def path(self, symbol: str, dt: np.datetime64) -> str:
+        return os.path.join(self.folder, symbol, f"{dt}.feather")
+
+    def list_symbols(self, start_dt: str, end_dt: str) -> List[str]:
+        """
+        扫描[start_dt, end_dt)内有数据文件的symbol, 排序返回
+        """
+        symbols: List[str] = []
+        for name in sorted(os.listdir(self.folder)):
+            folder: str = os.path.join(self.folder, name)
+            if not os.path.isdir(folder):
+                continue
+
+            if any(
+                start_dt <= file[: 10] < end_dt
+                for file in os.listdir(folder)
+                if file.endswith(".feather")
+            ):
+                symbols.append(name)
+
+        return symbols
+
+    def list_dts(self, symbol: str) -> List[np.datetime64]:
+        """
+        列出某symbol已有数据文件的日期, 排序返回
+        """
+        folder: str = os.path.join(self.folder, symbol)
+        if not os.path.isdir(folder):
+            return []
+
+        return sorted(
+            np.datetime64(file[: 10], "D")
+            for file in os.listdir(folder)
+            if file.endswith(".feather")
+        )
+
+    def read(self, symbol: str, dt: np.datetime64) -> pd.DataFrame | None:
+        """
+        读取一个symbol一天的trades, 无文件(未上市/停牌)返回None
+        """
+        path: str = self.path(symbol, dt)
+        if not os.path.exists(path):
+            return None
+
+        return pd.read_feather(path)
+
+    def read_tails(self, symbol: str, dt: np.datetime64) -> pd.DataFrame | None:
+        """
+        读取一天各方向最后一笔trade, 供次日计算交易时间间隔等指标
+        """
+        df: pd.DataFrame | None = self.read(symbol, dt)
+        if df is None:
+            return None
+
+        return self.tails(df)
+
+    @staticmethod
+    def tails(df: pd.DataFrame) -> pd.DataFrame:
+        """
+        取trades中各方向最后一笔, "尾部"的唯一定义
+
+        读盘路径(read_tails)与内存中刚聚合完的df取尾共用
+        """
+        return df.groupby("is_buyer_maker", sort=False).tail(1)
+
+    def write(
+        self, symbol: str, dt: np.datetime64, df: pd.DataFrame
+    ) -> None:
+        """
+        写入一个symbol一天的trades
+
+        先写临时文件再原子改名, 保证读侧看到的文件必是完整的
+        """
+        path: str = self.path(symbol, dt)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        tmp_path: str = f"{path}.tmp.{os.getpid()}"
+        df.to_feather(tmp_path)
+        os.replace(tmp_path, path)
+
+
+class FactorBase:
+    """
+    因子库, 建库写入与切片读取的全部接口
+
+    布局(folder/):
+        symbols.bin    换行分隔的symbol字符串, N轴, 建库时一次固定
+        datetimes.bin  int64毫秒close时间戳, 每完成一天追加一批,
+                       是库进度的唯一真源
+        x/<因子名>     float32行主序(T, N)矩阵, 字段因子按日追加,
+                       表达式因子每次整文件覆盖(见overwrite)
+        y/<标签名>     布局同x, 放前瞻的预测目标
+
+    崩溃恢复: 每日提交顺序为先写全部因子块最后追加datetimes,
+    重新打开库时把行数超过轴长的文件截断回轴长即可幂等续跑
+    """
+
+    ITEM_SIZE: int = 4  # float32字节数
+
+    def __init__(self, folder: str) -> None:
+        self.folder: str = folder
+        self.symbols_path: str = os.path.join(folder, "symbols.bin")
+        self.datetimes_path: str = os.path.join(folder, "datetimes.bin")
+
+        self.symbols: List[str] = []
+        self.datetimes: np.ndarray = np.empty(0, dtype=np.int64)
+        if self.exists():
+            with open(self.symbols_path, encoding="utf-8") as f:
+                self.symbols = [s for s in f.read().splitlines() if s]
+
+            if os.path.exists(self.datetimes_path):
+                self.datetimes = np.fromfile(
+                    self.datetimes_path, dtype=np.int64
+                )
+
+            self._truncate_overrun()
+
+    def exists(self) -> bool:
+        """
+        库是否已建过, 即N轴是否已写定
+        """
+        return os.path.exists(self.symbols_path)
+
+    def _truncate_overrun(self) -> None:
+        """
+        把行数超过轴长的因子文件截断回轴长, 以防上次崩溃的半日写入
+        """
+        n_bytes: int = len(self.datetimes) * len(self.symbols) * self.ITEM_SIZE
+        for mode in ("x", "y"):
+            for name in self.list_names(mode):
+                path: str = os.path.join(self.folder, mode, name)
+                if os.path.getsize(path) > n_bytes:
+                    with open(path, "r+b") as f:
+                        f.truncate(n_bytes)
+
+    def create(self, symbols: List[str]) -> None:
+        """
+        建库: 固定N轴并建好目录结构, 一个库只能建一次
+
+        N轴一旦写定, 后续所有因子文件的列顺序都以它为准
+        """
+        if self.exists():
+            raise RuntimeError(f"因子库{self.folder}已建过, 不可重建N轴")
+
+        for mode in ("x", "y"):
+            os.makedirs(os.path.join(self.folder, mode), exist_ok=True)
+
+        with open(self.symbols_path, "w", encoding="utf-8") as f:
+            f.write("".join(symbol + "\n" for symbol in symbols))
+
+        self.symbols = list(symbols)
+
+    def list_names(self, mode: Literal["x", "y"] = "x") -> List[str]:
+        """
+        返回某侧全部名字, mode="x"为因子、"y"为标签
+        """
+        folder: str = os.path.join(self.folder, mode)
+        if not os.path.isdir(folder):
+            return []
+
+        return sorted(os.listdir(folder))
+
+    def written_rows(self, name: str, mode: Literal["x", "y"] = "x") -> int:
+        """
+        某个名字已写入的行数, 无文件为0
+
+        与len(datetimes)的区别: 后者是全库已提交的行数, 每天
+        写完所有因子块才追加一批; 单个文件可以超前于它,
+        也可以落后于它
+        """
+        path: str = os.path.join(self.folder, mode, name)
+        row_bytes: int = len(self.symbols) * self.ITEM_SIZE
+        try:
+            return os.path.getsize(path) // row_bytes
+        except FileNotFoundError:
+            return 0
+
+    def append(
+        self, name: str, block: np.ndarray, mode: Literal["x", "y"] = "x"
+    ) -> None:
+        """
+        向因子文件尾部追加一个(bars, N)块
+        """
+        assert block.ndim == 2 and block.shape[1] == len(self.symbols)
+        with open(os.path.join(self.folder, mode, name), "ab") as f:
+            np.ascontiguousarray(block, dtype=np.float32).tofile(f)
+
+    def append_datetimes(self, stamps: np.ndarray) -> None:
+        """
+        追加一批close时间戳, 作为一天数据的提交点
+        """
+        stamps = np.asarray(stamps, dtype=np.int64)
+        with open(self.datetimes_path, "ab") as f:
+            stamps.tofile(f)
+
+        self.datetimes = np.concatenate([self.datetimes, stamps])
+
+    def overwrite(
+        self, name: str, block: np.ndarray, mode: Literal["x", "y"] = "x"
+    ) -> None:
+        """
+        用一个(T, N)块整文件替换某个因子
+
+        表达式因子每次都从第0行重算(见HistoryFactorGenerator的二趟),
+        所以是覆盖而不是追加。先写临时文件再原子改名, 与TradeBase.write
+        同一套办法: 中途崩溃留下的是旧的完整文件, 不会是半个新文件
+        """
+        assert block.ndim == 2 and block.shape[1] == len(self.symbols)
+        path: str = os.path.join(self.folder, mode, name)
+        tmp_path: str = f"{path}.tmp.{os.getpid()}"
+        np.ascontiguousarray(block, dtype=np.float32).tofile(tmp_path)
+        os.replace(tmp_path, path)
+
+    def memmap(
+        self, name: str, mode: Literal["x", "y"] = "x"
+    ) -> np.memmap:
+        """
+        某个因子的(已写行数, N)只读memmap视图
+
+        供按品种列切片读: 文件是行主序, 列切片省不了I/O(每行的页
+        都要碰), 但能把物化数组从T×N降到T×chunk, 这才是按列分块
+        求值的实际收益
+        """
+        n_rows: int = self.written_rows(name, mode)
+        return np.memmap(
+            os.path.join(self.folder, mode, name),
+            dtype=np.float32,
+            mode="r",
+            shape=(n_rows, len(self.symbols)),
+        )
+
+    def read(
+        self,
+        names: str | List[str],
+        start_dt: str | None = None,
+        end_dt: str | None = None,
+        mode: Literal["x", "y"] = "x",
+        n_worker: int = 1,
+    ) -> np.ndarray:
+        """
+        读取因子, 恒返回float32的(T, N, F)矩阵
+
+        [start_dt, end_dt)按UTC自然日左闭右开取整天,
+        注意行对应bar的close时间, 每天末根bar的close为次日零点,
+        故read(name, "2026-06-01", "2026-06-03")返回06-01、06-02
+        两整天, 06-03T00:00对应的bar会包含在内。时间轴取
+        self.datetimes, N轴取self.symbols
+
+        n_worker>1时并发读各因子文件, 只对多因子有意义
+        """
+        if isinstance(names, str):
+            names = [names]
+
+        # close是bar的右端点, 故两端都取searchsorted右侧, 选中
+        # close∈(start_dt, end_dt]的行, 即窗口起点在[start_dt, end_dt)内
+        stamps: np.ndarray = self.datetimes
+        start: int = 0 if start_dt is None else int(np.searchsorted(
+            stamps, np.datetime64(start_dt, "ms").astype(np.int64),
+            side="right",
+        ))
+        end: int = len(stamps) if end_dt is None else int(np.searchsorted(
+            stamps, np.datetime64(end_dt, "ms").astype(np.int64),
+            side="right",
+        ))
+
+        cube: np.ndarray = np.empty(
+            (end - start, len(self.symbols), len(names)),
+            dtype=np.float32,
+        )
+
+        def load(i: int) -> None:
+            cube[:, :, i] = self._read_rows(names[i], mode, start, end)
+
+        if n_worker > 1:
+            with ThreadPoolExecutor(n_worker) as pool:
+                list(pool.map(load, range(len(names))))
+        else:
+            for i in range(len(names)):
+                load(i)
+
+        return cube
+
+    def _read_rows(
+        self, name: str, mode: Literal["x", "y"], start: int, end: int
+    ) -> np.ndarray:
+        """
+        按字节偏移读取因子文件的[start, end)行
+        """
+        n_symbols: int = len(self.symbols)
+        n_want: int = (end - start) * n_symbols
+        path: str = os.path.join(self.folder, mode, name)
+        data: np.ndarray = np.fromfile(
+            path,
+            dtype=np.float32,
+            count=n_want,
+            offset=start * n_symbols * self.ITEM_SIZE,
+        )
+        if data.size != n_want:
+            raise ValueError(
+                f"{path}短于T轴: 期望读到行[{start}, {end}), "
+                f"实际不足, 文件可能未写完或损坏"
+            )
+
+        return data.reshape(-1, n_symbols)

tidequant-0.1.0/tidequant/engine/__init__.py

@@ -0,0 +1 @@
+from .base import Engine

tidequant-0.1.0/tidequant/engine/base.py

@@ -0,0 +1,78 @@
+"""
+负责执行任务的引擎基类
+"""
+
+import os
+import sys
+from abc import ABC, abstractmethod
+
+import setproctitle
+from loguru import logger
+
+
+class Engine(ABC):
+    """
+    引擎基类
+    
+    引擎运行的结果会被写入一个新的文件夹
+    """
+
+    def __init__(
+        self,
+        folder: str,
+        log_folder: str | None = None,
+        log_label: str = "main",
+    ) -> None:
+        self.folder: str = folder
+        # 默认与数据目录相同; 多进程时可指到共享目录
+        self.log_folder: str = folder if log_folder is None else log_folder
+        self.log_label: str = log_label
+
+        # 进程名和log文件名均使用类名
+        setproctitle.setproctitle(self.__class__.__name__)
+        self._init_logger()
+
+    def _init_logger(self) -> None:
+        """
+        配置logger, 同时输出到文件和控制台
+        """
+        self.configure_logger(
+            os.path.join(
+                self.log_folder, f"{self.__class__.__name__}.log"
+            ),
+            log_label=self.log_label,
+        )
+
+    @staticmethod
+    def configure_logger(
+        log_path: str,
+        log_label: str = "main",
+    ) -> None:
+        """
+        配置进程内logger; enqueue保证多进程写同一文件安全
+        """
+        log_format: str = (
+            "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level} | "
+            f"{log_label} | {{message}}"
+        )
+        logger.remove()
+        logger.add(sys.stderr, format=log_format)
+        logger.add(
+            log_path,
+            encoding="utf-8",
+            format=log_format,
+            enqueue=True,
+        )
+
+    @abstractmethod 
+    async def run(self) -> None:
+        """
+        运行任务
+        """
+        pass
+
+    async def close(self) -> None:
+        """
+        释放资源并关闭引擎
+        """
+        logger.remove()