ygo 1.0.5__tar.gz → 1.0.6b0__tar.gz

{ygo-1.0.5/ygo.egg-info → ygo-1.0.6b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.5
+Version: 1.0.6b0
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
 Requires-Python: >=3.8
@@ -8,6 +8,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clickhouse-driver>=0.2.9
 Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: exchange-calendars>=4.2.8
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: lark>=1.2.2
 Requires-Dist: loguru>=0.7.3

{ygo-1.0.5 → ygo-1.0.6b0}/pyproject.toml

@@ -4,13 +4,14 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygo"
-version = "1.0.5"
+version = "1.0.6beta"
 description = ""
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
     "clickhouse-driver>=0.2.9",
     "dynaconf>=3.2.11",
+    "exchange-calendars>=4.2.8",
     "joblib>=1.4.2",
     "lark>=1.2.2",
     "loguru>=0.7.3",

ygo-1.0.6b0/qdf/errors.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/5/16 10:47
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+
+from dataclasses import dataclass
+
+@dataclass
+class ParseError(Exception):
+    message: str
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
+
+@dataclass
+class CalculateError(Exception):
+    message: str
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
+
+@dataclass
+class CompileError(Exception):
+    message: str
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
+
+@dataclass
+class PolarsError(Exception):
+    message: str
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
+
+@dataclass
+class FailError:
+    expr: str
+    error: Exception
+
+    def __str__(self):
+        return f"""
+[失败表达式]: {self.expr}
+[错误类型]: {self.error.__class__.__name__}
+[错误信息]: \n{self.error}
+"""
+
+    def __repr__(self):
+        return self.__str__()

{ygo-1.0.5 → ygo-1.0.6b0}/qdf/expr.py

@@ -41,40 +41,40 @@ with warnings.catch_warnings():
         start: expr
         ?expr: ternary_expr
         ?ternary_expr: or_expr
-             | or_expr "?" or_expr ":" ternary_expr -> ternary
+            | or_expr "?" or_expr ":" ternary_expr -> ternary
         ?or_expr: and_expr
-             | or_expr "|" and_expr -> or_
+            | or_expr "|" and_expr -> or_
         ?and_expr: comp_expr
-             | and_expr "&" comp_expr -> and_
+            | and_expr "&" comp_expr -> and_
         ?comp_expr: eq_expr
-             | comp_expr "<" eq_expr -> lt
-             | comp_expr ">" eq_expr -> gt
-             | comp_expr "<=" eq_expr -> le
-             | comp_expr ">=" eq_expr -> ge
+            | comp_expr "<" eq_expr -> lt
+            | comp_expr ">" eq_expr -> gt
+            | comp_expr "<=" eq_expr -> le
+            | comp_expr ">=" eq_expr -> ge
         ?eq_expr: arith_expr
-             | eq_expr "==" arith_expr -> eq
-             | eq_expr "!=" arith_expr -> neq
+            | eq_expr "==" arith_expr -> eq
+            | eq_expr "!=" arith_expr -> neq
         ?arith_expr: term
-             | arith_expr "+" term -> add
-             | arith_expr "-" term -> sub
+            | arith_expr "+" term -> add
+            | arith_expr "-" term -> sub
         ?term: pow_expr
-             | term "*" pow_expr -> mul
-             | term "/" pow_expr -> div
-             | term "//" pow_expr -> floordiv // 取整
-             | term "%" pow_expr -> mod      // 求余
+            | term "*" pow_expr -> mul
+            | term "/" pow_expr -> div
+            | term "//" pow_expr -> floordiv // 取整
+            | term "%" pow_expr -> mod      // 求余
         ?pow_expr: factor
-             | factor "**" pow_expr -> pow
+            | factor "**" pow_expr -> pow
         ?factor: atom
-             | "-" factor -> neg
-             | "!" factor -> not_
-             | "~" factor -> not_
+            | "-" factor -> neg
+            | "!" factor -> not_
+            | "~" factor -> not_
         ?atom: function
-             | NAME
-             | NUMBER
-             | FLOAT
-             | "(" expr ")"
-             | implicit_mul // 隐式乘法
-             | attribute_access // 新增：属性访问
+            | NAME
+            | NUMBER
+            | FLOAT
+            | "(" expr ")"
+            | implicit_mul // 隐式乘法
+            | attribute_access // 新增：属性访问
         implicit_mul: (NUMBER | FLOAT) NAME -> implicit_mul  // 隐式乘法
         attribute_access: atom "." NAME -> attribute_access // 新增：属性访问
         function: NAME "(" expr_list ")" -> function
@@ -83,7 +83,7 @@ with warnings.catch_warnings():
         expr_list: (expr | keyword_arg) ("," (expr | keyword_arg))*  // 支持关键字参数
         NAME: /[a-zA-Z_$,][a-zA-Z0-9_$]*/
         NUMBER: /\d+/  // regex for numbers
-        FLOAT: /\d+\.\d+/
+        FLOAT: /\d+\.\d+([eE][+-]?\d+)?/ | /\d+[eE][+-]?\d+/  // 支持科学计数法
         %import common.WS
         %ignore WS
     """
@@ -181,7 +181,6 @@ parser = Lark(grammar, parser='lalr', transformer=ExprParser())
 def parse_expr(expression: str) -> Expr:
     return parser.parse(expression).children[0]
 
-
 class Expr:
 
     def __init__(self, expr: str | None = None):
@@ -203,6 +202,13 @@ class Expr:
         expr.alias = alias if alias is not None else str(expr)
         return expr
 
+    def __hash__(self):
+        return hash(str(self).strip())
+
+    def __eq__(self, other):
+        return str(self).strip() == str(other).strip()
+
+
     def to_rpn(self) -> list[Token]:
         """生成逆波兰表达式: (后缀表达式: 运算符在后)"""
         rpn = list()
@@ -285,6 +291,7 @@ class Expr:
         else:
             self.fn_name, self.args = expr_.fn_name, expr_.args
 
+
     @property
     def n_args(self) -> int:
         """返回表达式的参数个数"""

ygo-1.0.6b0/qdf/qdf.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/3/5 21:40
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+from __future__ import annotations
+
+import importlib.util
+import sys
+from functools import lru_cache
+from pathlib import Path
+
+import polars as pl
+from toolz import partial
+
+import ygo
+import ylog
+from .errors import CalculateError, CompileError, PolarsError, FailError
+from .expr import Expr
+
+# 动态加载模块
+module_name = "udf"
+module_path = Path(__file__).parent / "udf" / "__init__.py"
+spec = importlib.util.spec_from_file_location(module_name, module_path)
+module = importlib.util.module_from_spec(spec)
+sys.modules[module_name] = module
+spec.loader.exec_module(module)
+
+
+@lru_cache(maxsize=512)
+def parse_expr(expr: str) -> Expr:
+    return Expr(expr)
+
+
+class QDF:
+
+    def __init__(self,
+                 data: pl.LazyFrame | pl.DataFrame,
+                 index: tuple[str] = ("date", "time", "asset"),
+                 align: bool = True, ):
+        assert isinstance(data, (pl.LazyFrame, pl.DataFrame)), "data must be a polars DataFrame or LazyFrame"
+        self.data = data.with_columns(pl.col(pl.Decimal).cast(pl.Float32).round(5))
+        if isinstance(self.data, pl.LazyFrame):
+            self.data = self.data.collect()
+        self.index = index
+        self.dims = [self.data[name].drop_nulls().n_unique() for name in index]
+        if align:
+            lev_vals: list[pl.DataFrame] = [self.data.select(name).drop_nulls().unique() for name in index]
+            full_index = lev_vals[0]
+            for lev_val in lev_vals[1:]:
+                full_index = full_index.join(lev_val, how="cross")
+            self.data = full_index.join(self.data, on=index, how='left').sort(index)
+        self.failed = list()
+        self._expr_cache = dict()  # type: dict[Expr, str]
+        self._cur_expr_cache = dict()
+
+    def __str__(self):
+        return self.data.__str__()
+
+    def __repr__(self):
+        return self.data.__str__()
+
+    def register_udf(self, func: callable, name: str = None):
+        name = name if name is not None else func.__name__
+        setattr(module, name, func)
+
+    def _compile_expr(self, expr: str, cover: bool):
+        """str表达式 -> polars 表达式"""
+        try:
+            expr_parsed = Expr(expr)
+            alias = expr_parsed.alias  # if expr_parsed.alias is not None else str(expr_parsed)
+            current_cols = set(self.data.columns)
+            # columns = self.data.columns
+            if alias in current_cols and not cover:
+                return pl.col(alias), alias
+            # 如果该表达式已有对应列，直接复用
+            if expr_parsed in self._expr_cache and not cover:
+                expr_pl: pl.Expr = pl.col(self._expr_cache[expr_parsed]).alias(alias)
+                return expr_pl, alias
+            elif expr_parsed in self._cur_expr_cache and not cover:
+                expr_pl: pl.Expr = pl.col(self._cur_expr_cache[expr_parsed]).alias(alias)
+                return expr_pl, alias
+
+            def recur_compile(expr_: Expr):
+                """递归编译"""
+                alias_ = expr_.alias
+                if alias_ in current_cols and not cover:
+                    # 已存在：直接select数据源
+                    return pl.col(alias_)
+                if expr_ in self._expr_cache:
+                    return pl.col(self._expr_cache[expr_]).alias(alias_)
+                elif expr_ in self._cur_expr_cache:
+                    return pl.col(self._cur_expr_cache[expr_]).alias(alias_)
+                func = getattr(module, expr_.fn_name)
+                _params = ygo.fn_signature_params(func)
+                if "dims" in _params:
+                    func = partial(func, dims=self.dims)
+                args = list()
+                kwargs = dict()
+                for arg in expr_.args:
+                    if isinstance(arg, Expr):
+                        args.append(recur_compile(arg))
+                    elif isinstance(arg, dict):
+                        kwargs.update(arg)
+                    elif isinstance(arg, str):
+                        args.append(pl.col(arg))
+                    else:
+                        args.append(arg)  # or args.append(pl.lit(arg))
+                try:
+                    expr_pl: pl.Expr = func(*args, **kwargs).alias(alias_)
+                    self._cur_expr_cache[expr_] = alias_
+                    return expr_pl
+                except Exception as e:
+                    raise CompileError(message=f"{expr_.fn_name}({', '.join([str(arg) for arg in args])})\n{e}") from e
+
+            return recur_compile(expr_parsed), alias
+        except (CalculateError, CompileError, PolarsError) as e:
+            # 已经是你自己的错误类
+            raise e
+        except Exception as e:
+            # 所有未处理的错误统一抛出为 CompileError
+            raise CompileError(message=f"[编译器外层]\n{e}") from e
+
+    def sql(self, *exprs: str, cover: bool = False, ) -> pl.LazyFrame:
+        """
+        表达式查询
+        Parameters
+        ----------
+        exprs: str
+            表达式，比如 "ts_mean(close, 5) as close_ma5"
+        cover: bool
+            当遇到已经存在列名的时候，是否重新计算覆盖原来的列, 默认False，返回已经存在的列，跳过计算
+            - True: 重新计算并且返回新的结果，覆盖掉原来的列
+            - False, 返回已经存在的列，跳过计算
+        Returns
+        -------
+            polars.DataFrame
+        """
+        self.failed = list()
+        exprs_to_add = list()
+        exprs_select = list()
+        self._cur_expr_cache = {}
+        data = self.data.lazy()
+
+        for expr in exprs:
+            try:
+                compiled, alias = self._compile_expr(expr, cover)
+                if compiled is not None:
+                    exprs_to_add.append(compiled)
+                    exprs_select.append(alias)
+            except Exception as e:
+                self.failed.append(FailError(expr, e))
+        if self.failed:
+            ylog.warning(f"QDF.sql 失败：{len(self.failed)}/{len(exprs)}: \n {self.failed}")
+        for expr in exprs_to_add:
+            data = data.with_columns(expr).fill_nan(None)
+        try:
+            self.data = data.collect()
+            final_df = self.data.select(*self.index, *exprs_to_add)
+            self._expr_cache.update(self._cur_expr_cache)
+            return final_df
+        except Exception as e:
+            raise PolarsError(message=f"LazyFrame.collect() 阶段出错\n{e}") from e

{ygo-1.0.5 → ygo-1.0.6b0/ygo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.5
+Version: 1.0.6b0
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
 Requires-Python: >=3.8
@@ -8,6 +8,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clickhouse-driver>=0.2.9
 Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: exchange-calendars>=4.2.8
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: lark>=1.2.2
 Requires-Dist: loguru>=0.7.3

{ygo-1.0.5 → ygo-1.0.6b0}/ygo.egg-info/requires.txt

@@ -1,5 +1,6 @@
 clickhouse-driver>=0.2.9
 dynaconf>=3.2.11
+exchange-calendars>=4.2.8
 joblib>=1.4.2
 lark>=1.2.2
 loguru>=0.7.3

{ygo-1.0.5 → ygo-1.0.6b0}/ylog/core.py

@@ -94,7 +94,7 @@ class _Logger:
             colorize=True,
             backtrace=self.debug_mode,
             diagnose=self.debug_mode,
-            enqueue=True  # 异步写入
+            # enqueue=True  # 异步写入
         )
 
     def _setup_file_logging(self, retention_days: int, error_retention_days: int):
@@ -128,7 +128,7 @@ class _Logger:
                 compression="zip",
                 backtrace=True,
                 diagnose=self.debug_mode,
-                enqueue=True,  # 异步写入
+                # enqueue=True,  # 异步写入
                 filter=lambda record, lvl=level: record["level"].name == lvl,
                 catch=True  # 捕获格式化异常
             )

ygo-1.0.5/qdf/errors.py

@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2025/5/16 10:47
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-
-from dataclasses import dataclass
-
-@dataclass
-class ParseError(Exception):
-    message: str
-
-    def __str__(self):
-        return f"ParseError(message={self.message})"
-
-@dataclass
-class CalculateError(Exception):
-    message: str
-
-    def __str__(self):
-        return f"CalculateError(message={self.message})"
-
-@dataclass
-class PolarsError(Exception):
-    message: str
-
-    def __str__(self):
-        return f"PolarsError(message={self.message})"

ygo-1.0.5/qdf/qdf.py

@@ -1,158 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2025/3/5 21:40
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-from __future__ import annotations
-
-import importlib.util
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-
-import polars as pl
-from toolz import partial
-from functools import lru_cache
-
-import ygo
-import ylog
-from .errors import CalculateError, PolarsError
-from .expr import Expr
-import time
-
-# 动态加载模块
-module_name = "udf"
-module_path = Path(__file__).parent / "udf" / "__init__.py"
-spec = importlib.util.spec_from_file_location(module_name, module_path)
-module = importlib.util.module_from_spec(spec)
-sys.modules[module_name] = module
-spec.loader.exec_module(module)
-
-
-@dataclass
-class FailInfo:
-    expr: str
-    error: Exception
-
-    def __str__(self):
-        return f"""
-expr={self.expr}
-=================================================
-{self.error}
-=================================================
-"""
-
-    def __repr__(self):
-        return self.__str__()
-
-@lru_cache(maxsize=512)
-def parse_expr(expr: str) -> Expr:
-    return Expr(expr)
-
-class QDF:
-
-    def __init__(self,
-                 data: pl.LazyFrame,
-                 index: tuple[str] = ("date", "time", "asset"),
-                 align: bool = True,):
-        self.data = data.with_columns(pl.col(pl.Decimal).cast(pl.Float32))
-        self.dims = [self.data.select(index_).drop_nulls().unique().count().collect().item() for index_ in index]
-        if align:
-            lev_vals: list[pl.DataFrame] = [self.data.select(name).drop_nulls().unique() for name in index]
-            full_index = lev_vals[0]
-            for lev_val in lev_vals[1:]:
-                full_index = full_index.join(lev_val, how="cross")
-            self.data = full_index.join(self.data, on=index, how='left') #.sort(index).collect().lazy()
-
-        self.index = index
-        self.failed = list()
-
-    def __str__(self):
-        return self.data.__str__()
-
-    def __repr__(self):
-        return self.data.__str__()
-
-    def register_udf(self, func: callable, name: str = None):
-        name = name if name is not None else func.__name__
-        setattr(module, name, func)
-
-    def _compile_expr(self, expr: str, cover: bool):
-        expr_parsed = Expr(expr)
-        alias = expr_parsed.alias  # if expr_parsed.alias is not None else str(expr_parsed)
-        current_cols = set(self.data.collect_schema().keys())
-        columns = self.data.collect_schema().names()
-        if alias in current_cols and not cover:
-            return alias
-
-        def calc(expr_: Expr):
-            alias_ = expr_.alias
-            # _cols = self.data.collect_schema().names()
-            if alias_ in current_cols and not cover:
-                # 已存在：直接select数据源
-                return alias_
-            func = getattr(module, expr_.fn_name)
-            _params = ygo.fn_signature_params(func)
-            if "dims" in _params:
-                func = partial(func, dims=self.dims)
-            args = list()
-            kwargs = dict()
-            for arg in expr_.args:
-                if isinstance(arg, Expr):
-                    args.append(pl.col(calc(arg)))
-                elif isinstance(arg, dict):
-                    kwargs.update(arg)
-                elif isinstance(arg, str):
-                    args.append(pl.col(arg))
-                else:
-                    args.append(arg)  # or args.append(pl.lit(arg))
-            try:
-                expr_pl: pl.Expr = func(*args, **kwargs).alias(alias_)
-            except Exception as e:
-                raise CalculateError(f"{expr_.fn_name}({', '.join([str(arg) for arg in args])})\n{e}")
-            try:
-                self.data = self.data.with_columns(expr_pl)
-            except Exception as e:
-                raise PolarsError(f"{expr_}\n{e}")
-            return alias_
-
-        calc(expr_parsed)
-
-        columns.append(alias)
-        drop = current_cols.difference(set(columns))
-        self.data = self.data.drop(*drop)
-
-        return alias
-
-    def sql(self, *exprs: str, cover: bool = False,) -> pl.LazyFrame:
-        """
-        表达式查询
-        Parameters
-        ----------
-        exprs: str
-            表达式，比如 "ts_mean(close, 5) as close_ma5"
-        cover: bool
-            当遇到已经存在列名的时候，是否重新计算覆盖原来的列, 默认False，返回已经存在的列，跳过计算
-            - True: 重新计算并且返回新的结果，覆盖掉原来的列
-            - False, 返回已经存在的列，跳过计算
-        Returns
-        -------
-            polars.DataFrame
-        """
-        self.failed = list()
-        exprs_to_add = list()
-        for expr in exprs:
-            try:
-                compiled = self._compile_expr(expr, cover)
-                if compiled is not None:
-                    exprs_to_add.append(compiled)
-            except Exception as e:
-                self.failed.append(FailInfo(expr, e))
-        if self.failed:
-            ylog.warning(f"QDF.sql 失败：{len(self.failed)}/{len(exprs)}: \n {self.failed}")
-        final_df = self.data.with_columns(exprs_to_add).select(*self.index, *exprs_to_add).fill_nan(None).drop_nulls().sort(self.index)
-        return final_df.collect()
-