ygo 1.0.5__py3-none-any.whl → 1.0.6b0__py3-none-any.whl

qdf/errors.py

@@ -14,18 +14,52 @@ class ParseError(Exception):
     message: str
 
     def __str__(self):
-        return f"ParseError(message={self.message})"
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
 
 @dataclass
 class CalculateError(Exception):
     message: str
 
     def __str__(self):
-        return f"CalculateError(message={self.message})"
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
+
+@dataclass
+class CompileError(Exception):
+    message: str
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
 
 @dataclass
 class PolarsError(Exception):
     message: str
 
     def __str__(self):
-        return f"PolarsError(message={self.message})"
+        return self.message
+
+    def __repr__(self):
+        return self.__str__()
+
+@dataclass
+class FailError:
+    expr: str
+    error: Exception
+
+    def __str__(self):
+        return f"""
+[失败表达式]: {self.expr}
+[错误类型]: {self.error.__class__.__name__}
+[错误信息]: \n{self.error}
+"""
+
+    def __repr__(self):
+        return self.__str__()

qdf/expr.py

@@ -41,40 +41,40 @@ with warnings.catch_warnings():
         start: expr
         ?expr: ternary_expr
         ?ternary_expr: or_expr
-             | or_expr "?" or_expr ":" ternary_expr -> ternary
+            | or_expr "?" or_expr ":" ternary_expr -> ternary
         ?or_expr: and_expr
-             | or_expr "|" and_expr -> or_
+            | or_expr "|" and_expr -> or_
         ?and_expr: comp_expr
-             | and_expr "&" comp_expr -> and_
+            | and_expr "&" comp_expr -> and_
         ?comp_expr: eq_expr
-             | comp_expr "<" eq_expr -> lt
-             | comp_expr ">" eq_expr -> gt
-             | comp_expr "<=" eq_expr -> le
-             | comp_expr ">=" eq_expr -> ge
+            | comp_expr "<" eq_expr -> lt
+            | comp_expr ">" eq_expr -> gt
+            | comp_expr "<=" eq_expr -> le
+            | comp_expr ">=" eq_expr -> ge
         ?eq_expr: arith_expr
-             | eq_expr "==" arith_expr -> eq
-             | eq_expr "!=" arith_expr -> neq
+            | eq_expr "==" arith_expr -> eq
+            | eq_expr "!=" arith_expr -> neq
         ?arith_expr: term
-             | arith_expr "+" term -> add
-             | arith_expr "-" term -> sub
+            | arith_expr "+" term -> add
+            | arith_expr "-" term -> sub
         ?term: pow_expr
-             | term "*" pow_expr -> mul
-             | term "/" pow_expr -> div
-             | term "//" pow_expr -> floordiv // 取整
-             | term "%" pow_expr -> mod      // 求余
+            | term "*" pow_expr -> mul
+            | term "/" pow_expr -> div
+            | term "//" pow_expr -> floordiv // 取整
+            | term "%" pow_expr -> mod      // 求余
         ?pow_expr: factor
-             | factor "**" pow_expr -> pow
+            | factor "**" pow_expr -> pow
         ?factor: atom
-             | "-" factor -> neg
-             | "!" factor -> not_
-             | "~" factor -> not_
+            | "-" factor -> neg
+            | "!" factor -> not_
+            | "~" factor -> not_
         ?atom: function
-             | NAME
-             | NUMBER
-             | FLOAT
-             | "(" expr ")"
-             | implicit_mul // 隐式乘法
-             | attribute_access // 新增：属性访问
+            | NAME
+            | NUMBER
+            | FLOAT
+            | "(" expr ")"
+            | implicit_mul // 隐式乘法
+            | attribute_access // 新增：属性访问
         implicit_mul: (NUMBER | FLOAT) NAME -> implicit_mul  // 隐式乘法
         attribute_access: atom "." NAME -> attribute_access // 新增：属性访问
         function: NAME "(" expr_list ")" -> function
@@ -83,7 +83,7 @@ with warnings.catch_warnings():
         expr_list: (expr | keyword_arg) ("," (expr | keyword_arg))*  // 支持关键字参数
         NAME: /[a-zA-Z_$,][a-zA-Z0-9_$]*/
         NUMBER: /\d+/  // regex for numbers
-        FLOAT: /\d+\.\d+/
+        FLOAT: /\d+\.\d+([eE][+-]?\d+)?/ | /\d+[eE][+-]?\d+/  // 支持科学计数法
         %import common.WS
         %ignore WS
     """
@@ -181,7 +181,6 @@ parser = Lark(grammar, parser='lalr', transformer=ExprParser())
 def parse_expr(expression: str) -> Expr:
     return parser.parse(expression).children[0]
 
-
 class Expr:
 
     def __init__(self, expr: str | None = None):
@@ -203,6 +202,13 @@ class Expr:
         expr.alias = alias if alias is not None else str(expr)
         return expr
 
+    def __hash__(self):
+        return hash(str(self).strip())
+
+    def __eq__(self, other):
+        return str(self).strip() == str(other).strip()
+
+
     def to_rpn(self) -> list[Token]:
         """生成逆波兰表达式: (后缀表达式: 运算符在后)"""
         rpn = list()
@@ -285,6 +291,7 @@ class Expr:
         else:
             self.fn_name, self.args = expr_.fn_name, expr_.args
 
+
     @property
     def n_args(self) -> int:
         """返回表达式的参数个数"""

qdf/qdf.py

@@ -10,18 +10,16 @@ from __future__ import annotations
 
 import importlib.util
 import sys
-from dataclasses import dataclass
+from functools import lru_cache
 from pathlib import Path
 
 import polars as pl
 from toolz import partial
-from functools import lru_cache
 
 import ygo
 import ylog
-from .errors import CalculateError, PolarsError
+from .errors import CalculateError, CompileError, PolarsError, FailError
 from .expr import Expr
-import time
 
 # 动态加载模块
 module_name = "udf"
@@ -32,43 +30,32 @@ sys.modules[module_name] = module
 spec.loader.exec_module(module)
 
 
-@dataclass
-class FailInfo:
-    expr: str
-    error: Exception
-
-    def __str__(self):
-        return f"""
-expr={self.expr}
-=================================================
-{self.error}
-=================================================
-"""
-
-    def __repr__(self):
-        return self.__str__()
-
 @lru_cache(maxsize=512)
 def parse_expr(expr: str) -> Expr:
     return Expr(expr)
 
+
 class QDF:
 
     def __init__(self,
-                 data: pl.LazyFrame,
+                 data: pl.LazyFrame | pl.DataFrame,
                  index: tuple[str] = ("date", "time", "asset"),
-                 align: bool = True,):
-        self.data = data.with_columns(pl.col(pl.Decimal).cast(pl.Float32))
-        self.dims = [self.data.select(index_).drop_nulls().unique().count().collect().item() for index_ in index]
+                 align: bool = True, ):
+        assert isinstance(data, (pl.LazyFrame, pl.DataFrame)), "data must be a polars DataFrame or LazyFrame"
+        self.data = data.with_columns(pl.col(pl.Decimal).cast(pl.Float32).round(5))
+        if isinstance(self.data, pl.LazyFrame):
+            self.data = self.data.collect()
+        self.index = index
+        self.dims = [self.data[name].drop_nulls().n_unique() for name in index]
         if align:
             lev_vals: list[pl.DataFrame] = [self.data.select(name).drop_nulls().unique() for name in index]
             full_index = lev_vals[0]
             for lev_val in lev_vals[1:]:
                 full_index = full_index.join(lev_val, how="cross")
-            self.data = full_index.join(self.data, on=index, how='left') #.sort(index).collect().lazy()
-
-        self.index = index
+            self.data = full_index.join(self.data, on=index, how='left').sort(index)
         self.failed = list()
+        self._expr_cache = dict()  # type: dict[Expr, str]
+        self._cur_expr_cache = dict()
 
     def __str__(self):
         return self.data.__str__()
@@ -81,53 +68,63 @@ class QDF:
         setattr(module, name, func)
 
     def _compile_expr(self, expr: str, cover: bool):
-        expr_parsed = Expr(expr)
-        alias = expr_parsed.alias  # if expr_parsed.alias is not None else str(expr_parsed)
-        current_cols = set(self.data.collect_schema().keys())
-        columns = self.data.collect_schema().names()
-        if alias in current_cols and not cover:
-            return alias
-
-        def calc(expr_: Expr):
-            alias_ = expr_.alias
-            # _cols = self.data.collect_schema().names()
-            if alias_ in current_cols and not cover:
-                # 已存在：直接select数据源
-                return alias_
-            func = getattr(module, expr_.fn_name)
-            _params = ygo.fn_signature_params(func)
-            if "dims" in _params:
-                func = partial(func, dims=self.dims)
-            args = list()
-            kwargs = dict()
-            for arg in expr_.args:
-                if isinstance(arg, Expr):
-                    args.append(pl.col(calc(arg)))
-                elif isinstance(arg, dict):
-                    kwargs.update(arg)
-                elif isinstance(arg, str):
-                    args.append(pl.col(arg))
-                else:
-                    args.append(arg)  # or args.append(pl.lit(arg))
-            try:
-                expr_pl: pl.Expr = func(*args, **kwargs).alias(alias_)
-            except Exception as e:
-                raise CalculateError(f"{expr_.fn_name}({', '.join([str(arg) for arg in args])})\n{e}")
-            try:
-                self.data = self.data.with_columns(expr_pl)
-            except Exception as e:
-                raise PolarsError(f"{expr_}\n{e}")
-            return alias_
-
-        calc(expr_parsed)
-
-        columns.append(alias)
-        drop = current_cols.difference(set(columns))
-        self.data = self.data.drop(*drop)
-
-        return alias
-
-    def sql(self, *exprs: str, cover: bool = False,) -> pl.LazyFrame:
+        """str表达式 -> polars 表达式"""
+        try:
+            expr_parsed = Expr(expr)
+            alias = expr_parsed.alias  # if expr_parsed.alias is not None else str(expr_parsed)
+            current_cols = set(self.data.columns)
+            # columns = self.data.columns
+            if alias in current_cols and not cover:
+                return pl.col(alias), alias
+            # 如果该表达式已有对应列，直接复用
+            if expr_parsed in self._expr_cache and not cover:
+                expr_pl: pl.Expr = pl.col(self._expr_cache[expr_parsed]).alias(alias)
+                return expr_pl, alias
+            elif expr_parsed in self._cur_expr_cache and not cover:
+                expr_pl: pl.Expr = pl.col(self._cur_expr_cache[expr_parsed]).alias(alias)
+                return expr_pl, alias
+
+            def recur_compile(expr_: Expr):
+                """递归编译"""
+                alias_ = expr_.alias
+                if alias_ in current_cols and not cover:
+                    # 已存在：直接select数据源
+                    return pl.col(alias_)
+                if expr_ in self._expr_cache:
+                    return pl.col(self._expr_cache[expr_]).alias(alias_)
+                elif expr_ in self._cur_expr_cache:
+                    return pl.col(self._cur_expr_cache[expr_]).alias(alias_)
+                func = getattr(module, expr_.fn_name)
+                _params = ygo.fn_signature_params(func)
+                if "dims" in _params:
+                    func = partial(func, dims=self.dims)
+                args = list()
+                kwargs = dict()
+                for arg in expr_.args:
+                    if isinstance(arg, Expr):
+                        args.append(recur_compile(arg))
+                    elif isinstance(arg, dict):
+                        kwargs.update(arg)
+                    elif isinstance(arg, str):
+                        args.append(pl.col(arg))
+                    else:
+                        args.append(arg)  # or args.append(pl.lit(arg))
+                try:
+                    expr_pl: pl.Expr = func(*args, **kwargs).alias(alias_)
+                    self._cur_expr_cache[expr_] = alias_
+                    return expr_pl
+                except Exception as e:
+                    raise CompileError(message=f"{expr_.fn_name}({', '.join([str(arg) for arg in args])})\n{e}") from e
+
+            return recur_compile(expr_parsed), alias
+        except (CalculateError, CompileError, PolarsError) as e:
+            # 已经是你自己的错误类
+            raise e
+        except Exception as e:
+            # 所有未处理的错误统一抛出为 CompileError
+            raise CompileError(message=f"[编译器外层]\n{e}") from e
+
+    def sql(self, *exprs: str, cover: bool = False, ) -> pl.LazyFrame:
         """
         表达式查询
         Parameters
@@ -144,15 +141,26 @@ class QDF:
         """
         self.failed = list()
         exprs_to_add = list()
+        exprs_select = list()
+        self._cur_expr_cache = {}
+        data = self.data.lazy()
+
         for expr in exprs:
             try:
-                compiled = self._compile_expr(expr, cover)
+                compiled, alias = self._compile_expr(expr, cover)
                 if compiled is not None:
                     exprs_to_add.append(compiled)
+                    exprs_select.append(alias)
             except Exception as e:
-                self.failed.append(FailInfo(expr, e))
+                self.failed.append(FailError(expr, e))
         if self.failed:
             ylog.warning(f"QDF.sql 失败：{len(self.failed)}/{len(exprs)}: \n {self.failed}")
-        final_df = self.data.with_columns(exprs_to_add).select(*self.index, *exprs_to_add).fill_nan(None).drop_nulls().sort(self.index)
-        return final_df.collect()
-
+        for expr in exprs_to_add:
+            data = data.with_columns(expr).fill_nan(None)
+        try:
+            self.data = data.collect()
+            final_df = self.data.select(*self.index, *exprs_to_add)
+            self._expr_cache.update(self._cur_expr_cache)
+            return final_df
+        except Exception as e:
+            raise PolarsError(message=f"LazyFrame.collect() 阶段出错\n{e}") from e

{ygo-1.0.5.dist-info → ygo-1.0.6b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.5
+Version: 1.0.6b0
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
 Requires-Python: >=3.8
@@ -8,6 +8,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clickhouse-driver>=0.2.9
 Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: exchange-calendars>=4.2.8
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: lark>=1.2.2
 Requires-Dist: loguru>=0.7.3

{ygo-1.0.5.dist-info → ygo-1.0.6b0.dist-info}/RECORD

@@ -1,7 +1,7 @@
 qdf/__init__.py,sha256=XwH17ae6kX5Grhb_odgDqkNf6N-ambTVi3LoGzq1Fmc,7519
-qdf/errors.py,sha256=vSzX6S_72O1CA5xwzyC2hrVdPCqp6gfqV09Lh-TZ-ns,651
-qdf/expr.py,sha256=qRksAdc9TFUy-_li3DBDuDqsy8iWYTqDdCW3TmtPsZ0,8268
-qdf/qdf.py,sha256=zTFBLanMPTXfgW_lno8UI5Sg5TXUkw4Bnx6d3dnO2fw,5340
+qdf/errors.py,sha256=lJhhjDRdQOOKUFGlLQ9ELK4AexXBwYQSYus_V-kc5K8,1180
+qdf/expr.py,sha256=ck_BHMCV29Q8-szci1_v4ud964QI7JoRRcmA0ppupsc,8454
+qdf/qdf.py,sha256=mMNr6fn6oak1R2_LBGKcWvPdbmM6avxFHXI9lJmzXNU,6641
 qdf/udf/__init__.py,sha256=DdrSGaCB__5C1YL0vd_5rjIB3KLrAKn3h3k9k50L0jA,313
 qdf/udf/base_udf.py,sha256=6VDaCIGNLJxZ7UsoIDWtTH6PzUDj89b8FiwN-TEat2g,3437
 qdf/udf/cs_udf.py,sha256=HT3EKBwAhOxOFDQnpfwb4YcMTT3-lqFXkdysdn5_FI4,3179
@@ -16,10 +16,10 @@ ycat/yck.py,sha256=FlGMBuKEngB4TwFXMp4P3dLg9IfFmUg3eDqXzQ0kQoI,2738
 ygo/__init__.py,sha256=FMN06Tfa8_oV26eklBZCtGTyHZ6MghHxHj4PS_FSXCA,222
 ygo/exceptions.py,sha256=4Kd92kpwpsXHJJkSv4OqcN--PEEvIGGvDDgOOsk68gg,385
 ygo/ygo.py,sha256=vCMUur_41yY0QB4gj8K5wBZHql_cbmANhI8QwPRCTmo,11613
-ygo-1.0.5.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
+ygo-1.0.6b0.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
 ylog/__init__.py,sha256=2sIp4PHNoQMCi0QtIarTI4raACd7SdRHNY7fY5hKYwc,397
-ylog/core.py,sha256=To9kTRVUy5tpg299h0psDov3MFsBUhOHplqT2S_B4w4,7920
-ygo-1.0.5.dist-info/METADATA,sha256=_PzF9Of2_ECmmB3f9G09anaRVXgUlpGcuiH4mM6R6H4,2035
-ygo-1.0.5.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-ygo-1.0.5.dist-info/top_level.txt,sha256=FGbsOtsHgqWzZ9mGRSTCg0pLZEErR1lq5TFQSy2TL1w,18
-ygo-1.0.5.dist-info/RECORD,,
+ylog/core.py,sha256=jmz9JhklbVCQz-zahEXV6P-LEHnqU6opnY4CUEyo8Ss,7924
+ygo-1.0.6b0.dist-info/METADATA,sha256=1aSVsl0EZMP7YzHEW1omgNvKY6TE3QbW_rOhnNqBIFU,2078
+ygo-1.0.6b0.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+ygo-1.0.6b0.dist-info/top_level.txt,sha256=FGbsOtsHgqWzZ9mGRSTCg0pLZEErR1lq5TFQSy2TL1w,18
+ygo-1.0.6b0.dist-info/RECORD,,

ylog/core.py

@@ -94,7 +94,7 @@ class _Logger:
             colorize=True,
             backtrace=self.debug_mode,
             diagnose=self.debug_mode,
-            enqueue=True  # 异步写入
+            # enqueue=True  # 异步写入
         )
 
     def _setup_file_logging(self, retention_days: int, error_retention_days: int):
@@ -128,7 +128,7 @@ class _Logger:
                 compression="zip",
                 backtrace=True,
                 diagnose=self.debug_mode,
-                enqueue=True,  # 异步写入
+                # enqueue=True,  # 异步写入
                 filter=lambda record, lvl=level: record["level"].name == lvl,
                 catch=True  # 捕获格式化异常
             )