expr-codegen 0.12.1__tar.gz → 0.13.0__tar.gz

expr_codegen-0.13.0/.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: expr_codegen
-Version: 0.12.1
+Version: 0.13.0
 Summary: symbol expression to polars expression tool
 Author-email: wukan <wu-kan@163.com>
 License: BSD 3-Clause License
@@ -31,24 +31,22 @@ License: BSD 3-Clause License
         CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
         OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
         OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-        
-Keywords: polars,expression,talib
+License-File: LICENSE
+Keywords: expression,polars,talib
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
 Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
+Requires-Dist: ast-comments
 Requires-Dist: black
-Requires-Dist: Jinja2
-Requires-Dist: networkx
+Requires-Dist: jinja2
 Requires-Dist: loguru
+Requires-Dist: networkx
 Requires-Dist: sympy
-Requires-Dist: ast-comments
 Provides-Extra: streamlit
-Requires-Dist: streamlit; extra == "streamlit"
-Requires-Dist: streamlit-ace; extra == "streamlit"
-Requires-Dist: more_itertools; extra == "streamlit"
-Dynamic: license-file
+Requires-Dist: more-itertools; extra == 'streamlit'
+Requires-Dist: streamlit; extra == 'streamlit'
+Requires-Dist: streamlit-ace; extra == 'streamlit'
+Description-Content-Type: text/markdown
 
 # expr_codegen 表达式转译器
 
@@ -81,6 +79,8 @@ https://exprcodegen.streamlit.app
 import sys
 from io import StringIO
 
+import polars as pl
+
 from expr_codegen import codegen_exec
 
 
@@ -109,18 +109,20 @@ def _code_block_2():
     CPV = cs_zscore(_corr) + cs_zscore(_beta)
 
 
-code = StringIO()
+code = codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by', output_file=sys.stdout)  # 打印代码
+code = codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by', output_file="output.py")  # 保存到文件
+code = codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by')  # 只执行，不保存代码
 
-df = None  # 替换成真实的polars数据
-df = codegen_exec(df, _code_block_1, _code_block_2, output_file=sys.stdout)  # 打印代码
-df = codegen_exec(df, _code_block_1, _code_block_2, output_file="output.py")  # 保存到文件
-df = codegen_exec(df, _code_block_1, _code_block_2)  # 只执行，不保存代码
-df = codegen_exec(df, _code_block_1, _code_block_2, output_file=code)  # 保存到字符串
+code = StringIO()
+codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by', output_file=code)  # 保存到字符串
 code.seek(0)
 code.read()  # 读取代码
 
-df = codegen_exec(df.lazy(), _code_block_1, _code_block_2).collect()  # Lazy CPU
-df = codegen_exec(df.lazy(), _code_block_1, _code_block_2).collect(engine="gpu")  # Lazy GPU
+# TODO 替换成合适的数据
+df = pl.DataFrame()
+df = codegen_exec(df.lazy(), _code_block_1, _code_block_2, over_null='partition_by').collect()  # Lazy CPU
+df = codegen_exec(df.lazy(), _code_block_1, _code_block_2, over_null='partition_by').collect(engine="gpu")  # Lazy GPU
+
 ```
 
 ## 目录结构

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/README.md

@@ -29,6 +29,8 @@ https://exprcodegen.streamlit.app
 import sys
 from io import StringIO
 
+import polars as pl
+
 from expr_codegen import codegen_exec
 
 
@@ -57,18 +59,20 @@ def _code_block_2():
     CPV = cs_zscore(_corr) + cs_zscore(_beta)
 
 
-code = StringIO()
+code = codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by', output_file=sys.stdout)  # 打印代码
+code = codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by', output_file="output.py")  # 保存到文件
+code = codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by')  # 只执行，不保存代码
 
-df = None  # 替换成真实的polars数据
-df = codegen_exec(df, _code_block_1, _code_block_2, output_file=sys.stdout)  # 打印代码
-df = codegen_exec(df, _code_block_1, _code_block_2, output_file="output.py")  # 保存到文件
-df = codegen_exec(df, _code_block_1, _code_block_2)  # 只执行，不保存代码
-df = codegen_exec(df, _code_block_1, _code_block_2, output_file=code)  # 保存到字符串
+code = StringIO()
+codegen_exec(None, _code_block_1, _code_block_2, over_null='partition_by', output_file=code)  # 保存到字符串
 code.seek(0)
 code.read()  # 读取代码
 
-df = codegen_exec(df.lazy(), _code_block_1, _code_block_2).collect()  # Lazy CPU
-df = codegen_exec(df.lazy(), _code_block_1, _code_block_2).collect(engine="gpu")  # Lazy GPU
+# TODO 替换成合适的数据
+df = pl.DataFrame()
+df = codegen_exec(df.lazy(), _code_block_1, _code_block_2, over_null='partition_by').collect()  # Lazy CPU
+df = codegen_exec(df.lazy(), _code_block_1, _code_block_2, over_null='partition_by').collect(engine="gpu")  # Lazy GPU
+
 ```
 
 ## 目录结构

expr_codegen-0.13.0/expr_codegen/_version.py

@@ -0,0 +1 @@
+__version__ = "0.13.0"

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/expr_codegen/codes.py

@@ -381,15 +381,16 @@ def sources_to_asts(*sources, convert_xor: bool):
         if isinstance(node, ast.Assign):
             assigns.append(node)
             continue
-        if isinstance(node, ast_comments.Comment):
-            # 添加注释
-            if node.inline and isinstance(assigns[-1], ast.Assign):
-                assigns.append(node)
-                continue
         # TODO 是否要把其它语句也加入？是否有安全问题？
         if isinstance(node, (ast.Import, ast.ImportFrom)):
             raw.append(node)
             continue
+        if isinstance(node, ast_comments.Comment):
+            # 添加注释
+            if node.inline and isinstance(tree.body[i - 1], ast.Assign):
+                assigns.append(node)
+                continue
+
     return raw_to_code(raw), assigns_to_list(assigns), t.funcs_new, t.args_new, t.targets_new
 
 

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/expr_codegen/model.py

@@ -1,10 +1,10 @@
 from functools import reduce
-from itertools import product
+from itertools import product, permutations
 
 import networkx as nx
 from sympy import symbols
 
-from expr_codegen.dag import zero_indegree, hierarchy_pos, remove_paths_by_zero_outdegree, zero_outdegree
+from expr_codegen.dag import zero_indegree, hierarchy_pos, remove_paths_by_zero_outdegree
 from expr_codegen.expr import CL, get_symbols, get_children, get_key, is_simple_expr
 
 _RESERVED_WORD_ = {'_NONE_', '_TRUE_', '_FALSE_'}
@@ -122,6 +122,18 @@ class ListDictList:
         return l3
 
 
+def score1(row) -> int:
+    # 首尾相连打分加1
+    lst = [None] + [key for r in row for key in dict(r).keys()]
+    return sum([x == y for x, y in zip(lst[:-1], lst[1:])])
+
+
+def score2(row) -> float:
+    # 最后一个ts越靠前，打分越高
+    lst = ['ts'] + [key[0] for r in row for key in dict(r).keys()]
+    return lst[::-1].index('ts') / len(lst)
+
+
 def chain_create(nested_list):
     """接龙。多个列表，头尾相连
 
@@ -131,63 +143,22 @@ def chain_create(nested_list):
     alpha_031 = ((cs_rank(cs_rank(cs_rank(ts_decay_linear((-1 * cs_rank(cs_rank(ts_delta(CLOSE, 10)))), 10))))))
 
     """
-    # 两两取交集，交集为{}时，添加一个{None}，防止product时出错
-    neighbor_inter = [set(x) & set(y) or {None} for x, y in zip(nested_list[:-1], nested_list[1:])]
+    perms = []
+    for d in nested_list:
+        # 每一层生成排列
+        perms.append(permutations(d.items()))
 
-    # 查找最小数字，表示两两不重复
-    last_min = float('inf')
-    # 最小不重复的一行记录
+    last_score = float('-inf')
     last_row = None
-    last_rows = set()
-    for row in product(*neighbor_inter):
-        # 判断两两是否重复，重复为1，反之为0
-        result = sum([x == y for x, y in zip(row[:-1], row[1:])])
-        if last_min > result:
-            last_min = result
+    # 生成笛卡尔积
+    for row in product(*perms):
+        result = score1(row) + score2(row)
+        # print(result, row)
+        if result > last_score:
+            last_score = result
             last_row = row
-        if result == 0:
-            last_rows.add(last_row)
-            last_min = float('inf')
-            continue
-    last_rows.add(last_row)
-    last_rows = list(last_rows)
-
-    # last_rows中有多个满足条件的，优先保证最后一组ts在最前，ts后可提前filter减少计算量
-    last_row = last_rows[0]
-    for row in last_rows:
-        if len(row) == 0:
-            # 一行表达式
-            continue
-        if row[-1] is None:
-            continue
-        if row[-1][0] == 'ts':
-            last_row = row
-            break
-
-    # 如何移动才是难点 如果两个连续 ts/ts，那么如何移动
-
-    # 调整后的第0列
-    head = [None] + list(last_row)
-    # 调整后的第-1列
-    tail = list(last_row) + [None]
-
-    # 调整新列表
-    arr = []
-    for ll, hh, tt in zip(nested_list, head, tail):
-        d = []
-        for k, v in ll.items():
-            if len(d) == 0:
-                d.append((k, v))
-                continue
-            if k == hh:
-                d.insert(0, (k, v))
-            elif k == tt:
-                d.append((k, v))
-            else:
-                d.insert(1, (k, v))
-        arr.append(dict(d))
 
-    return arr
+    return [dict(ro) for ro in last_row]
 
 
 # ==========================
@@ -425,6 +396,7 @@ def dag_end(G):
 
             exprs_ldl.append(key, (node, expr, symbols, comment))
 
+    # 第0层是CLOSE等基础因子，剔除
     exprs_ldl._list = exprs_ldl.values()[1:]
 
     return exprs_ldl, G

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/expr_codegen/pandas/code.py

@@ -38,6 +38,7 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
             filename,
             date='date', asset='asset',
             extra_codes: Sequence[str] = (),
+            filter_last: bool = False,
             **kwargs):
     """基于模板的代码生成"""
     if filename is None:
@@ -53,7 +54,7 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
     # 处理过后的表达式
     exprs_dst = []
     syms_out = []
-
+    ts_func_name = None
     drop_symbols = exprs_ldl.drop_symbols()
     j = -1
     for i, row in enumerate(exprs_ldl.values()):
@@ -78,6 +79,7 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
             if len(groupbys['sort']) == 0:
                 groupbys['sort'] = f'df = df.sort_values(by=[_ASSET_, _DATE_]).reset_index(drop=True)'
             if k[0] == TS:
+                ts_func_name = func_name
                 # 时序需要排序
                 func_code = [f'    g.df = df.sort_values(by=[_DATE_])'] + func_code
             else:
@@ -93,6 +95,15 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
 
     syms1 = symbols_to_code(syms_dst)
     syms2 = symbols_to_code(syms_out)
+    if filter_last:
+        _groupbys = {'sort': groupbys['sort']}
+        if ts_func_name is None:
+            _groupbys['_filter_last'] = "df = filter_last(df.sort_values(by=[_DATE_]))"
+        for k, v in groupbys.items():
+            _groupbys[k] = v
+            if k == ts_func_name:
+                _groupbys[k + '_filter_last'] = "df = filter_last(df)"
+        groupbys = _groupbys
 
     try:
         env = jinja2.Environment(loader=FileSystemLoader(os.path.dirname(__file__)))

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/expr_codegen/pandas/printer.py

@@ -54,6 +54,8 @@ class PandasStrPrinter(StrPrinter):
             self._print_level -= 1
 
     def _print_Symbol(self, expr):
+        if expr.name in ('_NONE_', '_TRUE_', '_FALSE_'):
+            return expr.name
         return f"g[{expr.name}]"
 
     def _print_Equality(self, expr):

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/expr_codegen/pandas/ta.py

@@ -73,6 +73,9 @@ def ts_delay(x: pd.Series, d: int = 1) -> pd.Series:
 def ts_delta(x: pd.Series, d: int = 1) -> pd.Series:
     return x.diff(d)
 
+def ts_returns(x: pd.Series, d: int = 1) -> pd.Series:
+    return x.pct_change(d)
+
 
 def ts_max(x: pd.Series, d: int = 5) -> pd.Series:
     return x.rolling(d).max()

{expr_codegen-0.12.1 → expr_codegen-0.13.0}/expr_codegen/pandas/template.py.j2

@@ -22,6 +22,7 @@ _FALSE_ = False
 
 g = GlobalVariable()
 
+
 def unpack(x: Tuple, idx: int = 0) -> pd.Series:
     return x[idx]
 
@@ -30,9 +31,11 @@ def unpack(x: Tuple, idx: int = 0) -> pd.Series:
 {% endfor %}
 
 {% for key, value in funcs.items() %}
+
 def {{ key }}(df: pd.DataFrame) -> pd.DataFrame:
 {{ value }}
     return g.df
+
 {% endfor %}
 
 """
@@ -48,8 +51,12 @@ def {{ key }}(df: pd.DataFrame) -> pd.DataFrame:
 """
 
 
+def filter_last(df: pd.DataFrame) -> pd.DataFrame:
+    """过滤数据，只取最后一天。实盘时可用于减少计算量"""
+    return df[df[_DATE_] >= df[_DATE_].iloc[-1]]
+
+
 def main(df: pd.DataFrame) -> pd.DataFrame:
-    # logger.info("start...")
     {% for key, value in groupbys.items() %}
     {{ value-}}
     {% endfor %}
@@ -57,13 +64,4 @@ def main(df: pd.DataFrame) -> pd.DataFrame:
     # drop intermediate columns
     df = df.drop(columns=list(filter(lambda x: x.startswith("_"), df.columns)))
 
-    # logger.info('done')
-
-    # save
-    # df.to_parquet('output.parquet', compression='zstd')
-
     return df
-
-# if __name__ in ("__main__", "builtins"):
-#     # TODO: 数据加载或外部传入
-#     df_output = main(df_input)

{expr_codegen-0.12.1/expr_codegen/polars_over → expr_codegen-0.13.0/expr_codegen/polars}/code.py

@@ -7,7 +7,7 @@ from jinja2 import FileSystemLoader, TemplateNotFound
 
 from expr_codegen.expr import TS, CS, GP
 from expr_codegen.model import ListDictList
-from expr_codegen.polars_over.printer import PolarsStrPrinter
+from expr_codegen.polars.printer import PolarsStrPrinter
 
 
 def get_groupby_from_tuple(tup, func_name, drop_cols):
@@ -40,6 +40,7 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
             date='date', asset='asset',
             extra_codes: Sequence[str] = (),
             over_null: Literal['order_by', 'partition_by', None] = 'partition_by',
+            filter_last: bool = False,
             **kwargs):
     """基于模板的代码生成"""
     if filename is None:
@@ -58,7 +59,7 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
     # 处理过后的表达式
     exprs_dst = []
     syms_out = []
-
+    ts_func_name = None
     drop_symbols = exprs_ldl.drop_symbols()
     j = -1
     for i, row in enumerate(exprs_ldl.values()):
@@ -85,6 +86,7 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
                         # 不想等，打印注释，显示会更直观察
                         func_code.append(f"# {va} = {s1}")
                     if k[0] == TS:
+                        ts_func_name = func_name
                         # https://github.com/pola-rs/polars/issues/12925#issuecomment-2552764629
                         _sym = [f"{s}.is_not_null()" for s in set(sym)]
                         if len(_sym) > 1:
@@ -118,6 +120,15 @@ def codegen(exprs_ldl: ListDictList, exprs_src, syms_dst,
 
     syms1 = symbols_to_code(syms_dst)
     syms2 = symbols_to_code(syms_out)
+    if filter_last:
+        _groupbys = {'sort': groupbys['sort']}
+        if ts_func_name is None:
+            _groupbys['_filter_last'] = "df = filter_last(df.sort(_DATE_))"
+        for k, v in groupbys.items():
+            _groupbys[k] = v
+            if k == ts_func_name:
+                _groupbys[k + '_filter_last'] = "df = filter_last(df)"
+        groupbys = _groupbys
 
     try:
         env = jinja2.Environment(loader=FileSystemLoader(os.path.dirname(__file__)))

{expr_codegen-0.12.1/expr_codegen/polars_over → expr_codegen-0.13.0/expr_codegen/polars}/template.py.j2

@@ -8,6 +8,7 @@ import polars.selectors as cs  # noqa
 # from loguru import logger  # noqa
 from polars import DataFrame as _pl_DataFrame
 from polars import LazyFrame as _pl_LazyFrame
+
 # ===================================
 # 导入优先级，例如：ts_RSI在ta与talib中都出现了，优先使用ta
 # 运行时，后导入覆盖前导入，但IDE智能提示是显示先导入的
@@ -31,6 +32,7 @@ _NONE_ = None
 _TRUE_ = True
 _FALSE_ = False
 
+
 def unpack(x: pl.Expr, idx: int = 0) -> pl.Expr:
     return x.struct[idx]
 
@@ -39,9 +41,11 @@ def unpack(x: pl.Expr, idx: int = 0) -> pl.Expr:
 {% endfor %}
 
 {% for key, value in funcs.items() %}
+
 def {{ key }}(df: DataFrame) -> DataFrame:
 {{ value }}
     return df
+
 {% endfor %}
 
 """
@@ -57,8 +61,17 @@ def {{ key }}(df: DataFrame) -> DataFrame:
 """
 
 
+def filter_last(df: DataFrame) -> DataFrame:
+    """过滤数据，只取最后一天。实盘时可用于减少计算量
+    前一个调用的ts,这里可以直接调用，可以认为已经排序好
+        `df = filter_last(df)`
+    反之
+        `df = filter_last(df.sort(_DATE_))`
+    """
+    return df.filter(pl.col(_DATE_) >= df.select(pl.last(_DATE_))[0, 0])
+
+
 def main(df: DataFrame) -> DataFrame:
-    # logger.info("start...")
     {% for key, value in groupbys.items() %}
     {{ value-}}
     {% endfor %}
@@ -69,15 +82,6 @@ def main(df: DataFrame) -> DataFrame:
 
     # shrink
     df = df.select(cs.all().shrink_dtype())
-    # df = df.shrink_to_fit()
-
-    # logger.info('done')
-
-    # save
-    # df.write_parquet('output.parquet')
 
     return df
 
-# if __name__ in ("__main__", "builtins"):
-#     # TODO: 数据加载或外部传入
-#     df_output = main(df_input)