ygo 1.0.9__tar.gz → 1.0.10__tar.gz

{ygo-1.0.9/ygo.egg-info → ygo-1.0.10}/PKG-INFO

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.9
+Version: 1.0.10
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: clickhouse-df>=0.1.5
 Requires-Dist: clickhouse-driver>=0.2.9
+Requires-Dist: connectorx>=0.3.3
 Requires-Dist: dynaconf>=3.2.11
 Requires-Dist: exchange-calendars>=4.2.8
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: lark>=1.2.2
+Requires-Dist: lightgbm>=4.6.0
 Requires-Dist: loguru>=0.7.3
+Requires-Dist: mlflow>=2.17.2
 Requires-Dist: pandas>=2.0.3
 Requires-Dist: polars>=1.8.2
 Requires-Dist: pyarrow>=17.0.0
@@ -19,6 +23,7 @@ Requires-Dist: pymysql>=1.1.1
 Requires-Dist: sqlalchemy>=2.0.40
 Requires-Dist: sqlparse>=0.5.3
 Requires-Dist: toolz>=1.0.0
+Requires-Dist: torch>=2.5.1
 Requires-Dist: tqdm>=4.67.1
 Dynamic: license-file
 

{ygo-1.0.9 → ygo-1.0.10}/pyproject.toml

@@ -4,17 +4,21 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygo"
-version = "1.0.9"
+version = "1.0.10"
 description = ""
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
+    "clickhouse-df>=0.1.5",
     "clickhouse-driver>=0.2.9",
+    "connectorx>=0.3.3",
     "dynaconf>=3.2.11",
     "exchange-calendars>=4.2.8",
     "joblib>=1.4.2",
     "lark>=1.2.2",
+    "lightgbm>=4.6.0",
     "loguru>=0.7.3",
+    "mlflow>=2.17.2",
     "pandas>=2.0.3",
     "polars>=1.8.2",
     "pyarrow>=17.0.0",
@@ -22,12 +26,13 @@ dependencies = [
     "sqlalchemy>=2.0.40",
     "sqlparse>=0.5.3",
     "toolz>=1.0.0",
+    "torch>=2.5.1",
     "tqdm>=4.67.1",
 ]
 
 [tool.setuptools.packages.find]
 where = ["."]
-include = ["ygo", "ygo.*", "ylog", "ylog.*", "ycat", "ycat.*", "qdf", "qdf.*"]
+include = ["ygo", "ygo.*", "ylog", "ylog.*", "ycat", "ycat.*",]
 
 [project.urls]
 homepage = "https://github.com/link-yundi/ygo"

{ygo-1.0.9 → ygo-1.0.10}/ycat/__init__.py

@@ -13,12 +13,12 @@ from .client import (
     get_settings,
     sql,
     put,
-    create_engine_ck,
-    create_engine_mysql,
-    read_mysql,
-    read_ck,
     tb_path,
+    read_ck,
+    read_mysql,
 )
+from .qdf import from_polars
+from .updator import Updator
 
 __all__ = [
     "HOME",
@@ -26,9 +26,8 @@ __all__ = [
     "get_settings",
     "sql",
     "put",
-    "create_engine_ck",
-    "create_engine_mysql",
-    "read_mysql",
-    "read_ck",
     "tb_path",
+    "read_ck",
+    "read_mysql",
+    "Updator",
 ]

ygo-1.0.10/ycat/client.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2024/7/1 09:44
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import os
+import re
+import urllib
+from pathlib import Path
+
+import clickhouse_df
+import polars as pl
+from dynaconf import Dynaconf
+
+import ylog
+from .parse import extract_table_names_from_sql
+
+# 配置文件在 “~/.catdb/setting.toml”
+USERHOME = os.path.expanduser('~')  # 用户家目录
+NAME = "catdb"
+CONFIG_PATH = os.path.join(USERHOME, f".{NAME}", "settings.toml")
+if not os.path.exists(CONFIG_PATH):
+    try:
+        os.makedirs(os.path.dirname(CONFIG_PATH))
+    except FileExistsError as e:
+        ...
+    except Exception as e:
+        ylog.error(f"配置文件生成失败: {e}")
+    catdb_path = os.path.join(USERHOME, NAME)
+    template_content = f"""[paths]
+{NAME}="{catdb_path}"  # 本地数据库，默认家目录
+
+## 数据库配置：
+[database]
+[database.ck]
+# urls=["<host1>:<port1>", "<host2>:<port2>",]
+# user="xxx"
+# password="xxxxxx"
+[database.jy]
+# url="<host>:<port>"
+# user="xxxx"
+# password="xxxxxx"
+
+## 视情况自由增加其他配置
+    """
+    with open(CONFIG_PATH, "w") as f:
+        f.write(template_content)
+    ylog.info(f"生成配置文件: {CONFIG_PATH}")
+
+
+def get_settings():
+    try:
+        return Dynaconf(settings_files=[CONFIG_PATH])
+    except Exception as e:
+        ylog.error(f"读取配置文件失败: {e}")
+        return {}
+
+
+HOME = USERHOME
+CATDB = os.path.join(HOME, NAME)
+# 读取配置文件覆盖
+SETTINGS = get_settings()
+if SETTINGS is not None:
+    CATDB = SETTINGS["PATHS"][NAME]
+    if not CATDB.endswith(NAME):
+        CATDB = os.path.join(CATDB, NAME)
+
+
+# ======================== 本地数据库 catdb ========================
+def tb_path(tb_name: str) -> Path:
+    """
+    返回指定表名 完整的本地路径
+    Parameters
+    ----------
+    tb_name: str
+       表名，路径写法: a/b/c
+    Returns
+    -------
+        full_abs_path: pathlib.Path
+        完整的本地绝对路径 $HOME/catdb/a/b/c
+    """
+    return Path(CATDB, tb_name)
+
+
+def put(df: pl.DataFrame, tb_name: str, partitions: list[str] | None = None, abs_path: bool = False):
+    if not abs_path:
+        tbpath = tb_path(tb_name)
+    else:
+        tbpath = tb_name
+    if not tbpath.exists():
+        os.makedirs(tbpath, exist_ok=True)
+    if partitions is not None:
+        df.write_parquet(tbpath, partition_by=partitions)
+    else:
+        df.write_parquet(tbpath / "data.parquet")
+
+
+def sql(query: str, abs_path: bool = False, lazy: bool = True):
+    tbs = extract_table_names_from_sql(query)
+    convertor = dict()
+    for tb in tbs:
+        if not abs_path:
+            db_path = tb_path(tb)
+        else:
+            db_path = tb
+        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
+        convertor[tb] = format_tb
+    pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
+    new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
+    if not lazy:
+        return pl.sql(new_query).collect()
+    return pl.sql(new_query)
+
+
+def read_mysql(query: str, db_conf: str = "database.mysql") -> pl.DataFrame:
+    """
+    读取 mysql 返回 polars.DataFrame
+    :param query:
+    :param db_conf: .catdb/settings.toml 中的 database 配置
+    :return: polars.DataFrame
+    """
+    try:
+        db_setting = get_settings().get(db_conf, {})
+        if not isinstance(db_setting, dict):
+            raise ValueError(f"Database configuration '{db_conf}' is not a dictionary.")
+
+        required_keys = ['user', 'password', 'url']
+        missing_keys = [key for key in required_keys if key not in db_setting]
+        if missing_keys:
+            raise KeyError(f"Missing required keys in database config: {missing_keys}")
+
+        user = urllib.parse.quote_plus(db_setting['user'])
+        password = urllib.parse.quote_plus(db_setting['password'])
+        uri = f"mysql://{user}:{password}@{db_setting['url']}"
+        return pl.read_database_uri(query, uri)
+
+    except KeyError as e:
+        raise RuntimeError("Database configuration error: missing required fields.") from e
+    except Exception as e:
+        raise RuntimeError(f"Failed to execute MySQL query: {e}") from e
+
+
+def read_ck(query: str, db_conf: str = "database.ck") -> pl.DataFrame:
+    """
+    读取 clickhouse 集群 返回 polars.DataFrame
+    :param query:
+    :param db_conf: .catdb/settings.toml 中的 database 配置
+    :return: polars.DataFrame
+    """
+    try:
+        db_setting = get_settings().get(db_conf, {})
+        if not isinstance(db_setting, dict):
+            raise ValueError(f"Database configuration '{db_conf}' is not a dictionary.")
+
+        required_keys = ['user', 'password', 'urls']
+        missing_keys = [key for key in required_keys if key not in db_setting]
+        if missing_keys:
+            raise KeyError(f"Missing required keys in database config: {missing_keys}")
+
+        user = urllib.parse.quote_plus(db_setting['user'])
+        password = urllib.parse.quote_plus(db_setting['password'])
+
+        with clickhouse_df.connect(db_setting['urls'], user=user, password=password):
+            return clickhouse_df.to_polars(query)
+
+    except KeyError as e:
+        raise RuntimeError("Database configuration error: missing required fields.") from e
+    except Exception as e:
+        raise RuntimeError(f"Failed to execute ClickHouse query: {e}") from e

{ygo-1.0.9 → ygo-1.0.10}/ycat/parse.py

@@ -61,6 +61,4 @@ def extract_table_names_from_sql(sql_query):
     return table_names
 
 
-if __name__ == '__main__':
-    print(extract_table_names_from_sql("select * from c.a/b/c/d"))
 

{ygo-1.0.9 → ygo-1.0.10/ycat}/qdf/__init__.py

@@ -29,6 +29,7 @@ def signature(function):
 
 
 def from_polars(df: pl.DataFrame, index: tuple[str] = ("date", "time", "asset"), align: bool = True, ) -> QDF:
+    """polars dataframe 转为 表达式数据库"""
     return QDF(df, index, align,)
 
 

ygo-1.0.10/ycat/updator.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/5/23 01:34
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import ygo
+import ylog
+from .client import CATDB
+
+DATE_FORMAT = "%Y-%m-%d"
+
+
+class Updator:
+    """
+    数据更新器
+    路径：{ycat.CATDB}/updator/{name}
+    """
+
+    def __init__(self, name: str, update_time="16:30"):
+        """
+        数据更新器
+        :param name: 数据更新器名称
+        :param update_time: 数据更新时间，默认16:30
+        """
+        self.name = name
+        self._tb_path = Path(CATDB) / "updator" / name
+        os.makedirs(self._tb_path, exist_ok=True)
+        self._update_time = update_time
+        self.present = datetime.now().today()
+
+        if self.present.strftime("%H:%M") >= self._update_time:
+            self.last_date = self.present.strftime(DATE_FORMAT)
+        else:
+            self.last_date = (self.present - timedelta(days=1)).strftime(DATE_FORMAT)
+
+        self._tasks = list()
+        self._last_run_file = self._tb_path / f".last_run"
+        self.logger = ylog.get_logger("updator")
+
+    @property
+    def last_update_date(self):
+        return self._read_last_run_date()
+
+    def _read_last_run_date(self):
+        if self._last_run_file.exists():
+            with open(self._last_run_file, "r") as f:
+                return f.read().strip()
+        return
+
+    def _write_last_run_date(self, date_str: str):
+        with open(self._last_run_file, "w") as f:
+            f.write(date_str)
+
+    def wrap_fn(self, task_name: str, update_fn: callable):
+        """包装函数，添加异常处理"""
+        try:
+            update_fn()
+            return 0
+        except Exception as e:
+            self.logger.error(ygo.FailTaskError(task_name=task_name, error=e))
+            return 1
+
+    def add_task(self, task_name: str, update_fn: callable):
+        """添加任务"""
+        self._tasks.append((task_name, ygo.delay(self.wrap_fn)(task_name=task_name, update_fn=update_fn)))
+
+    def do(self,
+           overwrite: bool = False,
+           n_jobs: int = 10,
+           backend: str = "threading"):
+        """
+        执行任务
+        :param overwrite: 是否覆盖现有数据
+        :param n_jobs: 并发数
+        :param backend: loky/threading/multiprocessing
+        :return:
+        """
+        if not overwrite:
+            local_last_date = self._read_last_run_date()
+            if local_last_date is not None:
+                if local_last_date >= self.last_date:
+                    self.logger.info(f"[{self.name}] 已是最新数据，跳过更新")
+                    return
+        self.logger.info(f"[{self.name}] 更新数据")
+        failed_num = 0
+        with ygo.pool(n_jobs=n_jobs, backend=backend) as go:
+            for task_name, task in self._tasks:
+                go.submit(task, job_name=task_name)()
+            for status in go.do():
+                failed_num += status
+        if failed_num < 1:
+            self._write_last_run_date(self.last_date)
+            self.logger.info(f"[{self.name}] 更新成功，最新数据日期：{self.last_date}")
+        self.logger.info(f"[{self.name}] 更新完成，失败任务数：{str(failed_num).zfill(2)}/{str(len(self._tasks)).zfill(2)}")

ygo-1.0.10/ygo/__init__.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/4/28 15:25
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+
+from .exceptions import FailTaskError
+from .ygo import (
+    delay,
+    fn_params,
+    fn_signature_params,
+    fn_path,
+    fn_code,
+    fn_info,
+    module_from_str,
+    fn_from_str,
+    pool,
+)
+
+__all__ = [
+    "FailTaskError",
+    "delay",
+    "fn_params",
+    "fn_signature_params",
+    "fn_path",
+    "fn_code",
+    "fn_info",
+    "fn_from_str",
+    "module_from_str",
+    "pool"
+]

{ygo-1.0.9 → ygo-1.0.10}/ygo/exceptions.py

@@ -7,7 +7,23 @@ Created on 2024/12/18 下午7:01
 ---------------------------------------------
 """
 
+from dataclasses import dataclass
+
 class WarnException(Exception):
     """自定义异常类，仅用于警告"""
     def __init__(self, message):
-        super().__init__(message)  # 调用父类的构造函数
+        super().__init__(message)  # 调用父类的构造函数
+
+@dataclass
+class FailTaskError:
+    task_name: str
+    error: Exception
+
+    def __str__(self):
+        return f"""
+[失败任务]: {self.task_name}
+[错误信息]: \n{self.error}
+"""
+
+    def __repr__(self):
+        return self.__str__()

{ygo-1.0.9 → ygo-1.0.10}/ygo/ygo.py

@@ -18,7 +18,7 @@ from pathlib import Path
 from joblib import Parallel, delayed
 
 import ylog
-from .exceptions import WarnException
+from .exceptions import WarnException, FailTaskError
 
 with warnings.catch_warnings():
     warnings.simplefilter("ignore")
@@ -228,22 +228,10 @@ def run_job(job, task_id, queue):
     try:
         result = job()
     except WarnException as e:
-        warn_msg = f"""
-=============================================================
-{job.task_name}: {job.task_id}
-{e}
-=============================================================
-                    """
-        ylog.warning(warn_msg)
+        ylog.warning(FailTaskError(task_name=job.task_name, error=e))
         result = None
     except Exception as e:
-        error_msg = f"""
-=============================================================
-{job.task_name}: {job.task_id}
-{e}
-=============================================================
-                    """
-        ylog.error(error_msg)
+        ylog.error(FailTaskError(task_name=job.task_name, error=e))
         result = None
     queue.put((task_id, 1))
     return result

{ygo-1.0.9 → ygo-1.0.10/ygo.egg-info}/PKG-INFO

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.9
+Version: 1.0.10
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: clickhouse-df>=0.1.5
 Requires-Dist: clickhouse-driver>=0.2.9
+Requires-Dist: connectorx>=0.3.3
 Requires-Dist: dynaconf>=3.2.11
 Requires-Dist: exchange-calendars>=4.2.8
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: lark>=1.2.2
+Requires-Dist: lightgbm>=4.6.0
 Requires-Dist: loguru>=0.7.3
+Requires-Dist: mlflow>=2.17.2
 Requires-Dist: pandas>=2.0.3
 Requires-Dist: polars>=1.8.2
 Requires-Dist: pyarrow>=17.0.0
@@ -19,6 +23,7 @@ Requires-Dist: pymysql>=1.1.1
 Requires-Dist: sqlalchemy>=2.0.40
 Requires-Dist: sqlparse>=0.5.3
 Requires-Dist: toolz>=1.0.0
+Requires-Dist: torch>=2.5.1
 Requires-Dist: tqdm>=4.67.1
 Dynamic: license-file
 

{ygo-1.0.9 → ygo-1.0.10}/ygo.egg-info/SOURCES.txt

@@ -1,21 +1,20 @@
 LICENSE
 README.md
 pyproject.toml
-qdf/__init__.py
-qdf/errors.py
-qdf/expr.py
-qdf/qdf.py
-qdf/udf/__init__.py
-qdf/udf/base_udf.py
-qdf/udf/cs_udf.py
-qdf/udf/d_udf.py
-qdf/udf/ind_udf.py
-qdf/udf/ts_udf.py
 ycat/__init__.py
 ycat/client.py
-ycat/dtype.py
 ycat/parse.py
-ycat/yck.py
+ycat/updator.py
+ycat/qdf/__init__.py
+ycat/qdf/errors.py
+ycat/qdf/expr.py
+ycat/qdf/qdf.py
+ycat/qdf/udf/__init__.py
+ycat/qdf/udf/base_udf.py
+ycat/qdf/udf/cs_udf.py
+ycat/qdf/udf/d_udf.py
+ycat/qdf/udf/ind_udf.py
+ycat/qdf/udf/ts_udf.py
 ygo/__init__.py
 ygo/exceptions.py
 ygo/ygo.py

{ygo-1.0.9 → ygo-1.0.10}/ygo.egg-info/requires.txt

@@ -1,9 +1,13 @@
+clickhouse-df>=0.1.5
 clickhouse-driver>=0.2.9
+connectorx>=0.3.3
 dynaconf>=3.2.11
 exchange-calendars>=4.2.8
 joblib>=1.4.2
 lark>=1.2.2
+lightgbm>=4.6.0
 loguru>=0.7.3
+mlflow>=2.17.2
 pandas>=2.0.3
 polars>=1.8.2
 pyarrow>=17.0.0
@@ -11,4 +15,5 @@ pymysql>=1.1.1
 sqlalchemy>=2.0.40
 sqlparse>=0.5.3
 toolz>=1.0.0
+torch>=2.5.1
 tqdm>=4.67.1

{ygo-1.0.9 → ygo-1.0.10}/ygo.egg-info/top_level.txt

@@ -1,4 +1,3 @@
-qdf
 ycat
 ygo
 ylog

{ygo-1.0.9 → ygo-1.0.10}/ylog/__init__.py

@@ -7,7 +7,7 @@ Created on 2025/5/14 15:37
 ---------------------------------------------
 """
 
-from .core import trace, debug, info, warning, error, critical, update_config
+from .core import trace, debug, info, warning, error, critical, update_config, get_logger
 
 __all__ = [
     "trace",
@@ -16,5 +16,6 @@ __all__ = [
     "warning",
     "error",
     "critical",
-    "update_config"
+    "update_config",
+    "get_logger",
 ]