ygo 1.0.8__py3-none-any.whl → 1.0.10__py3-none-any.whl

ycat/__init__.py

@@ -13,12 +13,12 @@ from .client import (
     get_settings,
     sql,
     put,
-    create_engine_ck,
-    create_engine_mysql,
-    read_mysql,
-    read_ck,
     tb_path,
+    read_ck,
+    read_mysql,
 )
+from .qdf import from_polars
+from .updator import Updator
 
 __all__ = [
     "HOME",
@@ -26,9 +26,8 @@ __all__ = [
     "get_settings",
     "sql",
     "put",
-    "create_engine_ck",
-    "create_engine_mysql",
-    "read_mysql",
-    "read_ck",
     "tb_path",
+    "read_ck",
+    "read_mysql",
+    "Updator",
 ]

ycat/client.py

@@ -8,16 +8,15 @@ Created on 2024/7/1 09:44
 """
 import os
 import re
-from functools import partial
-from typing import Optional
+import urllib
+from pathlib import Path
 
+import clickhouse_df
 import polars as pl
 from dynaconf import Dynaconf
-from sqlalchemy import create_engine
 
 import ylog
 from .parse import extract_table_names_from_sql
-from .yck import connect, query_polars
 
 # 配置文件在 “~/.catdb/setting.toml”
 USERHOME = os.path.expanduser('~')  # 用户家目录
@@ -54,7 +53,7 @@ if not os.path.exists(CONFIG_PATH):
 
 def get_settings():
     try:
-        return Dynaconf(settings_files=[CONFIG_PATH]).as_dict()
+        return Dynaconf(settings_files=[CONFIG_PATH])
     except Exception as e:
         ylog.error(f"读取配置文件失败: {e}")
         return {}
@@ -71,7 +70,7 @@ if SETTINGS is not None:
 
 
 # ======================== 本地数据库 catdb ========================
-def tb_path(tb_name: str) -> str:
+def tb_path(tb_name: str) -> Path:
     """
     返回指定表名 完整的本地路径
     Parameters
@@ -80,26 +79,23 @@ def tb_path(tb_name: str) -> str:
        表名，路径写法: a/b/c
     Returns
     -------
-        full_abs_path: str
+        full_abs_path: pathlib.Path
         完整的本地绝对路径 $HOME/catdb/a/b/c
     """
-    return os.path.join(CATDB, tb_name)
+    return Path(CATDB, tb_name)
 
 
-def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None, abs_path: bool = False):
+def put(df: pl.DataFrame, tb_name: str, partitions: list[str] | None = None, abs_path: bool = False):
     if not abs_path:
         tbpath = tb_path(tb_name)
     else:
         tbpath = tb_name
-    if not os.path.exists(tbpath):
-        try:
-            os.makedirs(tbpath)
-        except FileExistsError as e:
-            pass
+    if not tbpath.exists():
+        os.makedirs(tbpath, exist_ok=True)
     if partitions is not None:
-        for field in partitions:
-            assert field in df.columns, f'dataframe must have Field `{field}`'
-    df.write_parquet(tbpath, partition_by=partitions)
+        df.write_parquet(tbpath, partition_by=partitions)
+    else:
+        df.write_parquet(tbpath / "data.parquet")
 
 
 def sql(query: str, abs_path: bool = False, lazy: bool = True):
@@ -119,27 +115,58 @@ def sql(query: str, abs_path: bool = False, lazy: bool = True):
     return pl.sql(new_query)
 
 
-def create_engine_ck(urls: list[str], user: str, password: str):
-    return partial(connect, urls, user, password)
-
-
-def read_ck(sql, eng) -> pl.DataFrame:
-    with eng() as conn:
-        return query_polars(sql, conn)
+def read_mysql(query: str, db_conf: str = "database.mysql") -> pl.DataFrame:
+    """
+    读取 mysql 返回 polars.DataFrame
+    :param query:
+    :param db_conf: .catdb/settings.toml 中的 database 配置
+    :return: polars.DataFrame
+    """
+    try:
+        db_setting = get_settings().get(db_conf, {})
+        if not isinstance(db_setting, dict):
+            raise ValueError(f"Database configuration '{db_conf}' is not a dictionary.")
+
+        required_keys = ['user', 'password', 'url']
+        missing_keys = [key for key in required_keys if key not in db_setting]
+        if missing_keys:
+            raise KeyError(f"Missing required keys in database config: {missing_keys}")
+
+        user = urllib.parse.quote_plus(db_setting['user'])
+        password = urllib.parse.quote_plus(db_setting['password'])
+        uri = f"mysql://{user}:{password}@{db_setting['url']}"
+        return pl.read_database_uri(query, uri)
+
+    except KeyError as e:
+        raise RuntimeError("Database configuration error: missing required fields.") from e
+    except Exception as e:
+        raise RuntimeError(f"Failed to execute MySQL query: {e}") from e
 
 
-def create_engine_mysql(url, user, password, database):
+def read_ck(query: str, db_conf: str = "database.ck") -> pl.DataFrame:
     """
-    :param url: <host>:<port>
-    :param user:
-    :param password:
-    :param database:
-    :return:
+    读取 clickhouse 集群 返回 polars.DataFrame
+    :param query:
+    :param db_conf: .catdb/settings.toml 中的 database 配置
+    :return: polars.DataFrame
     """
-    engine = create_engine(f"mysql+pymysql://{user}:{password}@{url}/{database}")
-    return engine
+    try:
+        db_setting = get_settings().get(db_conf, {})
+        if not isinstance(db_setting, dict):
+            raise ValueError(f"Database configuration '{db_conf}' is not a dictionary.")
+
+        required_keys = ['user', 'password', 'urls']
+        missing_keys = [key for key in required_keys if key not in db_setting]
+        if missing_keys:
+            raise KeyError(f"Missing required keys in database config: {missing_keys}")
 
+        user = urllib.parse.quote_plus(db_setting['user'])
+        password = urllib.parse.quote_plus(db_setting['password'])
 
-def read_mysql(sql, eng) -> pl.DataFrame:
-    with eng.connect() as conn:
-        return pl.read_database(sql, conn)
+        with clickhouse_df.connect(db_setting['urls'], user=user, password=password):
+            return clickhouse_df.to_polars(query)
+
+    except KeyError as e:
+        raise RuntimeError("Database configuration error: missing required fields.") from e
+    except Exception as e:
+        raise RuntimeError(f"Failed to execute ClickHouse query: {e}") from e

ycat/parse.py

@@ -61,6 +61,4 @@ def extract_table_names_from_sql(sql_query):
     return table_names
 
 
-if __name__ == '__main__':
-    print(extract_table_names_from_sql("select * from c.a/b/c/d"))
 

{qdf → ycat/qdf}/__init__.py

@@ -29,6 +29,7 @@ def signature(function):
 
 
 def from_polars(df: pl.DataFrame, index: tuple[str] = ("date", "time", "asset"), align: bool = True, ) -> QDF:
+    """polars dataframe 转为 表达式数据库"""
     return QDF(df, index, align,)
 
 

{qdf → ycat/qdf}/qdf.py

@@ -111,10 +111,10 @@ class QDF:
                     else:
                         args.append(arg)  # or args.append(pl.lit(arg))
                 try:
-                    expr_pl: pl.Expr = func(*args, **kwargs).alias(alias_)
-                    self._data_ = self._data_.with_columns(expr_pl)
+                    expr_pl: pl.Expr = func(*args, **kwargs)
+                    self._data_ = self._data_.with_columns(expr_pl.alias(alias_))
                     self._cur_expr_cache[expr_] = alias_
-                    return expr_pl
+                    return pl.col(alias_)
                 except Exception as e:
                     raise CompileError(message=f"{expr_.fn_name}({', '.join([str(arg) for arg in args])})\n{e}") from e
 
@@ -156,8 +156,6 @@ class QDF:
                 self.failed.append(FailError(expr, e))
         if self.failed:
             ylog.warning(f"QDF.sql 失败：{len(self.failed)}/{len(exprs)}: \n {self.failed}")
-        # for expr in exprs_to_add:
-        #     data = data.with_columns(expr).fill_nan(None)
         self._data_ = self._data_.with_columns(*exprs_to_add).fill_nan(None)
         new_expr_cache = dict()
         try:

ycat/updator.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/5/23 01:34
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import ygo
+import ylog
+from .client import CATDB
+
+DATE_FORMAT = "%Y-%m-%d"
+
+
+class Updator:
+    """
+    数据更新器
+    路径：{ycat.CATDB}/updator/{name}
+    """
+
+    def __init__(self, name: str, update_time="16:30"):
+        """
+        数据更新器
+        :param name: 数据更新器名称
+        :param update_time: 数据更新时间，默认16:30
+        """
+        self.name = name
+        self._tb_path = Path(CATDB) / "updator" / name
+        os.makedirs(self._tb_path, exist_ok=True)
+        self._update_time = update_time
+        self.present = datetime.now().today()
+
+        if self.present.strftime("%H:%M") >= self._update_time:
+            self.last_date = self.present.strftime(DATE_FORMAT)
+        else:
+            self.last_date = (self.present - timedelta(days=1)).strftime(DATE_FORMAT)
+
+        self._tasks = list()
+        self._last_run_file = self._tb_path / f".last_run"
+        self.logger = ylog.get_logger("updator")
+
+    @property
+    def last_update_date(self):
+        return self._read_last_run_date()
+
+    def _read_last_run_date(self):
+        if self._last_run_file.exists():
+            with open(self._last_run_file, "r") as f:
+                return f.read().strip()
+        return
+
+    def _write_last_run_date(self, date_str: str):
+        with open(self._last_run_file, "w") as f:
+            f.write(date_str)
+
+    def wrap_fn(self, task_name: str, update_fn: callable):
+        """包装函数，添加异常处理"""
+        try:
+            update_fn()
+            return 0
+        except Exception as e:
+            self.logger.error(ygo.FailTaskError(task_name=task_name, error=e))
+            return 1
+
+    def add_task(self, task_name: str, update_fn: callable):
+        """添加任务"""
+        self._tasks.append((task_name, ygo.delay(self.wrap_fn)(task_name=task_name, update_fn=update_fn)))
+
+    def do(self,
+           overwrite: bool = False,
+           n_jobs: int = 10,
+           backend: str = "threading"):
+        """
+        执行任务
+        :param overwrite: 是否覆盖现有数据
+        :param n_jobs: 并发数
+        :param backend: loky/threading/multiprocessing
+        :return:
+        """
+        if not overwrite:
+            local_last_date = self._read_last_run_date()
+            if local_last_date is not None:
+                if local_last_date >= self.last_date:
+                    self.logger.info(f"[{self.name}] 已是最新数据，跳过更新")
+                    return
+        self.logger.info(f"[{self.name}] 更新数据")
+        failed_num = 0
+        with ygo.pool(n_jobs=n_jobs, backend=backend) as go:
+            for task_name, task in self._tasks:
+                go.submit(task, job_name=task_name)()
+            for status in go.do():
+                failed_num += status
+        if failed_num < 1:
+            self._write_last_run_date(self.last_date)
+            self.logger.info(f"[{self.name}] 更新成功，最新数据日期：{self.last_date}")
+        self.logger.info(f"[{self.name}] 更新完成，失败任务数：{str(failed_num).zfill(2)}/{str(len(self._tasks)).zfill(2)}")

ygo/__init__.py

@@ -7,4 +7,28 @@ Created on 2025/4/28 15:25
 ---------------------------------------------
 """
 
-from .ygo import *
+from .exceptions import FailTaskError
+from .ygo import (
+    delay,
+    fn_params,
+    fn_signature_params,
+    fn_path,
+    fn_code,
+    fn_info,
+    module_from_str,
+    fn_from_str,
+    pool,
+)
+
+__all__ = [
+    "FailTaskError",
+    "delay",
+    "fn_params",
+    "fn_signature_params",
+    "fn_path",
+    "fn_code",
+    "fn_info",
+    "fn_from_str",
+    "module_from_str",
+    "pool"
+]

ygo/exceptions.py

@@ -7,7 +7,23 @@ Created on 2024/12/18 下午7:01
 ---------------------------------------------
 """
 
+from dataclasses import dataclass
+
 class WarnException(Exception):
     """自定义异常类，仅用于警告"""
     def __init__(self, message):
-        super().__init__(message)  # 调用父类的构造函数
+        super().__init__(message)  # 调用父类的构造函数
+
+@dataclass
+class FailTaskError:
+    task_name: str
+    error: Exception
+
+    def __str__(self):
+        return f"""
+[失败任务]: {self.task_name}
+[错误信息]: \n{self.error}
+"""
+
+    def __repr__(self):
+        return self.__str__()

ygo/ygo.py

@@ -18,7 +18,7 @@ from pathlib import Path
 from joblib import Parallel, delayed
 
 import ylog
-from .exceptions import WarnException
+from .exceptions import WarnException, FailTaskError
 
 with warnings.catch_warnings():
     warnings.simplefilter("ignore")
@@ -228,22 +228,10 @@ def run_job(job, task_id, queue):
     try:
         result = job()
     except WarnException as e:
-        warn_msg = f"""
-=============================================================
-{job.task_name}: {job.task_id}
-{e}
-=============================================================
-                    """
-        ylog.warning(warn_msg)
+        ylog.warning(FailTaskError(task_name=job.task_name, error=e))
         result = None
     except Exception as e:
-        error_msg = f"""
-=============================================================
-{job.task_name}: {job.task_id}
-{e}
-=============================================================
-                    """
-        ylog.error(error_msg)
+        ylog.error(FailTaskError(task_name=job.task_name, error=e))
         result = None
     queue.put((task_id, 1))
     return result

{ygo-1.0.8.dist-info → ygo-1.0.10.dist-info}/METADATA

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.8
+Version: 1.0.10
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: clickhouse-df>=0.1.5
 Requires-Dist: clickhouse-driver>=0.2.9
+Requires-Dist: connectorx>=0.3.3
 Requires-Dist: dynaconf>=3.2.11
 Requires-Dist: exchange-calendars>=4.2.8
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: lark>=1.2.2
+Requires-Dist: lightgbm>=4.6.0
 Requires-Dist: loguru>=0.7.3
+Requires-Dist: mlflow>=2.17.2
 Requires-Dist: pandas>=2.0.3
 Requires-Dist: polars>=1.8.2
 Requires-Dist: pyarrow>=17.0.0
@@ -19,6 +23,7 @@ Requires-Dist: pymysql>=1.1.1
 Requires-Dist: sqlalchemy>=2.0.40
 Requires-Dist: sqlparse>=0.5.3
 Requires-Dist: toolz>=1.0.0
+Requires-Dist: torch>=2.5.1
 Requires-Dist: tqdm>=4.67.1
 Dynamic: license-file
 

ygo-1.0.10.dist-info/RECORD

@@ -0,0 +1,24 @@
+ycat/__init__.py,sha256=8cVsQXL476hDIU_yDxfoAC9ILC4_Tnp1Y1dDTs6vkXA,529
+ycat/client.py,sha256=Z5lwybzXv6MADbbooDT-rhdr9JMI_t2TDAXt0ghongQ,5451
+ycat/parse.py,sha256=piz_iciF7JFnn8v_qtUEHP6NZ_MWJidsA5gBpLtuZzw,2195
+ycat/updator.py,sha256=dV2vhYVlaoK8eRQggAkNkZuu9zbOYFajicyggMt-r78,3393
+ycat/qdf/__init__.py,sha256=8fIBr0FUAYGc33CYnWILY0Ur2DXdctWjw28S5qDWhD4,7572
+ycat/qdf/errors.py,sha256=lJhhjDRdQOOKUFGlLQ9ELK4AexXBwYQSYus_V-kc5K8,1180
+ycat/qdf/expr.py,sha256=ck_BHMCV29Q8-szci1_v4ud964QI7JoRRcmA0ppupsc,8454
+ycat/qdf/qdf.py,sha256=XcnGyyfuRY1HqaG56kC5tB6psrIXqo9QVQtgH3mhips,7322
+ycat/qdf/udf/__init__.py,sha256=DdrSGaCB__5C1YL0vd_5rjIB3KLrAKn3h3k9k50L0jA,313
+ycat/qdf/udf/base_udf.py,sha256=6VDaCIGNLJxZ7UsoIDWtTH6PzUDj89b8FiwN-TEat2g,3437
+ycat/qdf/udf/cs_udf.py,sha256=HT3EKBwAhOxOFDQnpfwb4YcMTT3-lqFXkdysdn5_FI4,3179
+ycat/qdf/udf/d_udf.py,sha256=L9mkX6yDpQPwXvQTAebDepjEkirCqur1DfV2Fnl8KA0,5352
+ycat/qdf/udf/ind_udf.py,sha256=hDCKfcLFCgIhdC9dQ5GYxLemZaOE6K7kQyAnjUrwePM,6482
+ycat/qdf/udf/ts_udf.py,sha256=uUuZnKMY-V_uInP0nsBMblDpxY3ld3EwvrXTwWMqeig,5410
+ygo/__init__.py,sha256=kQK7CwVCz8NJTj5eS9Xrt_G1kPHvDIbe2sTzHgWITxI,590
+ygo/exceptions.py,sha256=0OYDYt_9KKo8mF2XBG5QkCMr3-ASp69VDSPOEwlIsrI,660
+ygo/ygo.py,sha256=kcXI5vzndNOJqEEEZOeWbn61O47gW72UDiUWN1v9AYc,11290
+ygo-1.0.10.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
+ylog/__init__.py,sha256=AoRCQ-o4gWAcJ8svw30wM5UJyccx45WhYIndrrkNv8o,428
+ylog/core.py,sha256=d6QCFRDTvlyxgvS6JphUGOgX5Mgx9qPv9wB3g-4YOJw,9225
+ygo-1.0.10.dist-info/METADATA,sha256=J0M_naeBSfTGAcep-85CtjOz-Tz0z2l8h_VKd7RLIq4,2235
+ygo-1.0.10.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+ygo-1.0.10.dist-info/top_level.txt,sha256=jEbfiz5fX4iSzDg8_Npdv5SIC_Kphmb1m3vuyD9ZC1E,14
+ygo-1.0.10.dist-info/RECORD,,

{ygo-1.0.8.dist-info → ygo-1.0.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{ygo-1.0.8.dist-info → ygo-1.0.10.dist-info}/top_level.txt

@@ -1,4 +1,3 @@
-qdf
 ycat
 ygo
 ylog

ylog/__init__.py

@@ -7,7 +7,7 @@ Created on 2025/5/14 15:37
 ---------------------------------------------
 """
 
-from .core import trace, debug, info, warning, error, critical, update_config
+from .core import trace, debug, info, warning, error, critical, update_config, get_logger
 
 __all__ = [
     "trace",
@@ -16,5 +16,6 @@ __all__ = [
     "warning",
     "error",
     "critical",
-    "update_config"
+    "update_config",
+    "get_logger",
 ]

ylog/core.py

@@ -27,21 +27,14 @@ class _Logger:
     5. 全局异常捕获：自动记录未处理异常
     """
 
-    _instance = None
-    _lock = Lock()
-
     def __new__(cls, *args, **kwargs):
-        """单例模式确保全局只有一个logger实例"""
-        if not cls._instance:
-            with cls._lock:
-                if not cls._instance:
-                    cls._instance = super().__new__(cls)
-        return cls._instance
+        """移除单例限制，允许自由创建多个实例"""
+        return super().__new__(cls)
 
     def __init__(
             self,
             log_dir: str = "logs",
-            app_name: str = "application",
+            app_name: str|None = None,
             retention_days: int = 7,
             error_retention_days: int = 30,
             enable_console: bool = True,
@@ -111,27 +104,48 @@ class _Logger:
             "TRACE": {"level": "TRACE", "retention": f"{retention_days} days", "rotation": "daily"},
             "DEBUG": {"level": "DEBUG", "retention": f"{retention_days} days", "rotation": "daily"},
             "INFO": {"level": "INFO", "retention": f"{retention_days} days", "rotation": "daily"},
-            "WARNING": {"level": "WARNING", "retention": f"{retention_days} days", "rotation": "daily"},
+            "WARNING": {"level": "WARNING", "retention": f"{error_retention_days} days", "rotation": "daily"},
             "ERROR": {"level": "ERROR", "retention": f"{error_retention_days} days", "rotation": "daily"},
             "CRITICAL": {"level": "CRITICAL", "retention": f"{error_retention_days} days", "rotation": "daily"}
         }
 
-        for level, config in levels.items():
-            log_file = self.log_dir / f"{self.app_name}_{level.lower()}_{{time:YYYY-MM-DD}}.log"
-
-            logger.add(
-                str(log_file),
-                level=config["level"],
-                format=common_format,
-                rotation=config["rotation"],
-                retention=config["retention"],
-                compression="zip",
-                backtrace=True,
-                diagnose=self.debug_mode,
-                # enqueue=True,  # 异步写入
-                filter=lambda record, lvl=level: record["level"].name == lvl,
-                catch=True  # 捕获格式化异常
-            )
+        # 错误级别以上的日志
+        error_levels = ("WARNING", "ERROR", "CRITICAL")
+        _error_log_file = f"{self.app_name}_{{time:YYYY-MM-DD}}.err.log" if self.app_name is not None else f"{{time:YYYY-MM-DD}}.err.log"
+        error_log_file = self.log_dir / _error_log_file
+        info_levels = ("INFO", "DEBUG", "TRACE")
+        _info_log_file = f"{self.app_name}_{{time:YYYY-MM-DD}}.log" if self.app_name is not None else f"{{time:YYYY-MM-DD}}.log"
+        info_log_file = self.log_dir / _info_log_file
+        # 错误级别以上的日志
+        logger.add(
+            str(error_log_file),
+            level="ERROR",
+            format=common_format,
+            rotation=levels["ERROR"]["rotation"],
+            retention=levels["ERROR"]["retention"],
+            compression="zip",
+            backtrace=True,
+            diagnose=self.debug_mode,
+            # enqueue=True,  # 异步写入
+            filter=lambda record: record["level"].name in error_levels,
+            catch=True  # 捕获格式化异常
+        )
+        # 错误级别以下的日志
+        logger.add(
+            str(info_log_file),
+            level="INFO",
+            format=common_format,
+            rotation=levels["INFO"]["rotation"],
+            retention=levels["INFO"]["retention"],
+            compression="zip",
+            backtrace=True,
+            diagnose=self.debug_mode,
+            # enqueue=True,  # 异步写入
+            filter=lambda record: record["level"].name in info_levels,
+            catch=True  # 捕获格式化异常
+        )
+
+
 
     def _setup_global_exception_handling(self):
         """配置全局异常捕获"""
@@ -217,7 +231,7 @@ class _Logger:
             self._setup_file_logging(self._retention_days, self._error_retention_days)
 
 # 初始化默认实例
-_default_logger = _Logger(app_name="app")
+_default_logger = _Logger()
 
 # 将日志方法绑定到模块级别
 trace = _default_logger.trace
@@ -227,3 +241,22 @@ warning = _default_logger.warning
 error = _default_logger.error
 critical = _default_logger.critical
 update_config = _default_logger.update_config
+
+def get_logger(app_name: str,
+               log_dir: str = "logs",
+               retention_days: int = 7,
+               error_retention_days: int = 30,
+               enable_console: bool = True,
+               enable_file: bool = True,
+               debug_mode: bool = False):
+    """获取指定应用的日志实例"""
+    return _Logger(
+        app_name=app_name,
+        log_dir=log_dir,
+        retention_days=retention_days,
+        error_retention_days=error_retention_days,
+        enable_console=enable_console,
+        enable_file=enable_file,
+        debug_mode=debug_mode
+    )
+

ycat/dtype.py

@@ -1,389 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2024/11/4 下午1:20
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-import functools
-import re
-from typing import Any
-import pyarrow as pa
-import re  # 正则解析 Decimal 类型
-
-from polars._typing import PolarsDataType
-from polars.datatypes import (
-    Binary,
-    Boolean,
-    Date,
-    Datetime,
-    Decimal,
-    Duration,
-    Float32,
-    Float64,
-    Int8,
-    Int16,
-    Int32,
-    Int64,
-    List,
-    Null,
-    String,
-    Time,
-    UInt8,
-    UInt16,
-    UInt32,
-    UInt64,
-)
-
-
-@functools.lru_cache(8)
-def integer_dtype_from_nbits(
-        bits: int,
-        *,
-        unsigned: bool,
-        default: PolarsDataType | None = None,
-) -> PolarsDataType | None:
-    """
-    Return matching Polars integer dtype from num bits and signed/unsigned flag.
-
-    Examples
-    --------
-    >>> integer_dtype_from_nbits(8, unsigned=False)
-    Int8
-    >>> integer_dtype_from_nbits(32, unsigned=True)
-    UInt32
-    """
-    dtype = {
-        (8, False): Int8,
-        (8, True): UInt8,
-        (16, False): Int16,
-        (16, True): UInt16,
-        (32, False): Int32,
-        (32, True): UInt32,
-        (64, False): Int64,
-        (64, True): UInt64,
-    }.get((bits, unsigned), None)
-
-    if dtype is None and default is not None:
-        return default
-    return dtype
-
-
-def timeunit_from_precision(precision: int | str | None) -> str | None:
-    """
-    Return `time_unit` from integer precision value.
-
-    Examples
-    --------
-    >>> timeunit_from_precision(3)
-    'ms'
-    >>> timeunit_from_precision(5)
-    'us'
-    >>> timeunit_from_precision(7)
-    'ns'
-    """
-    from math import ceil
-
-    if not precision:
-        return None
-    elif isinstance(precision, str):
-        if precision.isdigit():
-            precision = int(precision)
-        elif (precision := precision.lower()) in ("s", "ms", "us", "ns"):
-            return "ms" if precision == "s" else precision
-    try:
-        n = min(max(3, int(ceil(precision / 3)) * 3), 9)  # type: ignore[operator]
-        return {3: "ms", 6: "us", 9: "ns"}.get(n)
-    except TypeError:
-        return None
-
-
-def infer_dtype_from_database_typename(
-        value: str,
-        *,
-        raise_unmatched: bool = True,
-) -> PolarsDataType | None:
-    """
-    Attempt to infer Polars dtype from database cursor `type_code` string value.
-
-    Examples
-    --------
-    >>> infer_dtype_from_database_typename("INT2")
-    Int16
-    >>> infer_dtype_from_database_typename("NVARCHAR")
-    String
-    >>> infer_dtype_from_database_typename("NUMERIC(10,2)")
-    Decimal(precision=10, scale=2)
-    >>> infer_dtype_from_database_typename("TIMESTAMP WITHOUT TZ")
-    Datetime(time_unit='us', time_zone=None)
-    """
-    dtype: PolarsDataType | None = None
-
-    # normalise string name/case (eg: 'IntegerType' -> 'INTEGER')
-    original_value = value
-    value = value.upper().replace("TYPE", "")
-
-    # extract optional type modifier (eg: 'VARCHAR(64)' -> '64')
-    if re.search(r"\([\w,: ]+\)$", value):
-        modifier = value[value.find("(") + 1: -1]
-        value = value.split("(")[0]
-        # Nullable type
-        if value.upper() == "NULLABLE":
-            return infer_dtype_from_database_typename(modifier)
-    elif (
-            not value.startswith(("<", ">")) and re.search(r"\[[\w,\]\[: ]+]$", value)
-    ) or value.endswith(("[S]", "[MS]", "[US]", "[NS]")):
-        modifier = value[value.find("[") + 1: -1]
-        value = value.split("[")[0]
-    else:
-        modifier = ""
-
-    # array dtypes
-    array_aliases = ("ARRAY", "LIST", "[]")
-    if value.endswith(array_aliases) or value.startswith(array_aliases):
-        for a in array_aliases:
-            value = value.replace(a, "", 1) if value else ""
-
-        nested: PolarsDataType | None = None
-        if not value and modifier:
-            nested = infer_dtype_from_database_typename(
-                value=modifier,
-                raise_unmatched=False,
-            )
-        else:
-            if inner_value := infer_dtype_from_database_typename(
-                    value[1:-1]
-                    if (value[0], value[-1]) == ("<", ">")
-                    else re.sub(r"\W", "", re.sub(r"\WOF\W", "", value)),
-                    raise_unmatched=False,
-            ):
-                nested = inner_value
-            elif modifier:
-                nested = infer_dtype_from_database_typename(
-                    value=modifier,
-                    raise_unmatched=False,
-                )
-        if nested:
-            dtype = List(nested)
-
-    # float dtypes
-    elif value.startswith("FLOAT") or ("DOUBLE" in value) or (value == "REAL"):
-        dtype = (
-            Float32
-            if value == "FLOAT4"
-               or (value.endswith(("16", "32")) or (modifier in ("16", "32")))
-            else Float64
-        )
-
-    # integer dtypes
-    elif ("INTERVAL" not in value) and (
-            value.startswith(("INT", "UINT", "UNSIGNED"))
-            or value.endswith(("INT", "SERIAL"))
-            or ("INTEGER" in value)
-            or value == "ROWID"
-    ):
-        sz: Any
-        if "LARGE" in value or value.startswith("BIG") or value == "INT8":
-            sz = 64
-        elif "MEDIUM" in value or value in ("INT4", "SERIAL"):
-            sz = 32
-        elif "SMALL" in value or value == "INT2":
-            sz = 16
-        elif "TINY" in value:
-            sz = 8
-        else:
-            sz = None
-
-        sz = modifier if (not sz and modifier) else sz
-        if not isinstance(sz, int):
-            sz = int(sz) if isinstance(sz, str) and sz.isdigit() else None
-        if (
-                ("U" in value and "MEDIUM" not in value)
-                or ("UNSIGNED" in value)
-                or value == "ROWID"
-        ):
-            dtype = integer_dtype_from_nbits(sz, unsigned=True, default=UInt64)
-        else:
-            dtype = integer_dtype_from_nbits(sz, unsigned=False, default=Int64)
-
-    # number types (note: 'number' alone is not that helpful and requires refinement)
-    elif "NUMBER" in value and "CARDINAL" in value:
-        dtype = UInt64
-
-    # decimal dtypes
-    elif (is_dec := ("DECIMAL" in value)) or ("NUMERIC" in value):
-        if "," in modifier:
-            prec, scale = modifier.split(",")
-            dtype = Decimal(int(prec), int(scale))
-        else:
-            dtype = Decimal if is_dec else Float64
-
-    # string dtypes
-    elif (
-            any(tp in value for tp in ("VARCHAR", "STRING", "TEXT", "UNICODE"))
-            or value.startswith(("STR", "CHAR", "BPCHAR", "NCHAR", "UTF"))
-            or value.endswith(("_UTF8", "_UTF16", "_UTF32"))
-    ):
-        dtype = String
-
-    # binary dtypes
-    elif value in ("BYTEA", "BYTES", "BLOB", "CLOB", "BINARY"):
-        dtype = Binary
-
-    # boolean dtypes
-    elif value.startswith("BOOL"):
-        dtype = Boolean
-
-    # null dtype; odd, but valid
-    elif value == "NULL":
-        dtype = Null
-
-    # temporal dtypes
-    elif value.startswith(("DATETIME", "TIMESTAMP")) and not (value.endswith("[D]")):
-        if any((tz in value.replace(" ", "")) for tz in ("TZ", "TIMEZONE")):
-            if "WITHOUT" not in value:
-                return None  # there's a timezone, but we don't know what it is
-        unit = timeunit_from_precision(modifier) if modifier else "us"
-        dtype = Datetime(time_unit=(unit or "us"))  # type: ignore[arg-type]
-    else:
-        value = re.sub(r"\d", "", value)
-        if value in ("INTERVAL", "TIMEDELTA", "DURATION"):
-            dtype = Duration
-        elif value == "DATE":
-            dtype = Date
-        elif value == "TIME":
-            dtype = Time
-
-    if not dtype and raise_unmatched:
-        msg = f"cannot infer dtype from {original_value!r} string value"
-        raise ValueError(msg)
-
-    return dtype
-
-CLICKHOUSE_TO_ARROW_TYPE = {
-    # 整数类型
-    'Int8': pa.int8(),
-    'Int16': pa.int16(),
-    'Int32': pa.int32(),
-    'Int64': pa.int64(),
-    'UInt8': pa.uint8(),
-    'UInt16': pa.uint16(),
-    'UInt32': pa.uint32(),
-    'UInt64': pa.uint64(),
-
-    # 浮点类型
-    'Float32': pa.float32(),
-    'Float64': pa.float64(),
-
-    # 字符串类型
-    'String': pa.string(),
-    'FixedString': pa.string(),  # Arrow 不区分固定长度和动态长度字符串
-
-    # 日期和时间类型
-    'Date': pa.date32(),  # ClickHouse 的 Date 是 32 位（天）
-    'Date32': pa.date32(),
-    'DateTime': pa.timestamp('s'),  # ClickHouse DateTime 精度为秒
-    'DateTime64': pa.timestamp('ms'),  # 默认映射为毫秒精度（可根据需求调整）
-    'UUID': pa.binary(16),  # UUID 是 16 字节的二进制
-
-    # 布尔类型
-    'Boolean': pa.bool_(),
-
-    # 数组类型（嵌套类型）
-    'Array(Int8)': pa.list_(pa.int8()),
-    'Array(Int16)': pa.list_(pa.int16()),
-    'Array(Int32)': pa.list_(pa.int32()),
-    'Array(Int64)': pa.list_(pa.int64()),
-    'Array(UInt8)': pa.list_(pa.uint8()),
-    'Array(UInt16)': pa.list_(pa.uint16()),
-    'Array(UInt32)': pa.list_(pa.uint32()),
-    'Array(UInt64)': pa.list_(pa.uint64()),
-    'Array(Float32)': pa.list_(pa.float32()),
-    'Array(Float64)': pa.list_(pa.float64()),
-    'Array(String)': pa.list_(pa.string()),
-    'Array(Date)': pa.list_(pa.date32()),
-    'Array(DateTime)': pa.list_(pa.timestamp('s')),
-
-    # 嵌套类型（元组、枚举等）
-    # 注意：Arrow 不直接支持 Tuple，通常需要转换为 Struct
-    'Tuple': pa.struct([]),  # 需要动态定义每个字段的类型
-    # 枚举类型
-    'Enum8': pa.string(),  # 通常映射为字符串
-    'Enum16': pa.string(),
-
-    # Map 类型
-    'Map': pa.map_(pa.string(), pa.string()),  # 默认键值对是字符串（可根据需求调整）
-
-    # Nullable 类型（ClickHouse 的 Nullable 包装类型）
-    'Nullable(Int8)': pa.int8(),
-    'Nullable(Int16)': pa.int16(),
-    'Nullable(Int32)': pa.int32(),
-    'Nullable(Int64)': pa.int64(),
-    'Nullable(UInt8)': pa.uint8(),
-    'Nullable(UInt16)': pa.uint16(),
-    'Nullable(UInt32)': pa.uint32(),
-    'Nullable(UInt64)': pa.uint64(),
-    'Nullable(Float32)': pa.float32(),
-    'Nullable(Float64)': pa.float64(),
-    'Nullable(String)': pa.string(),
-    'Nullable(Date)': pa.date32(),
-    'Nullable(DateTime)': pa.timestamp('s'),
-    'Nullable(UUID)': pa.binary(16),
-}
-
-def map_clickhouse_decimal(ch_type: str) -> pa.DataType:
-    """
-    映射 ClickHouse 的 Decimal 类型到 Arrow 的 Decimal 类型
-    :param ch_type: ClickHouse 的 Decimal 类型描述，例如 'Decimal(10, 2)' 或 'Decimal128(38)'
-    :return: 对应的 Arrow Decimal 类型
-    """
-    # 匹配 ClickHouse 的 Decimal(p, s) 格式
-    decimal_match = re.match(r"Decimal(?:32|64|128)?\((\d+),\s*(\d+)\)", ch_type)
-    if decimal_match:
-        precision, scale = map(int, decimal_match.groups())
-        return pa.decimal128(precision, scale)
-
-    # 匹配 ClickHouse 的 Decimal(p) 格式，默认 scale 为 0
-    decimal_match_no_scale = re.match(r"Decimal(?:32|64|128)?\((\d+)\)", ch_type)
-    if decimal_match_no_scale:
-        precision = int(decimal_match_no_scale.group(1))
-        return pa.decimal128(precision, 0)
-
-    # 如果不匹配，抛出异常
-    raise ValueError(f"Unsupported ClickHouse Decimal type: {ch_type}")
-
-def map_clickhouse_to_arrow(ch_type: str) -> pa.DataType:
-    """
-    动态映射 ClickHouse 类型到 Arrow 类型
-    """
-    # 基础类型直接映射
-    if ch_type in CLICKHOUSE_TO_ARROW_TYPE:
-        return CLICKHOUSE_TO_ARROW_TYPE[ch_type]
-
-    # Decimal 类型处理
-    if ch_type.startswith("Decimal"):
-        return map_clickhouse_decimal(ch_type)
-
-    # 动态处理 Array 类型
-    if ch_type.startswith('Array('):
-        inner_type = ch_type[6:-1]  # 提取 Array 内的类型
-        return pa.list_(map_clickhouse_to_arrow(inner_type))
-
-    # 动态处理 Nullable 类型
-    if ch_type.startswith('Nullable('):
-        inner_type = ch_type[9:-1]  # 提取 Nullable 内的类型
-        return map_clickhouse_to_arrow(inner_type)
-
-    # 动态处理 Tuple 类型
-    if ch_type.startswith('Tuple('):
-        inner_types = ch_type[6:-1].split(',')  # 提取 Tuple 内的字段类型
-        return pa.struct([('field' + str(i), map_clickhouse_to_arrow(t.strip())) for i, t in enumerate(inner_types)])
-
-    # 动态处理 Map 类型
-    if ch_type.startswith('Map('):
-        key_type, value_type = ch_type[4:-1].split(',')
-        return pa.map_(map_clickhouse_to_arrow(key_type.strip()), map_clickhouse_to_arrow(value_type.strip()))
-
-    raise ValueError(f"Unsupported ClickHouse type: {ch_type}")
-

ycat/yck.py

@@ -1,87 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2024/11/4 上午9:01
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-from random import randint
-
-import pandas as pd
-import polars
-import polars as pl
-import pyarrow as pa
-from clickhouse_driver import Client
-
-from . import dtype
-
-
-def connect(urls: list[str], user: str, password: str) -> Client:
-    """
-    连接clickhouse服务器, 支持集群
-    Parameters
-    ----------
-    urls: List[str]
-        ["host1:port1", "host2:port2", "host3:port3"...]
-    user: str
-        用户名
-    password: str
-        密码
-    Returns
-    -------
-    client: Client
-        ClickHouse 数据库连接客户端，必须是一个有效的 `clickhouse_driver.Client` 实例
-    """
-    i = randint(0, len(urls) - 1)
-    url_ini = urls[i]
-    [host, port] = url_ini.split(":")
-    return Client(host, port=port, round_robin=True, alt_hosts=",".join(urls), user=user, password=password)
-
-
-def query_pandas(sql, conn) -> pd.DataFrame:
-    """
-    请求ck，返回 pandas.DataFrame
-    Parameters
-    ----------
-    sql: str
-        查询语句
-    conn: Client
-        ClickHouse 数据库连接客户端，必须是一个有效的 `clickhouse_driver.Client` 实例
-    Returns
-    -------
-    pandas.DataFrame
-        包含查询结果的 Pandas DataFrame。如果查询没有返回任何数据，则
-        返回一个空的 DataFrame 或者 None
-    """
-    return conn.query_dataframe(sql)
-
-
-def query_polars(sql, conn) -> pl.DataFrame:
-    """
-    请求ck，返回 polars.DataFrame
-    Parameters
-    ----------
-    sql: str
-        查询语句
-    conn: Client
-        ClickHouse 数据库连接客户端，必须是一个有效的 `clickhouse_driver.Client` 实例。
-    Returns
-    -------
-    polars.DataFrame
-        包含查询结果的 Polars DataFrame。如果查询没有返回任何数据，则
-        返回一个空的 DataFrame 或者 None
-    """
-    data, columns = conn.execute(sql, columnar=True, with_column_types=True)
-    # columns = {name: dtype.infer_dtype_from_database_typename(type_) for name, type_ in columns}
-    if len(data) < 1:
-        columns = {name: dtype.infer_dtype_from_database_typename(type_) for name, type_ in columns}
-        return pl.DataFrame(schema=columns)
-    columns = {name: dtype.map_clickhouse_to_arrow(type_) for name, type_ in columns}
-    # 构造 Arrow 表（逐列传递数据和类型）
-    arrow_table = pa.Table.from_arrays(
-        [pa.array(col, type=col_type) for col, col_type in zip(data, columns.values())],
-        schema=pa.schema(columns))
-
-    # 从 Arrow 表构造 Polars DataFrame
-    return pl.from_arrow(arrow_table)

ygo-1.0.8.dist-info/RECORD

@@ -1,25 +0,0 @@
-qdf/__init__.py,sha256=XwH17ae6kX5Grhb_odgDqkNf6N-ambTVi3LoGzq1Fmc,7519
-qdf/errors.py,sha256=lJhhjDRdQOOKUFGlLQ9ELK4AexXBwYQSYus_V-kc5K8,1180
-qdf/expr.py,sha256=ck_BHMCV29Q8-szci1_v4ud964QI7JoRRcmA0ppupsc,8454
-qdf/qdf.py,sha256=X3n2eB9C2Jm7dP4d_j-P9QvL5DJ9X1fe7GZfn3afxeU,7411
-qdf/udf/__init__.py,sha256=DdrSGaCB__5C1YL0vd_5rjIB3KLrAKn3h3k9k50L0jA,313
-qdf/udf/base_udf.py,sha256=6VDaCIGNLJxZ7UsoIDWtTH6PzUDj89b8FiwN-TEat2g,3437
-qdf/udf/cs_udf.py,sha256=HT3EKBwAhOxOFDQnpfwb4YcMTT3-lqFXkdysdn5_FI4,3179
-qdf/udf/d_udf.py,sha256=L9mkX6yDpQPwXvQTAebDepjEkirCqur1DfV2Fnl8KA0,5352
-qdf/udf/ind_udf.py,sha256=hDCKfcLFCgIhdC9dQ5GYxLemZaOE6K7kQyAnjUrwePM,6482
-qdf/udf/ts_udf.py,sha256=uUuZnKMY-V_uInP0nsBMblDpxY3ld3EwvrXTwWMqeig,5410
-ycat/__init__.py,sha256=zBMOFStzKSt_5jw4af6YFtPD5Svr8fJlZkP_AdUjCoA,554
-ycat/client.py,sha256=0zD9BP1TH0f0D_N7sgQ8bgYVKb9hZZRxDU0dkun-G9w,3933
-ycat/dtype.py,sha256=mRGLDe_Ho6-tDsoj5wwrAzozEoIYCAHGKdpRqgBfUcI,12577
-ycat/parse.py,sha256=9Kgr33nHYC96TGpZs98PAu0cbK-FrR0pfuf8lfD647I,2289
-ycat/yck.py,sha256=FlGMBuKEngB4TwFXMp4P3dLg9IfFmUg3eDqXzQ0kQoI,2738
-ygo/__init__.py,sha256=FMN06Tfa8_oV26eklBZCtGTyHZ6MghHxHj4PS_FSXCA,222
-ygo/exceptions.py,sha256=4Kd92kpwpsXHJJkSv4OqcN--PEEvIGGvDDgOOsk68gg,385
-ygo/ygo.py,sha256=vCMUur_41yY0QB4gj8K5wBZHql_cbmANhI8QwPRCTmo,11613
-ygo-1.0.8.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
-ylog/__init__.py,sha256=2sIp4PHNoQMCi0QtIarTI4raACd7SdRHNY7fY5hKYwc,397
-ylog/core.py,sha256=jmz9JhklbVCQz-zahEXV6P-LEHnqU6opnY4CUEyo8Ss,7924
-ygo-1.0.8.dist-info/METADATA,sha256=21pVOKJzFEmudbQyApruzgFa0O2PymVE5c5m06fNrzM,2076
-ygo-1.0.8.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-ygo-1.0.8.dist-info/top_level.txt,sha256=FGbsOtsHgqWzZ9mGRSTCg0pLZEErR1lq5TFQSy2TL1w,18
-ygo-1.0.8.dist-info/RECORD,,