ygo 1.0.11__py3-none-any.whl → 1.1.0__py3-none-any.whl

ygo-1.1.0.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: ygo
+Version: 1.1.0
+Project-URL: homepage, https://github.com/link-yundi/ygo
+Project-URL: repository, https://github.com/link-yundi/ygo
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: joblib>=1.5.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: tqdm>=4.67.1
+Dynamic: license-file
+
+# ygo
+一个轻量级 Python 工具包，底层基于 joblib 和 tqdm 、loguru 实现，支持
+- 并发执行（带进度条）
+- 延迟调用
+- 链式绑定参数
+- 函数信息获取
+- 模块/函数动态加载...
+- 并结合 ylog 提供日志记录能力
+
+### 安装
+```shell
+pip install -U ygo
+```
+
+### 🧰 功能概览
+
+| 模块   | 功能                                                         |
+| :----- | :----------------------------------------------------------- |
+| `ygo`  | 支持并发执行（带进度条）、延迟调用、函数信息获取以及模块/函数动态加载等功能 |
+| `ylog` | 日志模块，提供统一的日志输出接口                             |
+
+### 示例
+
+```
+├── a
+│   ├── __init__.py
+│   └── b
+│       ├── __init__.py
+│       └── c.py
+└── test.py
+
+c.py 中定义了目标函数
+def test_fn(a, b=2):
+    return a+b
+```
+
+#### 场景1: 并发执行
+
+```python
+import ygo
+import ylog
+from a.b.c import test_fn
+
+with ygo.pool(job_name="test parallel", show_progress=True) as go:
+    for i in range(10):
+        go.submit(test_fn)(a=i, b=2*i)
+    for res in go.do():
+        ylog.info(res)
+```
+
+#### ✅ `ygo.pool` 支持的参数
+
+| 参数名        | 类型 | 描述                                                         |
+| ------------- | ---- | ------------------------------------------------------------ |
+| n_jobs        | int  | 并行任务数(<=1 表示串行)                                     |
+| show_progress | bool | 是否显示进度条                                               |
+| backend       | str  | 执行后端（默认 'threading'，可选 'multiprocessing' 或 'loky'） |
+
+#### 场景2: 延迟调用
+
+```
+>>> fn = delay(test_fn)(a=1, b=2)
+>>> fn()
+3
+>>> # 逐步传递参数
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1)
+>>> fn2 = delay(fn1)(b=2)
+>>> fn2(c=3)
+6
+>>> # 参数更改
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1, b=2)
+>>> fn2 = delay(fn1)(c=3, b=5)
+>>> fn2()
+9
+```
+
+#### 场景3: 获取目标函数信息
+
+```
+>>> ygo.fn_info(test_fn)
+=============================================================
+    a.b.c.test_fn(a, b=2)
+=============================================================
+    def test_fn(a, b=2):
+    return a+b
+```
+
+#### 🔍 其他函数信息工具
+
+| 方法名                    | 描述                                     |
+| ------------------------- | ---------------------------------------- |
+| `fn_params(fn)`           | 获取函数实参                             |
+| `fn_signature_params(fn)` | 获取函数定义的所有参数名                 |
+| `fn_code(fn)`             | 获取函数源码字符串                       |
+| `fn_path(fn)`             | 获取函数所属模块路径                     |
+| `fn_from_str(s)`          | 根据字符串导入函数（如 "a.b.c.test_fn"） |
+| `module_from_str(s)`      | 根据字符串导入模块                       |
+
+#### 场景4: 通过字符串解析函数并执行
+
+```
+>>> ygo.fn_from_str("a.b.c.test_fn")(a=1, b=5)
+6
+```
+
+### 📝 日志记录（ylog）
+
+```python
+import ylog
+
+ylog.info("这是一个信息日志")
+ylog.warning("这是一个警告日志")
+ylog.error("这是一个错误日志", exc_info=True)
+
+
+
+# 为不同的模块使用不同的logger
+logger_app1 = ylog.get_logger("app1", )
+logger_app2 = ylog.get_logger("app2", )
+```
+
+#### 🔧 配置管理：`update_config`
+
+你可以通过 update_config 方法动态修改日志配置，例如设置日志级别、格式、是否启用颜色等。
+
+```python
+# 开启调试模式
+ylog.update_config(debug_mode=True)
+```
+
+#### 🧩 获取独立的 Logger 实例：`get_logger`
+
+在大型项目中，你可能希望为不同模块或组件创建独立的 logger 实例以区分日志来源。
+
+```python
+logger1 = ylog.get_logger("moduleA")
+logger2 = ylog.get_logger("moduleB")
+
+logger1.info("这是来自 moduleA 的日志")
+logger2.warning("这是来自 moduleB 的警告")
+```
+
+#### 📌 使用建议
+
+- 生产环境建议关闭 `debug_mode`，避免产生过多调试日志。
+- 对于复杂项目，推荐使用 `get_logger` 创建命名 logger，便于日志分类与分析。
+- 使用 `exc_info=True` 参数时，可自动打印异常堆栈信息，适用于错误捕获场景。

ygo-1.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+ygo/__init__.py,sha256=AlvzcS4Ge94nklq5AhPhCNCIg5D1F8gaZqhdzQpoXH4,679
+ygo/delay.py,sha256=66xtPXqyD630FL7LWL5qJKAIZvyGDwZyM4qPfk8Czlg,2206
+ygo/exceptions.py,sha256=0OYDYt_9KKo8mF2XBG5QkCMr3-ASp69VDSPOEwlIsrI,660
+ygo/pool.py,sha256=bnHm4TtnRoFBv5UvV7WpuObJoK4FdoRf65mvf82yEyI,7052
+ygo/utils.py,sha256=c-g4fJgeZp8diinkJhX4DAJBZEhH2tHYniUzRlt1EgU,3178
+ygo-1.1.0.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
+ylog/__init__.py,sha256=aNrUp1n3JJFMUt1JFEsq33bckIhSQwCiTQCmV9rOMYk,452
+ylog/core.py,sha256=d6QCFRDTvlyxgvS6JphUGOgX5Mgx9qPv9wB3g-4YOJw,9225
+ygo-1.1.0.dist-info/METADATA,sha256=Eelk_nhMZZ6nwqCtTtmLJabPuLB3d0PacwbS3rP8uDE,4651
+ygo-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygo-1.1.0.dist-info/top_level.txt,sha256=sY7lJBJ2ncfEMAxoNBVay0RVUixpVt9Osuwwy0_uWqU,9
+ygo-1.1.0.dist-info/RECORD,,

{ygo-1.0.11.dist-info → ygo-1.1.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{ygo-1.0.11.dist-info → ygo-1.1.0.dist-info}/top_level.txt

@@ -1,3 +1,2 @@
-ycat
 ygo
 ylog

ylog/__init__.py

@@ -9,6 +9,8 @@ Created on 2025/5/14 15:37
 
 from .core import trace, debug, info, warning, error, critical, update_config, get_logger
 
+__version__ = "v0.0.1"
+
 __all__ = [
     "trace",
     "debug",

ycat/__init__.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2025/5/14 18:29
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-
-from .client import (
-    HOME,
-    CATDB,
-    get_settings,
-    sql,
-    put,
-    tb_path,
-    read_ck,
-    read_mysql,
-)
-from .qdf import from_polars
-from .provider import Provider
-
-__all__ = [
-    "HOME",
-    "CATDB",
-    "get_settings",
-    "sql",
-    "put",
-    "tb_path",
-    "read_ck",
-    "read_mysql",
-    "Provider",
-]

ycat/client.py

@@ -1,172 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2024/7/1 09:44
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-import os
-import re
-import urllib
-from pathlib import Path
-
-import clickhouse_df
-import polars as pl
-from dynaconf import Dynaconf
-
-import ylog
-from .parse import extract_table_names_from_sql
-
-# 配置文件在 “~/.catdb/setting.toml”
-USERHOME = os.path.expanduser('~')  # 用户家目录
-NAME = "catdb"
-CONFIG_PATH = os.path.join(USERHOME, f".{NAME}", "settings.toml")
-if not os.path.exists(CONFIG_PATH):
-    try:
-        os.makedirs(os.path.dirname(CONFIG_PATH))
-    except FileExistsError as e:
-        ...
-    except Exception as e:
-        ylog.error(f"配置文件生成失败: {e}")
-    catdb_path = os.path.join(USERHOME, NAME)
-    template_content = f"""[paths]
-{NAME}="{catdb_path}"  # 本地数据库，默认家目录
-
-## 数据库配置：
-[database]
-[database.ck]
-# urls=["<host1>:<port1>", "<host2>:<port2>",]
-# user="xxx"
-# password="xxxxxx"
-[database.jy]
-# url="<host>:<port>"
-# user="xxxx"
-# password="xxxxxx"
-
-## 视情况自由增加其他配置
-    """
-    with open(CONFIG_PATH, "w") as f:
-        f.write(template_content)
-    ylog.info(f"生成配置文件: {CONFIG_PATH}")
-
-
-def get_settings():
-    try:
-        return Dynaconf(settings_files=[CONFIG_PATH])
-    except Exception as e:
-        ylog.error(f"读取配置文件失败: {e}")
-        return {}
-
-
-HOME = USERHOME
-CATDB = os.path.join(HOME, NAME)
-# 读取配置文件覆盖
-SETTINGS = get_settings()
-if SETTINGS is not None:
-    CATDB = SETTINGS["PATHS"][NAME]
-    if not CATDB.endswith(NAME):
-        CATDB = os.path.join(CATDB, NAME)
-
-
-# ======================== 本地数据库 catdb ========================
-def tb_path(tb_name: str) -> Path:
-    """
-    返回指定表名 完整的本地路径
-    Parameters
-    ----------
-    tb_name: str
-       表名，路径写法: a/b/c
-    Returns
-    -------
-        full_abs_path: pathlib.Path
-        完整的本地绝对路径 $HOME/catdb/a/b/c
-    """
-    return Path(CATDB, tb_name)
-
-
-def put(df: pl.DataFrame, tb_name: str, partitions: list[str] | None = None, abs_path: bool = False):
-    if not abs_path:
-        tbpath = tb_path(tb_name)
-    else:
-        tbpath = tb_name
-    if not tbpath.exists():
-        os.makedirs(tbpath, exist_ok=True)
-    if partitions is not None:
-        df.write_parquet(tbpath, partition_by=partitions)
-    else:
-        df.write_parquet(tbpath / "data.parquet")
-
-
-def sql(query: str, abs_path: bool = False, lazy: bool = True):
-    tbs = extract_table_names_from_sql(query)
-    convertor = dict()
-    for tb in tbs:
-        if not abs_path:
-            db_path = tb_path(tb)
-        else:
-            db_path = tb
-        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
-        convertor[tb] = format_tb
-    pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
-    new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
-    if not lazy:
-        return pl.sql(new_query).collect()
-    return pl.sql(new_query)
-
-
-def read_mysql(query: str, db_conf: str = "database.mysql") -> pl.DataFrame:
-    """
-    读取 mysql 返回 polars.DataFrame
-    :param query:
-    :param db_conf: .catdb/settings.toml 中的 database 配置
-    :return: polars.DataFrame
-    """
-    try:
-        db_setting = get_settings().get(db_conf, {})
-        if not isinstance(db_setting, dict):
-            raise ValueError(f"Database configuration '{db_conf}' is not a dictionary.")
-
-        required_keys = ['user', 'password', 'url']
-        missing_keys = [key for key in required_keys if key not in db_setting]
-        if missing_keys:
-            raise KeyError(f"Missing required keys in database config: {missing_keys}")
-
-        user = urllib.parse.quote_plus(db_setting['user'])
-        password = urllib.parse.quote_plus(db_setting['password'])
-        uri = f"mysql://{user}:{password}@{db_setting['url']}"
-        return pl.read_database_uri(query, uri)
-
-    except KeyError as e:
-        raise RuntimeError("Database configuration error: missing required fields.") from e
-    except Exception as e:
-        raise RuntimeError(f"Failed to execute MySQL query: {e}") from e
-
-
-def read_ck(query: str, db_conf: str = "database.ck") -> pl.DataFrame:
-    """
-    读取 clickhouse 集群 返回 polars.DataFrame
-    :param query:
-    :param db_conf: .catdb/settings.toml 中的 database 配置
-    :return: polars.DataFrame
-    """
-    try:
-        db_setting = get_settings().get(db_conf, {})
-        if not isinstance(db_setting, dict):
-            raise ValueError(f"Database configuration '{db_conf}' is not a dictionary.")
-
-        required_keys = ['user', 'password', 'urls']
-        missing_keys = [key for key in required_keys if key not in db_setting]
-        if missing_keys:
-            raise KeyError(f"Missing required keys in database config: {missing_keys}")
-
-        user = urllib.parse.quote_plus(db_setting['user'])
-        password = urllib.parse.quote_plus(db_setting['password'])
-
-        with clickhouse_df.connect(db_setting['urls'], user=user, password=password):
-            return clickhouse_df.to_polars(query)
-
-    except KeyError as e:
-        raise RuntimeError("Database configuration error: missing required fields.") from e
-    except Exception as e:
-        raise RuntimeError(f"Failed to execute ClickHouse query: {e}") from e

ycat/parse.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2024/11/6 下午7:25
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-import sqlparse
-import re
-
-def format_sql(sql_content):
-    """将sql语句进行规范化，并去除sql中的注释，输入和输出均为字符串"""
-    parse_str = sqlparse.format(sql_content, reindent=True, strip_comments=True)
-    return parse_str
-
-def extract_temp_tables(with_clause):
-    """从WITH子句中提取临时表名，输出为列表"""
-    temp_tables = re.findall(r'\b(\w+)\s*as\s*\(', with_clause, re.IGNORECASE)
-    return temp_tables
-
-def extract_table_names_from_sql(sql_query):
-    """从sql中提取对应的表名称，输出为列表"""
-    table_names = set()
-    # 解析SQL语句
-    parsed = sqlparse.parse(sql_query)
-    # 正则表达式模式，用于匹配表名
-    table_name_pattern = r'\bFROM\s+([^\s\(\)\,]+)|\bJOIN\s+([^\s\(\)\,]+)'
-
-    # 用于存储WITH子句中的临时表名
-    remove_with_name = []
-
-    # 遍历解析后的语句块
-    for statement in parsed:
-        # 转换为字符串
-        statement_str = str(statement)# .lower()
-
-        # 将字符串中的特殊语法置空
-        statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)
-
-        # 查找匹配的表名
-        matches = re.findall(table_name_pattern, statement_str, re.IGNORECASE)
-
-        for match in matches:
-            # 提取非空的表名部分
-            for name in match:
-                if name:
-                    # 对于可能包含命名空间的情况，只保留最后一部分作为表名
-                    table_name = name.split('.')[-1]
-                    # 去除表名中的特殊符号
-                    table_name = re.sub(r'("|`|\'|;)', '', table_name)
-                    table_names.add(table_name)
-
-        # 处理特殊的WITH语句
-        if 'with' in statement_str:
-            remove_with_name = extract_temp_tables(statement_str)
-    # 移除多余的表名
-    if remove_with_name:
-        table_names = list(set(table_names) - set(remove_with_name))
-
-    return table_names
-
-
-

ycat/provider.py

@@ -1,101 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2025/5/23 01:34
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-import os
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import ygo
-import ylog
-from .client import CATDB
-
-DATE_FORMAT = "%Y-%m-%d"
-
-
-class Provider:
-    """
-    数据更新器
-    路径：{ycat.CATDB}/provider/{name}
-    """
-
-    def __init__(self, name: str, update_time="16:30"):
-        """
-        数据更新器
-        :param name: 数据更新器名称
-        :param update_time: 数据更新时间，默认16:30
-        """
-        self.name = name
-        self._tb_path = Path(CATDB) / name
-        os.makedirs(self._tb_path, exist_ok=True)
-        self._update_time = update_time
-        self.present = datetime.now().today()
-
-        if self.present.strftime("%H:%M") >= self._update_time:
-            self.last_date = self.present.strftime(DATE_FORMAT)
-        else:
-            self.last_date = (self.present - timedelta(days=1)).strftime(DATE_FORMAT)
-
-        self._tasks = list()
-        self._last_run_file = self._tb_path / f".last_run"
-        self.logger = ylog.get_logger("provider")
-
-    @property
-    def last_update_date(self):
-        return self._read_last_run_date()
-
-    def _read_last_run_date(self):
-        if self._last_run_file.exists():
-            with open(self._last_run_file, "r") as f:
-                return f.read().strip()
-        return
-
-    def _write_last_run_date(self, date_str: str):
-        with open(self._last_run_file, "w") as f:
-            f.write(date_str)
-
-    def wrap_fn(self, task_name: str, update_fn: callable):
-        """包装函数，添加异常处理"""
-        try:
-            update_fn()
-            return 0
-        except Exception as e:
-            self.logger.error(ygo.FailTaskError(task_name=task_name, error=e))
-            return 1
-
-    def add_task(self, task_name: str, update_fn: callable):
-        """添加任务"""
-        self._tasks.append((task_name, ygo.delay(self.wrap_fn)(task_name=task_name, update_fn=update_fn)))
-
-    def do(self,
-           overwrite: bool = False,
-           n_jobs: int = 10,
-           backend: str = "threading"):
-        """
-        执行任务
-        :param overwrite: 是否覆盖现有数据
-        :param n_jobs: 并发数
-        :param backend: loky/threading/multiprocessing
-        :return:
-        """
-        if not overwrite:
-            local_last_date = self._read_last_run_date()
-            if local_last_date is not None:
-                if local_last_date >= self.last_date:
-                    self.logger.info(f"[{self.name}] 已是最新数据，跳过更新")
-                    return
-        self.logger.info(f"[{self.name}] 更新数据")
-        failed_num = 0
-        with ygo.pool(n_jobs=n_jobs, backend=backend) as go:
-            for task_name, task in self._tasks:
-                go.submit(task, job_name=task_name)()
-            for status in go.do():
-                failed_num += status
-        if failed_num < 1:
-            self._write_last_run_date(self.last_date)
-            self.logger.info(f"[{self.name}] 更新成功，最新数据日期：{self.last_date}")
-        self.logger.info(f"[{self.name}] 更新完成，失败任务数：{str(failed_num).zfill(2)}/{str(len(self._tasks)).zfill(2)}")