ygo 1.0.2__py3-none-any.whl → 1.2.12__py3-none-any.whl

ygo/utils.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/5/26 23:31
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+
+import inspect
+import os
+from functools import wraps
+from pathlib import Path
+import warnings
+from typing import Any
+
+from .delay import delay
+
+def deprecated(use_instead: str = None):
+    """
+    标记方法为弃用
+
+    Parameters
+    ----------
+    use_instead: str
+        推荐替代使用的方法或者类名称
+    Returns
+    -------
+
+    """
+
+    def decorator(func):
+        """
+        装饰器
+        Parameters
+        ----------
+        func: callable
+            被装饰的函数
+        Returns
+        -------
+
+        """
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            msg = f"`{func.__name__}` is deprecated. "
+            if use_instead:
+                msg += f"Please use `{use_instead}` instead."
+            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+def fn_params(func: callable):
+    """
+    获取fn的参数
+    Parameters
+    ----------
+    func: callable
+        需要获取参数的callable对象
+    Returns
+    -------
+    list[tuple]
+
+    """
+    stored = delay(func)().stored_kwargs.items()
+    return sorted(stored)
+
+
+def fn_signature_params(func: callable):
+    """获取fn所有定义的参数"""
+    return sorted(list(inspect.signature(func).parameters.keys()))
+
+
+def fn_path(fn: callable) -> str:
+    """
+    获取func所在的模块层级结构
+    Parameters
+    ----------
+    fn: callable
+        需要获取结构的callable对象
+    Returns
+    -------
+    str
+        用 `.` 连接各级层级
+    """
+    module = fn.__module__
+    # 检查模块是否有 __file__ 属性
+    if module.startswith('__main__'):
+        if hasattr(module, '__file__'):
+            module = module.__file__
+        else:
+            # 如果在交互式环境中，返回 None 或者一个默认值
+            module = "<interactive environment>"
+    if module.endswith('.py'):
+        module = module.split('.py')[0].split(str(Path(__file__).parent.parent.absolute()))[-1]
+        module = '.'.join(module.strip(os.sep).split(os.sep))
+    return module
+
+
+def fn_code(fn: callable) -> str:
+    """
+    返回fn具体的定义代码
+
+    Parameters
+    ----------
+    fn: callable
+        需要获取具体定义代码的callable对象
+
+    Returns
+    -------
+    str
+        以字符串封装定义代码
+
+    Examples
+    --------
+
+    >>> def test_fn(a, b=2):
+    >>>     return a+b
+    >>> print(fn_code())
+    def test_fn(a, b=2):
+        return a+b
+    """
+    return inspect.getsource(fn)
+
+
+def fn_info(fn: callable) -> str:
+    """获取函数的fn_mod, params, code"""
+    # mod = fn_path(fn)
+    params = fn_params(fn)
+    code = fn_code(fn)
+    all_define_params = sorted(list(inspect.signature(fn).parameters.keys()))
+
+    default_params = {k: v for k, v in params}
+    params_infos = list()
+    for p in all_define_params:
+        if p in default_params:
+            params_infos.append(f'{p}={default_params[p]}')
+        else:
+            params_infos.append(p)
+    params_infos = ', '.join(params_infos)
+
+    s = f"""
+=============================================================
+{fn.__name__}({params_infos})
+=============================================================
+{code}
+    """
+    return s
+
+@deprecated("lazy_import")
+def fn_from_str(s):
+    """
+    字符串导入对应fn
+    s: a.b.c.func
+    Parameters
+    ----------
+    s: str
+        模块的路径，分隔符 `.`
+    """
+    import importlib
+    *m_path, func = s.split(".")
+    m_path = ".".join(m_path)
+    mod = importlib.import_module(m_path)
+    _callable = getattr(mod, func)
+    return _callable
+
+@deprecated("lazy_import")
+def module_from_str(s):
+    """字符串导入模块"""
+    import importlib
+    m_path = ".".join(s.split('.'))
+    mod = importlib.import_module(m_path)
+    return mod
+
+def locate(path: str) -> Any:
+    """
+    Notes
+    -----
+    Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+
+    Locate an object by name or dotted path, importing as necessary.
+    This is similar to the pydoc function `locate`, except that it checks for
+    the module from the given path from back to front.
+    """
+    if path == "":
+        raise ImportError("Empty path")
+    from importlib import import_module
+    from types import ModuleType
+
+    parts = [part for part in path.split(".")]
+    for part in parts:
+        if not len(part):
+            raise ValueError(
+                f"Error loading '{path}': invalid dotstring."
+                + "\nRelative imports are not supported."
+            )
+    assert len(parts) > 0
+    part0 = parts[0]
+    try:
+        obj = import_module(part0)
+    except Exception as exc_import:
+        raise ImportError(
+            f"Error loading '{path}':\n{repr(exc_import)}"
+            + f"\nAre you sure that module '{part0}' is installed?"
+        ) from exc_import
+    for m in range(1, len(parts)):
+        part = parts[m]
+        try:
+            obj = getattr(obj, part)
+        except AttributeError as exc_attr:
+            parent_dotpath = ".".join(parts[:m])
+            if isinstance(obj, ModuleType):
+                mod = ".".join(parts[: m + 1])
+                try:
+                    obj = import_module(mod)
+                    continue
+                except ModuleNotFoundError as exc_import:
+                    raise ImportError(
+                        f"Error loading '{path}':\n{repr(exc_import)}"
+                        + f"\nAre you sure that '{part}' is importable from module '{parent_dotpath}'?"
+                    ) from exc_import
+                except Exception as exc_import:
+                    raise ImportError(
+                        f"Error loading '{path}':\n{repr(exc_import)}"
+                    ) from exc_import
+            raise ImportError(
+                f"Error loading '{path}':\n{repr(exc_attr)}"
+                + f"\nAre you sure that '{part}' is an attribute of '{parent_dotpath}'?"
+            ) from exc_attr
+    return obj

ygo-1.2.12.dist-info/METADATA

@@ -0,0 +1,119 @@
+Metadata-Version: 2.4
+Name: ygo
+Version: 1.2.12
+Summary: 一个轻量级 Python 工具包，支持 并发执行（带进度条）、延迟调用、链式绑定参数、函数信息获取、模块/函数动态加载。
+Project-URL: homepage, https://github.com/link-yundi/ygo
+Project-URL: repository, https://github.com/link-yundi/ygo
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: joblib>=1.5.0
+Requires-Dist: logair>=1.0.6
+Requires-Dist: psutil>=7.2.1
+Requires-Dist: tqdm>=4.67.1
+Dynamic: license-file
+
+# ygo
+一个轻量级 Python 工具包，底层基于 joblib 和 tqdm 、loguru 实现，支持
+- 并发执行（带进度条）
+- 延迟调用
+- 链式绑定参数
+- 函数信息获取
+- 模块/函数动态加载...
+- 并结合 ylog 提供日志记录能力
+
+### 安装
+```shell
+pip install -U ygo
+```
+
+### 🧰 功能概览
+
+| 模块   | 功能                                                         |
+| :----- | :----------------------------------------------------------- |
+| `ygo`  | 支持并发执行（带进度条）、延迟调用、函数信息获取以及模块/函数动态加载等功能 |
+| `ylog` | 日志模块，提供统一的日志输出接口                             |
+
+### 示例
+
+```
+├── a
+│   ├── __init__.py
+│   └── b
+│       ├── __init__.py
+│       └── c.py
+└── test.py
+
+c.py 中定义了目标函数
+def test_fn(a, b=2):
+    return a+b
+```
+
+#### 场景1: 并发执行
+
+```python
+import ygo
+import ylog
+from a.b.c import test_fn
+
+with ygo.pool(n_jobs=5, show_progress=True) as go:
+    for i in range(10):
+        go.submit(test_fn)(a=i, b=2*i)
+    for res in go.do():
+        ylog.info(res)
+```
+
+#### ✅ `ygo.pool` 支持的参数
+
+| 参数名        | 类型 | 描述                                                         |
+| ------------- | ---- | ------------------------------------------------------------ |
+| n_jobs        | int  | 并行任务数(<=1 表示串行)                                     |
+| show_progress | bool | 是否显示进度条                                               |
+| backend       | str  | 执行后端（默认 'threading'，可选 'multiprocessing' 或 'loky'） |
+
+#### 场景2: 延迟调用
+
+```
+>>> fn = delay(test_fn)(a=1, b=2)
+>>> fn()
+3
+>>> # 逐步传递参数
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1)
+>>> fn2 = delay(fn1)(b=2)
+>>> fn2(c=3)
+6
+>>> # 参数更改
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1, b=2)
+>>> fn2 = delay(fn1)(c=3, b=5)
+>>> fn2()
+9
+```
+
+#### 场景3: 获取目标函数信息
+
+```
+>>> ygo.fn_info(test_fn)
+=============================================================
+    a.b.c.test_fn(a, b=2)
+=============================================================
+    def test_fn(a, b=2):
+    return a+b
+```
+
+#### 🔍 其他函数信息工具
+
+| 方法名                    | 描述                                     |
+| ------------------------- | ---------------------------------------- |
+| `fn_params(fn)`           | 获取函数实参                             |
+| `fn_signature_params(fn)` | 获取函数定义的所有参数名                 |
+| `fn_code(fn)`             | 获取函数源码字符串                       |
+| `fn_path(fn)`             | 获取函数所属模块路径                     |
+| `fn_from_str(s)`          | 根据字符串导入函数（如 "a.b.c.test_fn"） |
+| `module_from_str(s)`      | 根据字符串导入模块                       |
+
+#### 场景4: 通过字符串解析函数并执行
+
+```
+>>> ygo.fn_from_str("a.b.c.test_fn")(a=1, b=5)
+6
+```

ygo-1.2.12.dist-info/RECORD

@@ -0,0 +1,11 @@
+ygo/__init__.py,sha256=vP3Vs9LarNsn2TV8QlpuRP94WWK7m3q-gIWw6vLRED0,537
+ygo/delay.py,sha256=66xtPXqyD630FL7LWL5qJKAIZvyGDwZyM4qPfk8Czlg,2206
+ygo/exceptions.py,sha256=0OYDYt_9KKo8mF2XBG5QkCMr3-ASp69VDSPOEwlIsrI,660
+ygo/lazy.py,sha256=NRE38vWKfA19fTRIbgCypdoxXhJulDtHMd7jw6CzhUc,1303
+ygo/pool.py,sha256=hKvUttBhK2gU4WHocvg-2bjS9uVJ9jITbRb1tL2JYY4,10699
+ygo/utils.py,sha256=45ZM7FUraDA69jwf1arbBuuS-a0wMVf9sT7ma5bGd4E,6224
+ygo-1.2.12.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
+ygo-1.2.12.dist-info/METADATA,sha256=awbNtAfd34cw1Rjyc2WHerxgw4U8kSoozKgtUNNCQWw,3612
+ygo-1.2.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygo-1.2.12.dist-info/top_level.txt,sha256=4E07GOD3KS5--r4rP13upXfTEgk3JGJpQmMkdkJ4t74,4
+ygo-1.2.12.dist-info/RECORD,,

{ygo-1.0.2.dist-info → ygo-1.2.12.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ygo-1.2.12.dist-info/top_level.txt

@@ -0,0 +1 @@
+ygo

ycat/__init__.py

@@ -1,34 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2025/5/14 18:29
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-
-from .client import (
-    HOME,
-    CATDB,
-    get_settings,
-    sql,
-    put,
-    create_engine_ck,
-    create_engine_mysql,
-    read_mysql,
-    read_ck,
-    tb_path,
-)
-
-__all__ = [
-    "HOME",
-    "CATDB",
-    "get_settings",
-    "sql",
-    "put",
-    "create_engine_ck",
-    "create_engine_mysql",
-    "read_mysql",
-    "read_ck",
-    "tb_path",
-]

ycat/client.py

@@ -1,142 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2024/7/1 09:44
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-import os
-import re
-from functools import partial
-from typing import Optional
-
-import polars as pl
-from dynaconf import Dynaconf
-from sqlalchemy import create_engine
-
-import ylog
-from .parse import extract_table_names_from_sql
-from .yck import connect, query_polars
-
-# 配置文件在 “~/.catdb/setting.toml”
-USERHOME = os.path.expanduser('~')  # 用户家目录
-NAME = "catdb"
-CONFIG_PATH = os.path.join(USERHOME, f".{NAME}", "settings.toml")
-if not os.path.exists(CONFIG_PATH):
-    try:
-        os.makedirs(os.path.dirname(CONFIG_PATH))
-    except FileExistsError as e:
-        ...
-    except Exception as e:
-        ylog.error(f"配置文件生成失败: {e}")
-    catdb_path = os.path.join(USERHOME, NAME)
-    template_content = f"""[paths]
-{NAME}="{catdb_path}"  # 本地数据库，默认家目录
-
-## 数据库配置：
-[database]
-[database.ck]
-# urls=["<host1>:<port1>", "<host2>:<port2>",]
-# user="xxx"
-# password="xxxxxx"
-[database.jy]
-# url="<host>:<port>"
-# user="xxxx"
-# password="xxxxxx"
-
-## 视情况自由增加其他配置
-    """
-    with open(CONFIG_PATH, "w") as f:
-        f.write(template_content)
-    ylog.info(f"生成配置文件: {CONFIG_PATH}")
-
-
-def get_settings():
-    try:
-        return Dynaconf(settings_files=[CONFIG_PATH]).as_dict()
-    except:
-        return {}
-
-
-HOME = USERHOME
-CATDB = os.path.join(HOME, NAME)
-# 读取配置文件覆盖
-SETTINGS = get_settings()
-if SETTINGS is not None:
-    CATDB = SETTINGS["PATHS"][NAME]
-    if not CATDB.endswith(NAME):
-        CATDB = os.path.join(CATDB, NAME)
-
-
-# ======================== 本地数据库 catdb ========================
-def tb_path(tb_name: str) -> str:
-    """
-    返回指定表名 完整的本地路径
-    Parameters
-    ----------
-    tb_name: str
-       表名，路径写法: a/b/c
-    Returns
-    -------
-        full_abs_path: str
-        完整的本地绝对路径 $HOME/catdb/a/b/c
-    """
-    return os.path.join(CATDB, tb_name)
-
-
-def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None, abs_path: bool = False):
-    if not abs_path:
-        tbpath = tb_path(tb_name)
-    else:
-        tbpath = tb_name
-    if not os.path.exists(tbpath):
-        try:
-            os.makedirs(tbpath)
-        except FileExistsError as e:
-            pass
-    if partitions is not None:
-        for field in partitions:
-            assert field in df.columns, f'dataframe must have Field `{field}`'
-    df.write_parquet(tbpath, partition_by=partitions)
-
-
-def sql(query: str, abs_path: bool = False):
-    tbs = extract_table_names_from_sql(query)
-    convertor = dict()
-    for tb in tbs:
-        if not abs_path:
-            db_path = tb_path(tb)
-        else:
-            db_path = tb
-        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
-        convertor[tb] = format_tb
-    pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
-    new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
-    return pl.sql(new_query).collect()
-
-
-def create_engine_ck(urls: list[str], user: str, password: str):
-    return partial(connect, urls, user, password)
-
-
-def read_ck(sql, eng) -> pl.DataFrame:
-    with eng() as conn:
-        return query_polars(sql, conn)
-
-
-def create_engine_mysql(url, user, password, database):
-    """
-    :param url: <host>:<port>
-    :param user:
-    :param password:
-    :param database:
-    :return:
-    """
-    engine = create_engine(f"mysql+pymysql://{user}:{password}@{url}/{database}")
-    return engine
-
-
-def read_mysql(sql, eng) -> pl.DataFrame:
-    with eng.connect() as conn:
-        return pl.read_database(sql, conn)