ygo 1.0.1__tar.gz

ygo-1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 link-yundi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ygo-1.0.1/PKG-INFO

@@ -0,0 +1,95 @@
+Metadata-Version: 2.4
+Name: ygo
+Version: 1.0.1
+Project-URL: homepage, https://github.com/link-yundi/ygo
+Project-URL: repository, https://github.com/link-yundi/ygo
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: clickhouse-driver>=0.2.9
+Requires-Dist: duckdb>=1.2.2
+Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: joblib>=1.4.2
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: pandas>=2.0.3
+Requires-Dist: polars>=1.8.2
+Requires-Dist: pyarrow>=17.0.0
+Requires-Dist: pymysql>=1.1.1
+Requires-Dist: sqlalchemy>=2.0.40
+Requires-Dist: sqlparse>=0.5.3
+Requires-Dist: tqdm>=4.67.1
+Dynamic: license-file
+
+# ygo
+并发执行(加入进度条)以及延迟调用（基于joblib），以及获取对应函数的相关信息
+
+### 安装
+```shell
+pip install -U git+https://github.com/link-yundi/ygo.git
+```
+
+### 示例
+
+```
+├── a
+│   ├── __init__.py
+│   └── b
+│       ├── __init__.py
+│       └── c.py
+└── test.py
+
+c.py 中定义了目标函数
+def test_fn(a, b=2):
+    return a+b
+```
+
+#### 场景1: 并发
+
+```python
+import ygo
+import ylog
+from a.b.c import test_fn
+
+with ygo.pool(job_name="test parallel", show_progress=True) as go:
+    for i in range(10):
+        go.submit(test_fn)(a=i, b=2*i)
+    for res in go.do():
+        ylog.info(res)
+```
+
+#### 场景2: 延迟调用
+
+```
+>>> fn = delay(test_fn)(a=1, b=2)
+>>> fn()
+3
+>>> # 逐步传递参数
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1)
+>>> fn2 = delay(fn1)(b=2)
+>>> fn2(c=3)
+6
+>>> # 参数更改
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1, b=2)
+>>> fn2 = delay(fn1)(c=3, b=5)
+>>> fn2()
+9
+```
+
+#### 场景3: 获取目标函数信息
+
+```
+>>> ygo.fn_info(test_fn)
+=============================================================
+    a.b.c.test_fn(a, b=2)
+=============================================================
+    def test_fn(a, b=2):
+    return a+b
+```
+
+#### 场景4: 通过字符串解析函数并执行
+
+```
+>>> ygo.fn_from_str("a.b.c.test_fn")(a=1, b=5)
+6
+```
+

ygo-1.0.1/README.md

@@ -0,0 +1,73 @@
+# ygo
+并发执行(加入进度条)以及延迟调用（基于joblib），以及获取对应函数的相关信息
+
+### 安装
+```shell
+pip install -U git+https://github.com/link-yundi/ygo.git
+```
+
+### 示例
+
+```
+├── a
+│   ├── __init__.py
+│   └── b
+│       ├── __init__.py
+│       └── c.py
+└── test.py
+
+c.py 中定义了目标函数
+def test_fn(a, b=2):
+    return a+b
+```
+
+#### 场景1: 并发
+
+```python
+import ygo
+import ylog
+from a.b.c import test_fn
+
+with ygo.pool(job_name="test parallel", show_progress=True) as go:
+    for i in range(10):
+        go.submit(test_fn)(a=i, b=2*i)
+    for res in go.do():
+        ylog.info(res)
+```
+
+#### 场景2: 延迟调用
+
+```
+>>> fn = delay(test_fn)(a=1, b=2)
+>>> fn()
+3
+>>> # 逐步传递参数
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1)
+>>> fn2 = delay(fn1)(b=2)
+>>> fn2(c=3)
+6
+>>> # 参数更改
+>>> fn1 = delay(lambda a, b, c: a+b+c)(a=1, b=2)
+>>> fn2 = delay(fn1)(c=3, b=5)
+>>> fn2()
+9
+```
+
+#### 场景3: 获取目标函数信息
+
+```
+>>> ygo.fn_info(test_fn)
+=============================================================
+    a.b.c.test_fn(a, b=2)
+=============================================================
+    def test_fn(a, b=2):
+    return a+b
+```
+
+#### 场景4: 通过字符串解析函数并执行
+
+```
+>>> ygo.fn_from_str("a.b.c.test_fn")(a=1, b=5)
+6
+```
+

ygo-1.0.1/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ygo"
+version = "1.0.1"
+description = ""
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "clickhouse-driver>=0.2.9",
+    "duckdb>=1.2.2",
+    "dynaconf>=3.2.11",
+    "joblib>=1.4.2",
+    "loguru>=0.7.3",
+    "pandas>=2.0.3",
+    "polars>=1.8.2",
+    "pyarrow>=17.0.0",
+    "pymysql>=1.1.1",
+    "sqlalchemy>=2.0.40",
+    "sqlparse>=0.5.3",
+    "tqdm>=4.67.1",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["ygo", "ygo.*", "ylog", "ylog.*", "ycat", "ycat.*"]
+
+[project.urls]
+homepage = "https://github.com/link-yundi/ygo"
+repository = "https://github.com/link-yundi/ygo"

ygo-1.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ygo-1.0.1/ycat/__init__.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2025/5/14 18:29
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+
+from .client import HOME, CATDB, SETTINGS, sql, put, create_engine_ck, create_engine_mysql, read_mysql, read_ck
+
+__all__ = [
+    "HOME",
+    "CATDB",
+    "SETTINGS",
+    "sql",
+    "put",
+    "create_engine_ck",
+    "create_engine_mysql",
+    "read_mysql",
+    "read_ck",
+]

ygo-1.0.1/ycat/client.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Created on 2024/7/1 09:44
+@author: ZhangYundi
+@email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import os
+import re
+from typing import Optional
+from .yck import connect, query_polars
+
+import duckdb
+import polars as pl
+import ylog
+from dynaconf import Dynaconf
+from sqlalchemy import create_engine
+from functools import partial
+
+from .parse import extract_table_names_from_sql
+
+# 配置文件在 “~/.catdb/setting.toml”
+USERHOME = os.path.expanduser('~')  # 用户家目录
+CONFIG_PATH = os.path.join(USERHOME, ".catdb", "settings.toml")
+if not os.path.exists(CONFIG_PATH):
+    try:
+        os.makedirs(os.path.dirname(CONFIG_PATH))
+    except FileExistsError as e:
+        ...
+    except Exception as e:
+        ylog.error(f"配置文件生成失败: {e}")
+    catdb_path = os.path.join(USERHOME, "catdb")
+    template_content = f"""[paths]
+catdb="{catdb_path}"  # 本地数据库，默认家目录
+
+## 数据库配置：
+[database]
+[database.ck]
+# urls=["<host1>:<port1>", "<host2>:<port2>",]
+# user="xxx"
+# password="xxxxxx"
+[database.jy]
+# url="<host>:<port>"
+# user="xxxx"
+# password="xxxxxx"
+
+## 视情况自由增加其他配置
+    """
+    with open(CONFIG_PATH, "w") as f:
+        f.write(template_content)
+    ylog.info(f"生成配置文件: {CONFIG_PATH}")
+
+
+def get_settings():
+    try:
+        return Dynaconf(settings_files=[CONFIG_PATH])
+    except:
+        return
+
+
+HOME = USERHOME
+CATDB = os.path.join(HOME, "catdb")
+# 读取配置文件覆盖
+SETTINGS = get_settings()
+if SETTINGS is not None:
+    CATDB = SETTINGS.paths.catdb
+
+
+# ======================== 本地数据库 catdb ========================
+def tb_path(tb_name: str) -> str:
+    """
+    返回指定表名 完整的本地路径
+    Parameters
+    ----------
+    tb_name: str
+       表名，路径写法: a/b/c
+    Returns
+    -------
+        full_abs_path: str
+        完整的本地绝对路径 $HOME/catdb/a/b/c
+    """
+    return os.path.join(CATDB, tb_name)
+
+def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None):
+    """
+    将数据写入duck_db支持的parquet格式文件
+    Parameters
+    ----------
+    df: pandas.DataFrame | pandas.Series | polars.DataFrame
+        写入的数据
+    tb_name: str
+        表名,支持路径写法, a/b/c
+    partitions: Optional[List[str]]
+        根据哪些字段进行分区，默认不分区
+    """
+    tbpath = tb_path(tb_name)
+    if not os.path.exists(tbpath):
+        try:
+            os.makedirs(tbpath)
+        except FileExistsError as e:
+            pass
+    if partitions is not None:
+        for field in partitions:
+            assert field in df.columns, f'dataframe must have Field `{field}`'
+    df.write_parquet(tbpath, partition_by=partitions)
+
+
+def sql(query: str):
+    """
+    从duckdb中读取数据, query语法与mysql一致，特殊语法请查duckdb官网: https://duckdb.org/docs/sql/query_syntax/select
+    Parameters
+    ----------
+    query: str
+        查询语句
+    Returns
+    -------
+    result: duckdb.DuckDBPyRelation
+        查询结果
+    """
+    tbs = extract_table_names_from_sql(query)
+    convertor = dict()
+    for tb in tbs:
+        db_path = tb_path(tb)
+        format_tb = f"read_parquet('{db_path}/**/*.parquet', hive_partitioning = true)"
+        convertor[tb] = format_tb
+    pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
+    new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
+    conn = duckdb.connect()
+    conn.execute("PRAGMA disable_progress_bar;")
+    conn.execute("PRAGMA threads=1;")
+    res = conn.execute(new_query).fetch_arrow_table()
+    res = pl.from_arrow(res)
+    conn.close()
+    return res
+
+def create_engine_ck(urls: list[str], user: str, password: str):
+    return partial(connect, urls, user, password)
+
+def read_ck(sql, eng)->pl.DataFrame:
+    with eng() as conn:
+        return query_polars(sql, conn)
+
+def create_engine_mysql(url, user, password, database):
+    """
+    :param url: <host>:<port>
+    :param user:
+    :param password:
+    :param database:
+    :return:
+    """
+    engine = create_engine(f"mysql+pymysql://{user}:{password}@{url}/{database}")
+    return engine
+
+def read_mysql(sql, eng) -> pl.DataFrame:
+    with eng.connect() as conn:
+        return pl.read_database(sql, conn)