ygo 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

ycat/__init__.py

@@ -7,16 +7,28 @@ Created on 2025/5/14 18:29
 ---------------------------------------------
 """
 
-from .client import HOME, CATDB, SETTINGS, sql, put, create_engine_ck, create_engine_mysql, read_mysql, read_ck
+from .client import (
+    HOME,
+    CATDB,
+    get_settings,
+    sql,
+    put,
+    create_engine_ck,
+    create_engine_mysql,
+    read_mysql,
+    read_ck,
+    tb_path,
+)
 
 __all__ = [
     "HOME",
     "CATDB",
-    "SETTINGS",
+    "get_settings",
     "sql",
     "put",
     "create_engine_ck",
     "create_engine_mysql",
     "read_mysql",
     "read_ck",
+    "tb_path",
 ]

ycat/client.py

@@ -8,21 +8,21 @@ Created on 2024/7/1 09:44
 """
 import os
 import re
+from functools import partial
 from typing import Optional
-from .yck import connect, query_polars
 
-import duckdb
 import polars as pl
-import ylog
 from dynaconf import Dynaconf
 from sqlalchemy import create_engine
-from functools import partial
 
+import ylog
 from .parse import extract_table_names_from_sql
+from .yck import connect, query_polars
 
 # 配置文件在 “~/.catdb/setting.toml”
 USERHOME = os.path.expanduser('~')  # 用户家目录
-CONFIG_PATH = os.path.join(USERHOME, ".catdb", "settings.toml")
+NAME = "catdb"
+CONFIG_PATH = os.path.join(USERHOME, f".{NAME}", "settings.toml")
 if not os.path.exists(CONFIG_PATH):
     try:
         os.makedirs(os.path.dirname(CONFIG_PATH))
@@ -30,9 +30,9 @@ if not os.path.exists(CONFIG_PATH):
         ...
     except Exception as e:
         ylog.error(f"配置文件生成失败: {e}")
-    catdb_path = os.path.join(USERHOME, "catdb")
+    catdb_path = os.path.join(USERHOME, NAME)
     template_content = f"""[paths]
-catdb="{catdb_path}"  # 本地数据库，默认家目录
+{NAME}="{catdb_path}"  # 本地数据库，默认家目录
 
 ## 数据库配置：
 [database]
@@ -54,17 +54,19 @@ catdb="{catdb_path}"  # 本地数据库，默认家目录
 
 def get_settings():
     try:
-        return Dynaconf(settings_files=[CONFIG_PATH])
+        return Dynaconf(settings_files=[CONFIG_PATH]).as_dict()
     except:
-        return
+        return {}
 
 
 HOME = USERHOME
-CATDB = os.path.join(HOME, "catdb")
+CATDB = os.path.join(HOME, NAME)
 # 读取配置文件覆盖
 SETTINGS = get_settings()
 if SETTINGS is not None:
-    CATDB = SETTINGS.paths.catdb
+    CATDB = SETTINGS["PATHS"][NAME]
+    if not CATDB.endswith(NAME):
+        CATDB = os.path.join(CATDB, NAME)
 
 
 # ======================== 本地数据库 catdb ========================
@@ -82,19 +84,12 @@ def tb_path(tb_name: str) -> str:
     """
     return os.path.join(CATDB, tb_name)
 
-def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None):
-    """
-    将数据写入duck_db支持的parquet格式文件
-    Parameters
-    ----------
-    df: pandas.DataFrame | pandas.Series | polars.DataFrame
-        写入的数据
-    tb_name: str
-        表名,支持路径写法, a/b/c
-    partitions: Optional[List[str]]
-        根据哪些字段进行分区，默认不分区
-    """
-    tbpath = tb_path(tb_name)
+
+def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None, abs_path: bool = False):
+    if not abs_path:
+        tbpath = tb_path(tb_name)
+    else:
+        tbpath = tb_name
     if not os.path.exists(tbpath):
         try:
             os.makedirs(tbpath)
@@ -106,41 +101,30 @@ def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None):
     df.write_parquet(tbpath, partition_by=partitions)
 
 
-def sql(query: str):
-    """
-    从duckdb中读取数据, query语法与mysql一致，特殊语法请查duckdb官网: https://duckdb.org/docs/sql/query_syntax/select
-    Parameters
-    ----------
-    query: str
-        查询语句
-    Returns
-    -------
-    result: duckdb.DuckDBPyRelation
-        查询结果
-    """
+def sql(query: str, abs_path: bool = False):
     tbs = extract_table_names_from_sql(query)
     convertor = dict()
     for tb in tbs:
-        db_path = tb_path(tb)
-        format_tb = f"read_parquet('{db_path}/**/*.parquet', hive_partitioning = true)"
+        if not abs_path:
+            db_path = tb_path(tb)
+        else:
+            db_path = tb
+        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
         convertor[tb] = format_tb
     pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
     new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
-    conn = duckdb.connect()
-    conn.execute("PRAGMA disable_progress_bar;")
-    conn.execute("PRAGMA threads=1;")
-    res = conn.execute(new_query).fetch_arrow_table()
-    res = pl.from_arrow(res)
-    conn.close()
-    return res
+    return pl.sql(new_query).collect()
+
 
 def create_engine_ck(urls: list[str], user: str, password: str):
     return partial(connect, urls, user, password)
 
-def read_ck(sql, eng)->pl.DataFrame:
+
+def read_ck(sql, eng) -> pl.DataFrame:
     with eng() as conn:
         return query_polars(sql, conn)
 
+
 def create_engine_mysql(url, user, password, database):
     """
     :param url: <host>:<port>
@@ -152,6 +136,7 @@ def create_engine_mysql(url, user, password, database):
     engine = create_engine(f"mysql+pymysql://{user}:{password}@{url}/{database}")
     return engine
 
+
 def read_mysql(sql, eng) -> pl.DataFrame:
     with eng.connect() as conn:
-        return pl.read_database(sql, conn)
+        return pl.read_database(sql, conn)

ycat/parse.py

@@ -33,7 +33,7 @@ def extract_table_names_from_sql(sql_query):
     # 遍历解析后的语句块
     for statement in parsed:
         # 转换为字符串
-        statement_str = str(statement).lower()
+        statement_str = str(statement)# .lower()
 
         # 将字符串中的特殊语法置空
         statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)

{ygo-1.0.1.dist-info → ygo-1.0.3.dist-info}/METADATA

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.1
+Version: 1.0.3
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clickhouse-driver>=0.2.9
-Requires-Dist: duckdb>=1.2.2
 Requires-Dist: dynaconf>=3.2.11
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: loguru>=0.7.3

{ygo-1.0.1.dist-info → ygo-1.0.3.dist-info}/RECORD

@@ -1,15 +1,15 @@
-ycat/__init__.py,sha256=R5sHDLS6E2Rw7m6nWny-NibmfPu_DeFabpA_AnIA3IQ,477
-ycat/client.py,sha256=q4Hdbat1q78S9YwrKGTtYNT6fiatgE_8-NXZktM1FS0,4377
+ycat/__init__.py,sha256=zBMOFStzKSt_5jw4af6YFtPD5Svr8fJlZkP_AdUjCoA,554
+ycat/client.py,sha256=tsrJUeoy_cFiHjX7hnh0dtnO_3njhtMLrQcAfi0TYgQ,3796
 ycat/dtype.py,sha256=mRGLDe_Ho6-tDsoj5wwrAzozEoIYCAHGKdpRqgBfUcI,12577
-ycat/parse.py,sha256=D9vhz-qLR-UJzFoEVWlQ44M-AMykHT8k3Lf7BwnaTHY,2287
+ycat/parse.py,sha256=9Kgr33nHYC96TGpZs98PAu0cbK-FrR0pfuf8lfD647I,2289
 ycat/yck.py,sha256=FlGMBuKEngB4TwFXMp4P3dLg9IfFmUg3eDqXzQ0kQoI,2738
 ygo/__init__.py,sha256=FMN06Tfa8_oV26eklBZCtGTyHZ6MghHxHj4PS_FSXCA,222
 ygo/exceptions.py,sha256=4Kd92kpwpsXHJJkSv4OqcN--PEEvIGGvDDgOOsk68gg,385
 ygo/ygo.py,sha256=vCMUur_41yY0QB4gj8K5wBZHql_cbmANhI8QwPRCTmo,11613
-ygo-1.0.1.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
+ygo-1.0.3.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
 ylog/__init__.py,sha256=2sIp4PHNoQMCi0QtIarTI4raACd7SdRHNY7fY5hKYwc,397
 ylog/core.py,sha256=uO_r5wDrBN5edV_TDHUNzcqRs6DMDDrrjfyqYZhak4w,7716
-ygo-1.0.1.dist-info/METADATA,sha256=zGBxxHMOrJU474buyCQU745rfjkdnctVOWDXPGL4XEc,2009
-ygo-1.0.1.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-ygo-1.0.1.dist-info/top_level.txt,sha256=jEbfiz5fX4iSzDg8_Npdv5SIC_Kphmb1m3vuyD9ZC1E,14
-ygo-1.0.1.dist-info/RECORD,,
+ygo-1.0.3.dist-info/METADATA,sha256=ksiTv_WXxiia5RlkTkbHRU8v4vZudgatBIaVe7PZOfo,1980
+ygo-1.0.3.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+ygo-1.0.3.dist-info/top_level.txt,sha256=jEbfiz5fX4iSzDg8_Npdv5SIC_Kphmb1m3vuyD9ZC1E,14
+ygo-1.0.3.dist-info/RECORD,,

{ygo-1.0.1.dist-info → ygo-1.0.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any