ygo 1.0.1__tar.gz → 1.0.2__tar.gz

{ygo-1.0.1/ygo.egg-info → ygo-1.0.2}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.1
+Version: 1.0.2
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clickhouse-driver>=0.2.9
-Requires-Dist: duckdb>=1.2.2
 Requires-Dist: dynaconf>=3.2.11
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: loguru>=0.7.3

{ygo-1.0.1 → ygo-1.0.2}/pyproject.toml

@@ -4,13 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygo"
-version = "1.0.1"
+version = "1.0.2"
 description = ""
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
     "clickhouse-driver>=0.2.9",
-    "duckdb>=1.2.2",
     "dynaconf>=3.2.11",
     "joblib>=1.4.2",
     "loguru>=0.7.3",

{ygo-1.0.1 → ygo-1.0.2}/ycat/__init__.py

@@ -7,16 +7,28 @@ Created on 2025/5/14 18:29
 ---------------------------------------------
 """
 
-from .client import HOME, CATDB, SETTINGS, sql, put, create_engine_ck, create_engine_mysql, read_mysql, read_ck
+from .client import (
+    HOME,
+    CATDB,
+    get_settings,
+    sql,
+    put,
+    create_engine_ck,
+    create_engine_mysql,
+    read_mysql,
+    read_ck,
+    tb_path,
+)
 
 __all__ = [
     "HOME",
     "CATDB",
-    "SETTINGS",
+    "get_settings",
     "sql",
     "put",
     "create_engine_ck",
     "create_engine_mysql",
     "read_mysql",
     "read_ck",
+    "tb_path",
 ]

{ygo-1.0.1 → ygo-1.0.2}/ycat/client.py

@@ -8,21 +8,21 @@ Created on 2024/7/1 09:44
 """
 import os
 import re
+from functools import partial
 from typing import Optional
-from .yck import connect, query_polars
 
-import duckdb
 import polars as pl
-import ylog
 from dynaconf import Dynaconf
 from sqlalchemy import create_engine
-from functools import partial
 
+import ylog
 from .parse import extract_table_names_from_sql
+from .yck import connect, query_polars
 
 # 配置文件在 “~/.catdb/setting.toml”
 USERHOME = os.path.expanduser('~')  # 用户家目录
-CONFIG_PATH = os.path.join(USERHOME, ".catdb", "settings.toml")
+NAME = "catdb"
+CONFIG_PATH = os.path.join(USERHOME, f".{NAME}", "settings.toml")
 if not os.path.exists(CONFIG_PATH):
     try:
         os.makedirs(os.path.dirname(CONFIG_PATH))
@@ -30,9 +30,9 @@ if not os.path.exists(CONFIG_PATH):
         ...
     except Exception as e:
         ylog.error(f"配置文件生成失败: {e}")
-    catdb_path = os.path.join(USERHOME, "catdb")
+    catdb_path = os.path.join(USERHOME, NAME)
     template_content = f"""[paths]
-catdb="{catdb_path}"  # 本地数据库，默认家目录
+{NAME}="{catdb_path}"  # 本地数据库，默认家目录
 
 ## 数据库配置：
 [database]
@@ -54,17 +54,19 @@ catdb="{catdb_path}"  # 本地数据库，默认家目录
 
 def get_settings():
     try:
-        return Dynaconf(settings_files=[CONFIG_PATH])
+        return Dynaconf(settings_files=[CONFIG_PATH]).as_dict()
     except:
-        return
+        return {}
 
 
 HOME = USERHOME
-CATDB = os.path.join(HOME, "catdb")
+CATDB = os.path.join(HOME, NAME)
 # 读取配置文件覆盖
 SETTINGS = get_settings()
 if SETTINGS is not None:
-    CATDB = SETTINGS.paths.catdb
+    CATDB = SETTINGS["PATHS"][NAME]
+    if not CATDB.endswith(NAME):
+        CATDB = os.path.join(CATDB, NAME)
 
 
 # ======================== 本地数据库 catdb ========================
@@ -82,19 +84,12 @@ def tb_path(tb_name: str) -> str:
     """
     return os.path.join(CATDB, tb_name)
 
-def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None):
-    """
-    将数据写入duck_db支持的parquet格式文件
-    Parameters
-    ----------
-    df: pandas.DataFrame | pandas.Series | polars.DataFrame
-        写入的数据
-    tb_name: str
-        表名,支持路径写法, a/b/c
-    partitions: Optional[List[str]]
-        根据哪些字段进行分区，默认不分区
-    """
-    tbpath = tb_path(tb_name)
+
+def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None, abs_path: bool = False):
+    if not abs_path:
+        tbpath = tb_path(tb_name)
+    else:
+        tbpath = tb_name
     if not os.path.exists(tbpath):
         try:
             os.makedirs(tbpath)
@@ -106,41 +101,30 @@ def put(df: pl.DataFrame, tb_name: str, partitions: Optional[list[str]] = None):
     df.write_parquet(tbpath, partition_by=partitions)
 
 
-def sql(query: str):
-    """
-    从duckdb中读取数据, query语法与mysql一致，特殊语法请查duckdb官网: https://duckdb.org/docs/sql/query_syntax/select
-    Parameters
-    ----------
-    query: str
-        查询语句
-    Returns
-    -------
-    result: duckdb.DuckDBPyRelation
-        查询结果
-    """
+def sql(query: str, abs_path: bool = False):
     tbs = extract_table_names_from_sql(query)
     convertor = dict()
     for tb in tbs:
-        db_path = tb_path(tb)
-        format_tb = f"read_parquet('{db_path}/**/*.parquet', hive_partitioning = true)"
+        if not abs_path:
+            db_path = tb_path(tb)
+        else:
+            db_path = tb
+        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
         convertor[tb] = format_tb
     pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
     new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
-    conn = duckdb.connect()
-    conn.execute("PRAGMA disable_progress_bar;")
-    conn.execute("PRAGMA threads=1;")
-    res = conn.execute(new_query).fetch_arrow_table()
-    res = pl.from_arrow(res)
-    conn.close()
-    return res
+    return pl.sql(new_query).collect()
+
 
 def create_engine_ck(urls: list[str], user: str, password: str):
     return partial(connect, urls, user, password)
 
-def read_ck(sql, eng)->pl.DataFrame:
+
+def read_ck(sql, eng) -> pl.DataFrame:
     with eng() as conn:
         return query_polars(sql, conn)
 
+
 def create_engine_mysql(url, user, password, database):
     """
     :param url: <host>:<port>
@@ -152,6 +136,7 @@ def create_engine_mysql(url, user, password, database):
     engine = create_engine(f"mysql+pymysql://{user}:{password}@{url}/{database}")
     return engine
 
+
 def read_mysql(sql, eng) -> pl.DataFrame:
     with eng.connect() as conn:
-        return pl.read_database(sql, conn)
+        return pl.read_database(sql, conn)

{ygo-1.0.1 → ygo-1.0.2}/ycat/parse.py

@@ -33,7 +33,7 @@ def extract_table_names_from_sql(sql_query):
     # 遍历解析后的语句块
     for statement in parsed:
         # 转换为字符串
-        statement_str = str(statement).lower()
+        statement_str = str(statement)# .lower()
 
         # 将字符串中的特殊语法置空
         statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)

{ygo-1.0.1 → ygo-1.0.2/ygo.egg-info}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.4
 Name: ygo
-Version: 1.0.1
+Version: 1.0.2
 Project-URL: homepage, https://github.com/link-yundi/ygo
 Project-URL: repository, https://github.com/link-yundi/ygo
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: clickhouse-driver>=0.2.9
-Requires-Dist: duckdb>=1.2.2
 Requires-Dist: dynaconf>=3.2.11
 Requires-Dist: joblib>=1.4.2
 Requires-Dist: loguru>=0.7.3

{ygo-1.0.1 → ygo-1.0.2}/ygo.egg-info/requires.txt

@@ -1,5 +1,4 @@
 clickhouse-driver>=0.2.9
-duckdb>=1.2.2
 dynaconf>=3.2.11
 joblib>=1.4.2
 loguru>=0.7.3