lidb 0.1.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

lidb/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/7/17 14:09
+# Description:
+
+from .init import (
+    NAME,
+    DB_PATH,
+    CONFIG_PATH,
+    LOGS_PATH,
+    get_settings,
+)
+
+from .database import (
+    sql,
+    put,
+    has,
+    tb_path,
+)
+
+__version__ = "1.0.0"

lidb/database.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Copyright (c) 2025 ZhangYundi
+Licensed under the MIT License.
+Created on 2024/7/1 09:44
+Email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import re
+from pathlib import Path
+from .parse import extract_table_names_from_sql
+from .init import DB_PATH, logger
+
+# ======================== 本地数据库 catdb ========================
+def tb_path(tb_name: str) -> Path:
+    """
+    返回指定表名 完整的本地路径
+    Parameters
+    ----------
+    tb_name: str
+       表名，路径写法: a/b/c
+    Returns
+    -------
+    pathlib.Path
+        full_abs_path: pathlib.Path
+        完整的本地绝对路径 $DB_PATH/a/b/c
+    """
+    return Path(DB_PATH, tb_name)
+
+
+def put(df, tb_name: str, partitions: list[str] | None = None, abs_path: bool = False):
+    """
+    将一个DataFrame写入到指定名称的表格目录中，支持分区存储。
+
+    该函数负责将给定的DataFrame（df）根据提供的表名（tb_name）写入到本地文件系统中。
+    如果指定了分区（partitions），则会按照这些分区列将数据分割存储。此外，可以通过abs_path参数
+    指定tb_name是否为绝对路径。如果目录不存在，会自动创建目录。
+
+    Parameters
+    ----------
+    df: polars.DataFrame
+    tb_name: str
+        表的名称，用于确定存储数据的目录
+    partitions: list[str] | None
+        指定用于分区的列名列表。如果未提供，则不进行分区。
+    abs_path: bool
+        tb_name是否应被视为绝对路径。默认为False。
+
+    Returns
+    -------
+
+    """
+    if df is None:
+        logger.warning(f"put failed: input data is None.")
+        return
+    if df.is_empty():
+        logger.warning(f"put failed: input data is empty.")
+        return
+    if not abs_path:
+        tbpath = tb_path(tb_name)
+    else:
+        tbpath = Path(tb_name)
+    if not tbpath.exists():
+        tbpath.mkdir(parents=True, exist_ok=True)
+    if partitions is not None:
+        df.write_parquet(tbpath, partition_by=partitions)
+    else:
+        df.write_parquet(tbpath / "data.parquet")
+
+def has(tb_name: str) -> bool:
+    """
+    判定给定的表名是否存在
+    Parameters
+    ----------
+    tb_name: str
+
+    Returns
+    -------
+
+    """
+    return tb_path(tb_name).exists()
+
+def sql(query: str, abs_path: bool = False):
+    """
+    sql 查询，从本地paquet文件中查询数据
+
+    Parameters
+    ----------
+    query: str
+        sql查询语句
+    abs_path: bool
+        是否使用绝对路径作为表路径。默认为False
+    Returns
+    -------
+
+    """
+    import polars as pl
+
+    tbs = extract_table_names_from_sql(query)
+    convertor = dict()
+    for tb in tbs:
+        if not abs_path:
+            db_path = tb_path(tb)
+        else:
+            db_path = tb
+        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
+        convertor[tb] = format_tb
+    pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
+    new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
+    return pl.sql(new_query)
+

lidb/init.py

@@ -0,0 +1,65 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/7/17 14:40
+# Description:
+
+from pathlib import Path
+from dynaconf import Dynaconf
+from loguru import logger
+import sys
+
+
+USERHOME = Path("~").expanduser() # 用户家目录
+NAME = "lidb"
+DB_PATH = USERHOME / NAME
+CONFIG_PATH = DB_PATH / "conf" / "settings.toml"
+LOGS_PATH = DB_PATH / "logs"
+
+logger.remove()
+
+console_format = (
+    "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
+    "<level>{level}</level> | "
+    f"<cyan>{NAME}</cyan>:<cyan>{{function}}</cyan>:<cyan>{{line}}</cyan> - "
+    "<level>{message}</level>"
+)
+
+logger.add(
+    sys.stderr,
+    format=console_format,
+    level="TRACE"
+)
+
+logger.add(
+    LOGS_PATH / "{time:YYYYMMDD}.log",
+    retention="10 days",
+    format=f"{{time:YYYY-MM-DD HH:mm:ss}} | {{level}} | {NAME}:{{function}}:{{line}} - {{message}}",
+    level="TRACE"
+)
+
+if not CONFIG_PATH.exists():
+    try:
+        CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        logger.error(f"Create settings file failed: {e}")
+    with open(CONFIG_PATH, "w") as f:
+        template_content = f"""[global]
+path="{DB_PATH}" 
+"""
+    with open(CONFIG_PATH, "w") as f:
+        f.write(template_content)
+    logger.info(f"Settings file created: {CONFIG_PATH}")
+
+def get_settings():
+    try:
+        return Dynaconf(settings_files=[CONFIG_PATH])
+    except Exception as e:
+        logger.error(f"Read settings file failed: {e}")
+        return {}
+
+# 读取配置文件覆盖
+_settiings = get_settings()
+if _settiings is not None:
+    setting_db_path = _settiings.get(f"global.path", "")
+    if setting_db_path:
+        DB_PATH = Path(setting_db_path)

lidb/parse.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Copyright (c) 2025 ZhangYundi
+Licensed under the MIT License.
+Created on 2024/11/6 下午7:25
+Email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import sqlparse
+import re
+
+def format_sql(sql_content):
+    """将sql语句进行规范化，并去除sql中的注释，输入和输出均为字符串"""
+    parse_str = sqlparse.format(sql_content, reindent=True, strip_comments=True)
+    return parse_str
+
+def extract_temp_tables(with_clause):
+    """从WITH子句中提取临时表名，输出为列表"""
+    temp_tables = re.findall(r'\b(\w+)\s*as\s*\(', with_clause, re.IGNORECASE)
+    return temp_tables
+
+def extract_table_names_from_sql(sql_query):
+    """从sql中提取对应的表名称，输出为列表"""
+    table_names = set()
+    # 解析SQL语句
+    parsed = sqlparse.parse(sql_query)
+    # 正则表达式模式，用于匹配表名
+    table_name_pattern = r'\bFROM\s+([^\s\(\)\,]+)|\bJOIN\s+([^\s\(\)\,]+)'
+
+    # 用于存储WITH子句中的临时表名
+    remove_with_name = []
+
+    # 遍历解析后的语句块
+    for statement in parsed:
+        # 转换为字符串
+        statement_str = str(statement)# .lower()
+
+        # 将字符串中的特殊语法置空
+        statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)
+
+        # 查找匹配的表名
+        matches = re.findall(table_name_pattern, statement_str, re.IGNORECASE)
+
+        for match in matches:
+            # 提取非空的表名部分
+            for name in match:
+                if name:
+                    # 对于可能包含命名空间的情况，只保留最后一部分作为表名
+                    table_name = name.split('.')[-1]
+                    # 去除表名中的特殊符号
+                    table_name = re.sub(r'("|`|\'|;)', '', table_name)
+                    table_names.add(table_name)
+
+        # 处理特殊的WITH语句
+        if 'with' in statement_str:
+            remove_with_name = extract_temp_tables(statement_str)
+    # 移除多余的表名
+    if remove_with_name:
+        table_names = list(set(table_names) - set(remove_with_name))
+
+    return table_names
+
+
+

lidb-1.0.0.dist-info/METADATA

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: lidb
+Version: 1.0.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: polars>=1.31.0
+Requires-Dist: sqlparse>=0.5.3

lidb-1.0.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+lidb/__init__.py,sha256=tI-DSNlrTaj3X6WmvGIIa8KTUlsI5oIPxIagOLDcbqk,292
+lidb/database.py,sha256=OktJZCPVIaBUYENGMeWNB8NPOy1R01kwWtkfuUzP07E,3147
+lidb/init.py,sha256=HPMqUVF8FWa3i9NTu7x3fQaSAOJ08r9JjelHrUgl0tA,1681
+lidb/parse.py,sha256=N1BBZoUhvLj58biZfEhFs4cGsqaZqsanx27bAp_P7Oo,2236
+lidb-1.0.0.dist-info/METADATA,sha256=094xQkA7LNLqHXvyU9sE5TgdU5Au4o2hXSfZ6elUSkU,234
+lidb-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lidb-1.0.0.dist-info/top_level.txt,sha256=NgXJNwt6ld6oLXtW1vOPaEh-VO5R0JEX_KmGIJR4ueE,5
+lidb-1.0.0.dist-info/RECORD,,

lidb-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+lidb

lidb-0.1.0.dist-info/METADATA

@@ -1,6 +0,0 @@
-Metadata-Version: 2.4
-Name: lidb
-Version: 0.1.0
-Summary: Add your description here
-Requires-Python: >=3.13
-Description-Content-Type: text/markdown

lidb-0.1.0.dist-info/RECORD

@@ -1,5 +0,0 @@
-main.py,sha256=uFJbm6oYi9ntspj1wxHb53SGZo54jI_23GvhG7KS9hY,82
-lidb-0.1.0.dist-info/METADATA,sha256=yn8hppY_738XRXsi4XjKEVXdwjnwvDEzpwpNhPunoQA,147
-lidb-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lidb-0.1.0.dist-info/top_level.txt,sha256=ZAMgPdWghn6xTRBO6Kc3ML1y3ZrZLnjZlqbboKXc_AE,5
-lidb-0.1.0.dist-info/RECORD,,

lidb-0.1.0.dist-info/top_level.txt

@@ -1 +0,0 @@
-main

main.py

@@ -1,6 +0,0 @@
-def main():
-    print("Hello from lidb!")
-
-
-if __name__ == "__main__":
-    main()