lidb 0.1.0__tar.gz → 1.0.0__tar.gz

lidb-1.0.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: lidb
+Version: 1.0.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: polars>=1.31.0
+Requires-Dist: sqlparse>=0.5.3

lidb-1.0.0/lidb/__init__.py

@@ -0,0 +1,21 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/7/17 14:09
+# Description:
+
+from .init import (
+    NAME,
+    DB_PATH,
+    CONFIG_PATH,
+    LOGS_PATH,
+    get_settings,
+)
+
+from .database import (
+    sql,
+    put,
+    has,
+    tb_path,
+)
+
+__version__ = "1.0.0"

lidb-1.0.0/lidb/database.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Copyright (c) 2025 ZhangYundi
+Licensed under the MIT License.
+Created on 2024/7/1 09:44
+Email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import re
+from pathlib import Path
+from .parse import extract_table_names_from_sql
+from .init import DB_PATH, logger
+
+# ======================== 本地数据库 catdb ========================
+def tb_path(tb_name: str) -> Path:
+    """
+    返回指定表名 完整的本地路径
+    Parameters
+    ----------
+    tb_name: str
+       表名，路径写法: a/b/c
+    Returns
+    -------
+    pathlib.Path
+        full_abs_path: pathlib.Path
+        完整的本地绝对路径 $DB_PATH/a/b/c
+    """
+    return Path(DB_PATH, tb_name)
+
+
+def put(df, tb_name: str, partitions: list[str] | None = None, abs_path: bool = False):
+    """
+    将一个DataFrame写入到指定名称的表格目录中，支持分区存储。
+
+    该函数负责将给定的DataFrame（df）根据提供的表名（tb_name）写入到本地文件系统中。
+    如果指定了分区（partitions），则会按照这些分区列将数据分割存储。此外，可以通过abs_path参数
+    指定tb_name是否为绝对路径。如果目录不存在，会自动创建目录。
+
+    Parameters
+    ----------
+    df: polars.DataFrame
+    tb_name: str
+        表的名称，用于确定存储数据的目录
+    partitions: list[str] | None
+        指定用于分区的列名列表。如果未提供，则不进行分区。
+    abs_path: bool
+        tb_name是否应被视为绝对路径。默认为False。
+
+    Returns
+    -------
+
+    """
+    if df is None:
+        logger.warning(f"put failed: input data is None.")
+        return
+    if df.is_empty():
+        logger.warning(f"put failed: input data is empty.")
+        return
+    if not abs_path:
+        tbpath = tb_path(tb_name)
+    else:
+        tbpath = Path(tb_name)
+    if not tbpath.exists():
+        tbpath.mkdir(parents=True, exist_ok=True)
+    if partitions is not None:
+        df.write_parquet(tbpath, partition_by=partitions)
+    else:
+        df.write_parquet(tbpath / "data.parquet")
+
+def has(tb_name: str) -> bool:
+    """
+    判定给定的表名是否存在
+    Parameters
+    ----------
+    tb_name: str
+
+    Returns
+    -------
+
+    """
+    return tb_path(tb_name).exists()
+
+def sql(query: str, abs_path: bool = False):
+    """
+    sql 查询，从本地paquet文件中查询数据
+
+    Parameters
+    ----------
+    query: str
+        sql查询语句
+    abs_path: bool
+        是否使用绝对路径作为表路径。默认为False
+    Returns
+    -------
+
+    """
+    import polars as pl
+
+    tbs = extract_table_names_from_sql(query)
+    convertor = dict()
+    for tb in tbs:
+        if not abs_path:
+            db_path = tb_path(tb)
+        else:
+            db_path = tb
+        format_tb = f"read_parquet('{db_path}/**/*.parquet')"
+        convertor[tb] = format_tb
+    pattern = re.compile("|".join(re.escape(k) for k in convertor.keys()))
+    new_query = pattern.sub(lambda m: convertor[m.group(0)], query)
+    return pl.sql(new_query)
+

lidb-1.0.0/lidb/init.py

@@ -0,0 +1,65 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/7/17 14:40
+# Description:
+
+from pathlib import Path
+from dynaconf import Dynaconf
+from loguru import logger
+import sys
+
+
+USERHOME = Path("~").expanduser() # 用户家目录
+NAME = "lidb"
+DB_PATH = USERHOME / NAME
+CONFIG_PATH = DB_PATH / "conf" / "settings.toml"
+LOGS_PATH = DB_PATH / "logs"
+
+logger.remove()
+
+console_format = (
+    "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
+    "<level>{level}</level> | "
+    f"<cyan>{NAME}</cyan>:<cyan>{{function}}</cyan>:<cyan>{{line}}</cyan> - "
+    "<level>{message}</level>"
+)
+
+logger.add(
+    sys.stderr,
+    format=console_format,
+    level="TRACE"
+)
+
+logger.add(
+    LOGS_PATH / "{time:YYYYMMDD}.log",
+    retention="10 days",
+    format=f"{{time:YYYY-MM-DD HH:mm:ss}} | {{level}} | {NAME}:{{function}}:{{line}} - {{message}}",
+    level="TRACE"
+)
+
+if not CONFIG_PATH.exists():
+    try:
+        CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        logger.error(f"Create settings file failed: {e}")
+    with open(CONFIG_PATH, "w") as f:
+        template_content = f"""[global]
+path="{DB_PATH}" 
+"""
+    with open(CONFIG_PATH, "w") as f:
+        f.write(template_content)
+    logger.info(f"Settings file created: {CONFIG_PATH}")
+
+def get_settings():
+    try:
+        return Dynaconf(settings_files=[CONFIG_PATH])
+    except Exception as e:
+        logger.error(f"Read settings file failed: {e}")
+        return {}
+
+# 读取配置文件覆盖
+_settiings = get_settings()
+if _settiings is not None:
+    setting_db_path = _settiings.get(f"global.path", "")
+    if setting_db_path:
+        DB_PATH = Path(setting_db_path)

lidb-1.0.0/lidb/parse.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+---------------------------------------------
+Copyright (c) 2025 ZhangYundi
+Licensed under the MIT License.
+Created on 2024/11/6 下午7:25
+Email: yundi.xxii@outlook.com
+---------------------------------------------
+"""
+import sqlparse
+import re
+
+def format_sql(sql_content):
+    """将sql语句进行规范化，并去除sql中的注释，输入和输出均为字符串"""
+    parse_str = sqlparse.format(sql_content, reindent=True, strip_comments=True)
+    return parse_str
+
+def extract_temp_tables(with_clause):
+    """从WITH子句中提取临时表名，输出为列表"""
+    temp_tables = re.findall(r'\b(\w+)\s*as\s*\(', with_clause, re.IGNORECASE)
+    return temp_tables
+
+def extract_table_names_from_sql(sql_query):
+    """从sql中提取对应的表名称，输出为列表"""
+    table_names = set()
+    # 解析SQL语句
+    parsed = sqlparse.parse(sql_query)
+    # 正则表达式模式，用于匹配表名
+    table_name_pattern = r'\bFROM\s+([^\s\(\)\,]+)|\bJOIN\s+([^\s\(\)\,]+)'
+
+    # 用于存储WITH子句中的临时表名
+    remove_with_name = []
+
+    # 遍历解析后的语句块
+    for statement in parsed:
+        # 转换为字符串
+        statement_str = str(statement)# .lower()
+
+        # 将字符串中的特殊语法置空
+        statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)
+
+        # 查找匹配的表名
+        matches = re.findall(table_name_pattern, statement_str, re.IGNORECASE)
+
+        for match in matches:
+            # 提取非空的表名部分
+            for name in match:
+                if name:
+                    # 对于可能包含命名空间的情况，只保留最后一部分作为表名
+                    table_name = name.split('.')[-1]
+                    # 去除表名中的特殊符号
+                    table_name = re.sub(r'("|`|\'|;)', '', table_name)
+                    table_names.add(table_name)
+
+        # 处理特殊的WITH语句
+        if 'with' in statement_str:
+            remove_with_name = extract_temp_tables(statement_str)
+    # 移除多余的表名
+    if remove_with_name:
+        table_names = list(set(table_names) - set(remove_with_name))
+
+    return table_names
+
+
+

lidb-1.0.0/lidb.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: lidb
+Version: 1.0.0
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: polars>=1.31.0
+Requires-Dist: sqlparse>=0.5.3

lidb-1.0.0/lidb.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+README.md
+main.py
+pyproject.toml
+lidb/__init__.py
+lidb/database.py
+lidb/init.py
+lidb/parse.py
+lidb.egg-info/PKG-INFO
+lidb.egg-info/SOURCES.txt
+lidb.egg-info/dependency_links.txt
+lidb.egg-info/requires.txt
+lidb.egg-info/top_level.txt
+tests/test_conf.py

lidb-1.0.0/lidb.egg-info/requires.txt

@@ -0,0 +1,4 @@
+dynaconf>=3.2.11
+loguru>=0.7.3
+polars>=1.31.0
+sqlparse>=0.5.3

lidb-1.0.0/lidb.egg-info/top_level.txt

@@ -0,0 +1 @@
+lidb

lidb-1.0.0/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "lidb"
+version = "1.0.0"
+description = ""
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "dynaconf>=3.2.11",
+    "loguru>=0.7.3",
+    "polars>=1.31.0",
+    "sqlparse>=0.5.3",
+]
+
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["lidb", "lidb.*"]
+

lidb-1.0.0/tests/test_conf.py

@@ -0,0 +1,5 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/7/17 14:15
+# Description:
+

lidb-0.1.0/PKG-INFO

@@ -1,6 +0,0 @@
-Metadata-Version: 2.4
-Name: lidb
-Version: 0.1.0
-Summary: Add your description here
-Requires-Python: >=3.13
-Description-Content-Type: text/markdown

lidb-0.1.0/lidb.egg-info/PKG-INFO

@@ -1,6 +0,0 @@
-Metadata-Version: 2.4
-Name: lidb
-Version: 0.1.0
-Summary: Add your description here
-Requires-Python: >=3.13
-Description-Content-Type: text/markdown

lidb-0.1.0/lidb.egg-info/SOURCES.txt

@@ -1,7 +0,0 @@
-README.md
-main.py
-pyproject.toml
-lidb.egg-info/PKG-INFO
-lidb.egg-info/SOURCES.txt
-lidb.egg-info/dependency_links.txt
-lidb.egg-info/top_level.txt

lidb-0.1.0/lidb.egg-info/top_level.txt

@@ -1 +0,0 @@
-main

lidb-0.1.0/pyproject.toml

@@ -1,7 +0,0 @@
-[project]
-name = "lidb"
-version = "0.1.0"
-description = "Add your description here"
-readme = "README.md"
-requires-python = ">=3.13"
-dependencies = []