lidb 1.0.9__tar.gz → 1.1.0__tar.gz

{lidb-1.0.9 → lidb-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.9
+Version: 1.1.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

{lidb-1.0.9 → lidb-1.1.0}/lidb/__init__.py

@@ -17,4 +17,6 @@ from .database import (
     read_ck,
 )
 
-__version__ = "1.0.9"
+from .parse import parse_hive_partition_structure
+
+__version__ = "1.1.0"

{lidb-1.0.9 → lidb-1.1.0}/lidb/parse.py

@@ -7,19 +7,29 @@ Created on 2024/11/6 下午7:25
 Email: yundi.xxii@outlook.com
 ---------------------------------------------
 """
-import sqlparse
 import re
+from pathlib import Path
+from typing import Dict, Set
+from urllib.parse import unquote
+from collections import defaultdict
+
+import polars
+import polars as pl
+import sqlparse
+
 
 def format_sql(sql_content):
     """将sql语句进行规范化，并去除sql中的注释，输入和输出均为字符串"""
     parse_str = sqlparse.format(sql_content, reindent=True, strip_comments=True)
     return parse_str
 
+
 def extract_temp_tables(with_clause):
     """从WITH子句中提取临时表名，输出为列表"""
     temp_tables = re.findall(r'\b(\w+)\s*as\s*\(', with_clause, re.IGNORECASE)
     return temp_tables
 
+
 def extract_table_names_from_sql(sql_query):
     """从sql中提取对应的表名称，输出为列表"""
     table_names = set()
@@ -34,7 +44,7 @@ def extract_table_names_from_sql(sql_query):
     # 遍历解析后的语句块
     for statement in parsed:
         # 转换为字符串
-        statement_str = str(statement)# .lower()
+        statement_str = str(statement)  # .lower()
 
         # 将字符串中的特殊语法置空
         statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)
@@ -62,4 +72,39 @@ def extract_table_names_from_sql(sql_query):
     return table_names
 
 
+def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*.parquet") -> polars.DataFrame:
+    """
+    通用Hive分区结构解析器
+
+    Args:
+        root_path: 根路径 (如 /data)
+        file_pattern: 文件匹配模式 (默认 "*.parquet")
+
+    Returns:
+        polars.DataFrame
+    """
+    if isinstance(root_path, str):
+        root_path = Path(root_path)
+
+    partition_combinations = set()
+
+    for file_path in root_path.rglob(file_pattern):
+        relative_path = file_path.relative_to(root_path)
+
+        # 收集分区信息
+        partition_dict = {}
+        for part in relative_path.parts[:-1]:  # 排除文件名
+            if '=' in part:
+                key, value = part.split('=', 1)
+                value = unquote(value)
+
+                partition_dict[key] = value
+
+        # 记录分区组合
+        combination = tuple(sorted(partition_dict.items()))
+        partition_combinations.add(combination)
+
+    # 转换为普通dict
+    res = [dict(combo) for combo in partition_combinations]
 
+    return pl.DataFrame(res)

{lidb-1.0.9 → lidb-1.1.0}/lidb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.9
+Version: 1.1.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

{lidb-1.0.9 → lidb-1.1.0}/lidb.egg-info/SOURCES.txt

@@ -9,4 +9,5 @@ lidb.egg-info/SOURCES.txt
 lidb.egg-info/dependency_links.txt
 lidb.egg-info/requires.txt
 lidb.egg-info/top_level.txt
-tests/test_conf.py
+tests/test_conf.py
+tests/test_parse.py

{lidb-1.0.9 → lidb-1.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lidb"
-version = "1.0.9"
+version = "1.1.0"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"

lidb-1.1.0/tests/test_parse.py

@@ -0,0 +1,18 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/8/18 11:16
+# Description:
+
+import lidb
+import logair
+
+logger = logair.get_logger("lidb.test")
+
+def test_parse_hive_partition_structure():
+    root_path = lidb.tb_path("mc")
+    file_pattern = "*.parquet"
+    result = lidb.parse.parse_hive_partition_structure(root_path, file_pattern)
+    logger.info(result["freq"].unique())
+
+if __name__ == '__main__':
+    test_parse_hive_partition_structure()