lidb 1.0.9__tar.gz → 1.0.10__tar.gz

{lidb-1.0.9 → lidb-1.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.9
+Version: 1.0.10
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

{lidb-1.0.9 → lidb-1.0.10}/lidb/__init__.py

@@ -17,4 +17,6 @@ from .database import (
     read_ck,
 )
 
-__version__ = "1.0.9"
+from .parse import parse_hive_partition_structure
+
+__version__ = "1.0.10"

{lidb-1.0.9 → lidb-1.0.10}/lidb/parse.py

@@ -7,19 +7,26 @@ Created on 2024/11/6 下午7:25
 Email: yundi.xxii@outlook.com
 ---------------------------------------------
 """
-import sqlparse
 import re
+from pathlib import Path
+from typing import Dict, Set
+from urllib.parse import unquote
+
+import sqlparse
+
 
 def format_sql(sql_content):
     """将sql语句进行规范化，并去除sql中的注释，输入和输出均为字符串"""
     parse_str = sqlparse.format(sql_content, reindent=True, strip_comments=True)
     return parse_str
 
+
 def extract_temp_tables(with_clause):
     """从WITH子句中提取临时表名，输出为列表"""
     temp_tables = re.findall(r'\b(\w+)\s*as\s*\(', with_clause, re.IGNORECASE)
     return temp_tables
 
+
 def extract_table_names_from_sql(sql_query):
     """从sql中提取对应的表名称，输出为列表"""
     table_names = set()
@@ -34,7 +41,7 @@ def extract_table_names_from_sql(sql_query):
     # 遍历解析后的语句块
     for statement in parsed:
         # 转换为字符串
-        statement_str = str(statement)# .lower()
+        statement_str = str(statement)  # .lower()
 
         # 将字符串中的特殊语法置空
         statement_str = re.sub(r'(substring|extract)\s*\(((.|\s)*?)\)', '', statement_str)
@@ -62,4 +69,34 @@ def extract_table_names_from_sql(sql_query):
     return table_names
 
 
+def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*.parquet") -> Dict[str, Set[str]]:
+    """
+    通用Hive分区结构解析器
+
+    Args:
+        root_path: 根路径 (如 /data)
+        file_pattern: 文件匹配模式 (默认 "*.parquet")
+
+    Returns:
+        Dict[str, Set[str]]: 分区键到值集合的映射
+    """
+    if isinstance(root_path, str):
+        root_path = Path(root_path)
+    partitions: Dict[str, Set[str]] = {}
+
+    # 查找所有匹配的文件
+    for file_path in root_path.rglob(file_pattern):
+        # 获取相对于根路径的路径
+        relative_path = file_path.relative_to(root_path)
+
+        # 解析路径中的分区信息
+        for part in relative_path.parts[:-1]:  # 排除文件名部分
+            if '=' in part:
+                key, value = part.split('=', 1)
+                value = unquote(value)
+
+                if key not in partitions:
+                    partitions[key] = set()
+                partitions[key].add(value)
 
+    return partitions

{lidb-1.0.9 → lidb-1.0.10}/lidb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.9
+Version: 1.0.10
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

{lidb-1.0.9 → lidb-1.0.10}/lidb.egg-info/SOURCES.txt

@@ -9,4 +9,5 @@ lidb.egg-info/SOURCES.txt
 lidb.egg-info/dependency_links.txt
 lidb.egg-info/requires.txt
 lidb.egg-info/top_level.txt
-tests/test_conf.py
+tests/test_conf.py
+tests/test_parse.py

{lidb-1.0.9 → lidb-1.0.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lidb"
-version = "1.0.9"
+version = "1.0.10"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"

lidb-1.0.10/tests/test_parse.py

@@ -0,0 +1,18 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/8/18 11:16
+# Description:
+
+import lidb
+import logair
+
+logger = logair.get_logger("lidb.test")
+
+def test_parse_hive_partition_structure():
+    root_path = lidb.tb_path("mc")
+    file_pattern = "*.parquet"
+    result = lidb.parse.parse_hive_partition_structure(root_path, file_pattern)
+    logger.info(result)
+
+if __name__ == '__main__':
+    test_parse_hive_partition_structure()