lidb 1.0.10__tar.gz → 1.1.0__tar.gz

{lidb-1.0.10 → lidb-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.10
+Version: 1.1.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

{lidb-1.0.10 → lidb-1.1.0}/lidb/__init__.py

@@ -19,4 +19,4 @@ from .database import (
 
 from .parse import parse_hive_partition_structure
 
-__version__ = "1.0.10"
+__version__ = "1.1.0"

{lidb-1.0.10 → lidb-1.1.0}/lidb/parse.py

@@ -11,7 +11,10 @@ import re
 from pathlib import Path
 from typing import Dict, Set
 from urllib.parse import unquote
+from collections import defaultdict
 
+import polars
+import polars as pl
 import sqlparse
 
 
@@ -69,7 +72,7 @@ def extract_table_names_from_sql(sql_query):
     return table_names
 
 
-def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*.parquet") -> Dict[str, Set[str]]:
+def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*.parquet") -> polars.DataFrame:
     """
     通用Hive分区结构解析器
 
@@ -78,25 +81,30 @@ def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*
         file_pattern: 文件匹配模式 (默认 "*.parquet")
 
     Returns:
-        Dict[str, Set[str]]: 分区键到值集合的映射
+        polars.DataFrame
     """
     if isinstance(root_path, str):
         root_path = Path(root_path)
-    partitions: Dict[str, Set[str]] = {}
 
-    # 查找所有匹配的文件
+    partition_combinations = set()
+
     for file_path in root_path.rglob(file_pattern):
-        # 获取相对于根路径的路径
         relative_path = file_path.relative_to(root_path)
 
-        # 解析路径中的分区信息
-        for part in relative_path.parts[:-1]:  # 排除文件名部分
+        # 收集分区信息
+        partition_dict = {}
+        for part in relative_path.parts[:-1]:  # 排除文件名
             if '=' in part:
                 key, value = part.split('=', 1)
                 value = unquote(value)
 
-                if key not in partitions:
-                    partitions[key] = set()
-                partitions[key].add(value)
+                partition_dict[key] = value
+
+        # 记录分区组合
+        combination = tuple(sorted(partition_dict.items()))
+        partition_combinations.add(combination)
+
+    # 转换为普通dict
+    res = [dict(combo) for combo in partition_combinations]
 
-    return partitions
+    return pl.DataFrame(res)

{lidb-1.0.10 → lidb-1.1.0}/lidb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.10
+Version: 1.1.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

{lidb-1.0.10 → lidb-1.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lidb"
-version = "1.0.10"
+version = "1.1.0"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"

{lidb-1.0.10 → lidb-1.1.0}/tests/test_parse.py

@@ -12,7 +12,7 @@ def test_parse_hive_partition_structure():
     root_path = lidb.tb_path("mc")
     file_pattern = "*.parquet"
     result = lidb.parse.parse_hive_partition_structure(root_path, file_pattern)
-    logger.info(result)
+    logger.info(result["freq"].unique())
 
 if __name__ == '__main__':
     test_parse_hive_partition_structure()