lidb 1.0.10__py3-none-any.whl → 1.1.0__py3-none-any.whl

lidb/__init__.py

@@ -19,4 +19,4 @@ from .database import (
 
 from .parse import parse_hive_partition_structure
 
-__version__ = "1.0.10"
+__version__ = "1.1.0"

lidb/parse.py

@@ -11,7 +11,10 @@ import re
 from pathlib import Path
 from typing import Dict, Set
 from urllib.parse import unquote
+from collections import defaultdict
 
+import polars
+import polars as pl
 import sqlparse
 
 
@@ -69,7 +72,7 @@ def extract_table_names_from_sql(sql_query):
     return table_names
 
 
-def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*.parquet") -> Dict[str, Set[str]]:
+def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*.parquet") -> polars.DataFrame:
     """
     通用Hive分区结构解析器
 
@@ -78,25 +81,30 @@ def parse_hive_partition_structure(root_path: Path | str, file_pattern: str = "*
         file_pattern: 文件匹配模式 (默认 "*.parquet")
 
     Returns:
-        Dict[str, Set[str]]: 分区键到值集合的映射
+        polars.DataFrame
     """
     if isinstance(root_path, str):
         root_path = Path(root_path)
-    partitions: Dict[str, Set[str]] = {}
 
-    # 查找所有匹配的文件
+    partition_combinations = set()
+
     for file_path in root_path.rglob(file_pattern):
-        # 获取相对于根路径的路径
         relative_path = file_path.relative_to(root_path)
 
-        # 解析路径中的分区信息
-        for part in relative_path.parts[:-1]:  # 排除文件名部分
+        # 收集分区信息
+        partition_dict = {}
+        for part in relative_path.parts[:-1]:  # 排除文件名
             if '=' in part:
                 key, value = part.split('=', 1)
                 value = unquote(value)
 
-                if key not in partitions:
-                    partitions[key] = set()
-                partitions[key].add(value)
+                partition_dict[key] = value
+
+        # 记录分区组合
+        combination = tuple(sorted(partition_dict.items()))
+        partition_combinations.add(combination)
+
+    # 转换为普通dict
+    res = [dict(combo) for combo in partition_combinations]
 
-    return partitions
+    return pl.DataFrame(res)

{lidb-1.0.10.dist-info → lidb-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lidb
-Version: 1.0.10
+Version: 1.1.0
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: dynaconf>=3.2.11

lidb-1.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+lidb/__init__.py,sha256=dCfYBd1A4-lAcGv-9kVjLxmWva2DnH8ev0s-3nJk3vQ,313
+lidb/database.py,sha256=U1h80jgmkRfgQW8sdhb7B3ISvQJUvPefLurWMsbzqa0,5372
+lidb/init.py,sha256=jLHpeL5mIM4YjdMYAndZlDilMiKXJMr_51Ke3ZSJWCM,1170
+lidb/parse.py,sha256=5lrYtT_XyRfZYX_AwfmhDYZywPBkAbzIVwJk_7l5Nrw,3470
+lidb-1.1.0.dist-info/METADATA,sha256=jkGViIS9bnzJtVGF0vzgJm4W4_MaMIb6h00rYuyezN8,303
+lidb-1.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lidb-1.1.0.dist-info/top_level.txt,sha256=NgXJNwt6ld6oLXtW1vOPaEh-VO5R0JEX_KmGIJR4ueE,5
+lidb-1.1.0.dist-info/RECORD,,

lidb-1.0.10.dist-info/RECORD

@@ -1,8 +0,0 @@
-lidb/__init__.py,sha256=MG3U8f-IJ6X1EkgdHjDNqUpbJsdqX3C4mfSze0V_9xY,314
-lidb/database.py,sha256=U1h80jgmkRfgQW8sdhb7B3ISvQJUvPefLurWMsbzqa0,5372
-lidb/init.py,sha256=jLHpeL5mIM4YjdMYAndZlDilMiKXJMr_51Ke3ZSJWCM,1170
-lidb/parse.py,sha256=8NpWRjoVc45QDgNUoEcsAQQ-AcciBjpstar5kbSc_Lc,3364
-lidb-1.0.10.dist-info/METADATA,sha256=-5CuLQhl2nsgANTWYG25IsWpJlQ9tjM00NasRe1rs04,304
-lidb-1.0.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lidb-1.0.10.dist-info/top_level.txt,sha256=NgXJNwt6ld6oLXtW1vOPaEh-VO5R0JEX_KmGIJR4ueE,5
-lidb-1.0.10.dist-info/RECORD,,