iparq 0.3.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

iparq/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.1"

iparq/source.py

@@ -54,6 +54,9 @@ class ColumnInfo(BaseModel):
         column_index (int): The index of the column.
         compression_type (str): The compression type used for the column.
         has_bloom_filter (bool): Whether the column has a bloom filter.
+        has_min_max (bool): Whether min/max statistics are available.
+        min_value (Optional[str]): The minimum value in the column (as string for display).
+        max_value (Optional[str]): The maximum value in the column (as string for display).
     """
 
     row_group: int
@@ -61,6 +64,9 @@ class ColumnInfo(BaseModel):
     column_index: int
     compression_type: str
     has_bloom_filter: Optional[bool] = False
+    has_min_max: Optional[bool] = False
+    min_value: Optional[str] = None
+    max_value: Optional[str] = None
 
 
 class ParquetColumnInfo(BaseModel):
@@ -203,6 +209,59 @@ def print_bloom_filter_info(parquet_metadata, column_info: ParquetColumnInfo) ->
         )
 
 
+def print_min_max_statistics(parquet_metadata, column_info: ParquetColumnInfo) -> None:
+    """
+    Updates the column_info model with min/max statistics information.
+
+    Args:
+        parquet_metadata: The Parquet file metadata.
+        column_info: The ParquetColumnInfo model to update.
+    """
+    try:
+        num_row_groups = parquet_metadata.num_row_groups
+        num_columns = parquet_metadata.num_columns
+
+        for i in range(num_row_groups):
+            row_group = parquet_metadata.row_group(i)
+
+            for j in range(num_columns):
+                column_chunk = row_group.column(j)
+
+                # Find the corresponding column in our model
+                for col in column_info.columns:
+                    if col.row_group == i and col.column_index == j:
+                        # Check if this column has statistics
+                        if column_chunk.is_stats_set:
+                            stats = column_chunk.statistics
+                            col.has_min_max = stats.has_min_max
+
+                            if stats.has_min_max:
+                                # Convert values to string for display, handling potential None values
+                                try:
+                                    col.min_value = (
+                                        str(stats.min)
+                                        if stats.min is not None
+                                        else "null"
+                                    )
+                                    col.max_value = (
+                                        str(stats.max)
+                                        if stats.max is not None
+                                        else "null"
+                                    )
+                                except Exception:
+                                    # Fallback for complex types that might not stringify well
+                                    col.min_value = "<unable to display>"
+                                    col.max_value = "<unable to display>"
+                        else:
+                            col.has_min_max = False
+                        break
+    except Exception as e:
+        console.print(
+            f"Error while collecting min/max statistics: {e}",
+            style="blink bold red underline on white",
+        )
+
+
 def print_column_info_table(column_info: ParquetColumnInfo) -> None:
     """
     Prints the column information using a Rich table.
@@ -218,15 +277,27 @@ def print_column_info_table(column_info: ParquetColumnInfo) -> None:
     table.add_column("Index", justify="center")
     table.add_column("Compression", style="magenta")
     table.add_column("Bloom Filter", justify="center")
+    table.add_column("Min Value", style="yellow")
+    table.add_column("Max Value", style="yellow")
 
     # Add rows to the table
     for col in column_info.columns:
+        # Format min/max values for display
+        min_display = (
+            col.min_value if col.has_min_max and col.min_value is not None else "N/A"
+        )
+        max_display = (
+            col.max_value if col.has_min_max and col.max_value is not None else "N/A"
+        )
+
         table.add_row(
             str(col.row_group),
             col.column_name,
             str(col.column_index),
             col.compression_type,
             "✅" if col.has_bloom_filter else "❌",
+            min_display,
+            max_display,
         )
 
     # Print the table
@@ -290,6 +361,7 @@ def inspect_single_file(
     # Collect information
     print_compression_types(parquet_metadata, column_info)
     print_bloom_filter_info(parquet_metadata, column_info)
+    print_min_max_statistics(parquet_metadata, column_info)
 
     # Filter columns if requested
     if column_filter:

{iparq-0.3.0.dist-info → iparq-0.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iparq
-Version: 0.3.0
+Version: 0.4.1
 Summary: Display version compression and bloom filter information about a parquet file
 Author-email: MiguelElGallo <miguel.zurcher@gmail.com>
 License-File: LICENSE
@@ -13,6 +13,7 @@ Provides-Extra: checks
 Requires-Dist: mypy>=1.14.1; extra == 'checks'
 Requires-Dist: ruff>=0.9.3; extra == 'checks'
 Provides-Extra: test
+Requires-Dist: pytest-cov>=4.0.0; extra == 'test'
 Requires-Dist: pytest>=7.0; extra == 'test'
 Description-Content-Type: text/markdown
 
@@ -24,6 +25,8 @@ Description-Content-Type: text/markdown
 
 [![Upload Python Package](https://github.com/MiguelElGallo/iparq/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/python-publish.yml)
 
+[![codecov](https://codecov.io/gh/MiguelElGallo/iparq/branch/main/graph/badge.svg)](https://codecov.io/gh/MiguelElGallo/iparq)
+
 ![alt text](media/iparq.png)
 After reading [this blog](https://duckdb.org/2025/01/22/parquet-encodings.html), I began to wonder which Parquet version and compression methods the everyday tools we rely on actually use, only to find that there's no straightforward way to determine this. That curiosity and the difficulty of quickly discovering such details motivated me to create iparq (Information Parquet). My goal with iparq is to help users easily identify the specifics of the Parquet files generated by different engines, making it clear which features—like newer encodings or certain compression algorithms—the creator of the parquet is using.
 
@@ -134,31 +137,25 @@ iparq inspect important.parquet temp_*.parquet
 
 When inspecting multiple files, each file's results are displayed with a header showing the filename. The utility will read the metadata of each file and print the compression codecs used in the parquet files.
 
-## Example ouput - Bloom Filters
+## Example output - Bloom Filters
 
 ```log
 ParquetMetaModel(
-    created_by='DuckDB version v1.2.1 (build 8e52ec4395)',
-    num_columns=1,
-    num_rows=100000000,
-    num_row_groups=10,
-    format_version='1.0',
-    serialized_size=1196
+    created_by='parquet-cpp-arrow version 14.0.2',
+    num_columns=3,
+    num_rows=3,
+    num_row_groups=1,
+    format_version='2.6',
+    serialized_size=2223
 )
-                   Parquet Column Information                   
-┏━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
-┃ Row Group ┃ Column Name ┃ Index ┃ Compression ┃ Bloom Filter ┃
-┡━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
-│     0     │ r           │   0   │ SNAPPY      │      ✅      │
-│     1     │ r           │   0   │ SNAPPY      │      ✅      │
-│     2     │ r           │   0   │ SNAPPY      │      ✅      │
-│     3     │ r           │   0   │ SNAPPY      │      ✅      │
-│     4     │ r           │   0   │ SNAPPY      │      ✅      │
-│     5     │ r           │   0   │ SNAPPY      │      ✅      │
-│     6     │ r           │   0   │ SNAPPY      │      ✅      │
-│     7     │ r           │   0   │ SNAPPY      │      ✅      │
-│     8     │ r           │   0   │ SNAPPY      │      ✅      │
-│     9     │ r           │   0   │ SNAPPY      │      ✅      │
-└───────────┴─────────────┴───────┴─────────────┴──────────────┘
+                            Parquet Column Information                             
+┏━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━┓
+┃           ┃ Column     ┃       ┃           ┃   Bloom    ┃           ┃           ┃
+┃ Row Group ┃ Name       ┃ Index ┃ Compress… ┃   Filter   ┃ Min Value ┃ Max Value ┃
+┡━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━┩
+│     0     │ one        │   0   │ SNAPPY    │     ✅     │ -1.0      │ 2.5       │
+│     0     │ two        │   1   │ SNAPPY    │     ✅     │ bar       │ foo       │
+│     0     │ three      │   2   │ SNAPPY    │     ✅     │ False     │ True      │
+└───────────┴────────────┴───────┴───────────┴────────────┴───────────┴───────────┘
 Compression codecs: {'SNAPPY'}
 ```

iparq-0.4.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+iparq/__init__.py,sha256=pMtTmSUht-XtbR_7Doz6bsQqopJJd8rZ8I8zy2HwwoA,22
+iparq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+iparq/source.py,sha256=HT76AP19e-TheGB5cHlGqsyJGvdsPVMbAYuSo2xH0Bk,15476
+iparq-0.4.1.dist-info/METADATA,sha256=aKFzoI4pVaAa_D-fW-FeicRHO6PepqOfMuKBXySYS4I,6018
+iparq-0.4.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+iparq-0.4.1.dist-info/entry_points.txt,sha256=vrE2lwvuheySWTOJdr_gh9AT47ck02WCHo0muRq5HS8,43
+iparq-0.4.1.dist-info/licenses/LICENSE,sha256=apqXCIYD_rrtbJVE-Ex1-1X7N0cBwZTOm4KL3TEFmYA,1067
+iparq-0.4.1.dist-info/RECORD,,

iparq-0.3.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-iparq/__init__.py,sha256=VrXpHDu3erkzwl_WXrqINBm9xWkcyUy53IQOj042dOs,22
-iparq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-iparq/source.py,sha256=wMg7c2lw1TSoCd5h3zvMNT_bAqWJMObRq5aPyXrVBik,12288
-iparq-0.3.0.dist-info/METADATA,sha256=mWSvqv57D5PmtzqarAm-vAQoHFDPoiMkHS-nJcnoM_g,6018
-iparq-0.3.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-iparq-0.3.0.dist-info/entry_points.txt,sha256=vrE2lwvuheySWTOJdr_gh9AT47ck02WCHo0muRq5HS8,43
-iparq-0.3.0.dist-info/licenses/LICENSE,sha256=apqXCIYD_rrtbJVE-Ex1-1X7N0cBwZTOm4KL3TEFmYA,1067
-iparq-0.3.0.dist-info/RECORD,,