iparq 0.1.5__py3-none-any.whl → 0.2.0__py3-none-any.whl

iparq/py.typed

@@ -0,0 +1 @@
+# This empty file marks the package as typed for mypy

iparq/source.py

@@ -1,8 +1,11 @@
+from typing import List, Optional
+
 import pyarrow.parquet as pq
 import typer
 from pydantic import BaseModel
 from rich import print
 from rich.console import Console
+from rich.table import Table
 
 app = typer.Typer()
 console = Console()
@@ -29,6 +32,36 @@ class ParquetMetaModel(BaseModel):
     serialized_size: int
 
 
+class ColumnInfo(BaseModel):
+    """
+    ColumnInfo is a data model representing information about a column in a Parquet file.
+
+    Attributes:
+        row_group (int): The row group index.
+        column_name (str): The name of the column.
+        column_index (int): The index of the column.
+        compression_type (str): The compression type used for the column.
+        has_bloom_filter (bool): Whether the column has a bloom filter.
+    """
+
+    row_group: int
+    column_name: str
+    column_index: int
+    compression_type: str
+    has_bloom_filter: Optional[bool] = False
+
+
+class ParquetColumnInfo(BaseModel):
+    """
+    ParquetColumnInfo is a data model representing information about all columns in a Parquet file.
+
+    Attributes:
+        columns (List[ColumnInfo]): List of column information.
+    """
+
+    columns: List[ColumnInfo] = []
+
+
 def read_parquet_metadata(filename: str):
     """
     Reads the metadata of a Parquet file and extracts the compression codecs used.
@@ -94,30 +127,104 @@ def print_parquet_metadata(parquet_metadata):
         pass
 
 
-def print_compression_types(parquet_metadata) -> None:
+def print_compression_types(parquet_metadata, column_info: ParquetColumnInfo) -> None:
     """
-    Prints the compression type for each column in each row group of the Parquet file.
+    Collects compression type information for each column and adds it to the column_info model.
+
+    Args:
+        parquet_metadata: The Parquet file metadata.
+        column_info: The ParquetColumnInfo model to update.
     """
     try:
         num_row_groups = parquet_metadata.num_row_groups
         num_columns = parquet_metadata.num_columns
-        console.print("[bold underline]Column Compression Info:[/bold underline]")
+
         for i in range(num_row_groups):
-            console.print(f"[bold]Row Group {i}:[/bold]")
+            row_group = parquet_metadata.row_group(i)
             for j in range(num_columns):
-                column_chunk = parquet_metadata.row_group(i).column(j)
+                column_chunk = row_group.column(j)
                 compression = column_chunk.compression
-                column_name = parquet_metadata.schema.column(j).name
-                console.print(
-                    f"  Column '{column_name}' (Index {j}): [italic]{compression}[/italic]"
+                column_name = parquet_metadata.schema.names[j]
+
+                # Create or update column info
+                column_info.columns.append(
+                    ColumnInfo(
+                        row_group=i,
+                        column_name=column_name,
+                        column_index=j,
+                        compression_type=compression,
+                    )
                 )
     except Exception as e:
         console.print(
-            f"Error while printing compression types: {e}",
+            f"Error while collecting compression types: {e}",
             style="blink bold red underline on white",
         )
-    finally:
-        pass
+
+
+def print_bloom_filter_info(parquet_metadata, column_info: ParquetColumnInfo) -> None:
+    """
+    Updates the column_info model with bloom filter information.
+
+    Args:
+        parquet_metadata: The Parquet file metadata.
+        column_info: The ParquetColumnInfo model to update.
+    """
+    try:
+        num_row_groups = parquet_metadata.num_row_groups
+        num_columns = parquet_metadata.num_columns
+
+        for i in range(num_row_groups):
+            row_group = parquet_metadata.row_group(i)
+
+            for j in range(num_columns):
+                column_chunk = row_group.column(j)
+
+                # Find the corresponding column in our model
+                for col in column_info.columns:
+                    if col.row_group == i and col.column_index == j:
+                        # Check if this column has bloom filters
+                        has_bloom_filter = (
+                            hasattr(column_chunk, "is_stats_set")
+                            and column_chunk.is_stats_set
+                        )
+                        col.has_bloom_filter = has_bloom_filter
+                        break
+    except Exception as e:
+        console.print(
+            f"Error while collecting bloom filter information: {e}",
+            style="blink bold red underline on white",
+        )
+
+
+def print_column_info_table(column_info: ParquetColumnInfo) -> None:
+    """
+    Prints the column information using a Rich table.
+
+    Args:
+        column_info: The ParquetColumnInfo model to display.
+    """
+    table = Table(title="Parquet Column Information")
+
+    # Add table columns
+    table.add_column("Row Group", justify="center", style="cyan")
+    table.add_column("Column Name", style="green")
+    table.add_column("Index", justify="center")
+    table.add_column("Compression", style="magenta")
+    table.add_column("Bloom Filter", justify="center")
+
+    # Add rows to the table
+    for col in column_info.columns:
+        table.add_row(
+            str(col.row_group),
+            col.column_name,
+            str(col.column_index),
+            col.compression_type,
+            "✅" if col.has_bloom_filter else "❌",
+        )
+
+    # Print the table
+    console.print(table)
 
 
 @app.command()
@@ -134,7 +241,17 @@ def main(filename: str):
     (parquet_metadata, compression) = read_parquet_metadata(filename)
 
     print_parquet_metadata(parquet_metadata)
-    print_compression_types(parquet_metadata)
+
+    # Create a model to store column information
+    column_info = ParquetColumnInfo()
+
+    # Collect information
+    print_compression_types(parquet_metadata, column_info)
+    print_bloom_filter_info(parquet_metadata, column_info)
+
+    # Print the information as a table
+    print_column_info_table(column_info)
+
     print(f"Compression codecs: {compression}")
 
 

{iparq-0.1.5.dist-info → iparq-0.2.0.dist-info}/METADATA

@@ -1,13 +1,14 @@
 Metadata-Version: 2.4
 Name: iparq
-Version: 0.1.5
-Summary: Display version and compression information about a parquet file
+Version: 0.2.0
+Summary: Display version compression and bloom filter information about a parquet file
 Author-email: MiguelElGallo <miguel.zurcher@gmail.com>
 License-File: LICENSE
 Requires-Python: >=3.9
-Requires-Dist: pyarrow>=19.0.0
-Requires-Dist: pydantic>=2.10.6
-Requires-Dist: typer>=0.15.1
+Requires-Dist: pyarrow
+Requires-Dist: pydantic
+Requires-Dist: rich
+Requires-Dist: typer[all]
 Provides-Extra: checks
 Requires-Dist: mypy>=1.14.1; extra == 'checks'
 Requires-Dist: ruff>=0.9.3; extra == 'checks'
@@ -26,8 +27,24 @@ Description-Content-Type: text/markdown
 ![alt text](media/iparq.png)
 After reading [this blog](https://duckdb.org/2025/01/22/parquet-encodings.html), I began to wonder which Parquet version and compression methods the everyday tools we rely on actually use, only to find that there’s no straightforward way to determine this. That curiosity and the difficulty of quickly discovering such details motivated me to create iparq (Information Parquet). My goal with iparq is to help users easily identify the specifics of the Parquet files generated by different engines, making it clear which features—like newer encodings or certain compression algorithms—the creator of the parquet is using.
 
+***New*** Bloom filters information: Displays if there are bloom filters.
+Read more about bloom filters in this [great article](https://duckdb.org/2025/03/07/parquet-bloom-filters-in-duckdb.html).
+
+
 ## Installation
 
+### Zero installation - Recommended
+
+1) Make sure to have Astral’s UV installed by following the steps here:  
+
+    <https://docs.astral.sh/uv/getting-started/installation/>
+
+2) Execute the following command:
+
+    ```sh
+    uvx iparq yourparquet.parquet
+    ```
+
 ### Using pip
 
 1) Install the package using pip:
@@ -80,7 +97,63 @@ iparq <filename>
 
 Replace `<filename>` with the path to your .parquet file. The utility will read the metadata of the file and print the compression codecs used in the parquet file.
 
-## Example output
+## Example ouput - Bloom Filters
+
+```log
+ParquetMetaModel(
+    created_by='DuckDB version v1.2.1 (build 8e52ec4395)',
+    num_columns=1,
+    num_rows=100000000,
+    num_row_groups=10,
+    format_version='1.0',
+    serialized_size=1196
+)
+Column Compression Info:
+Row Group 0:
+  Column 'r' (Index 0): SNAPPY
+Row Group 1:
+  Column 'r' (Index 0): SNAPPY
+Row Group 2:
+  Column 'r' (Index 0): SNAPPY
+Row Group 3:
+  Column 'r' (Index 0): SNAPPY
+Row Group 4:
+  Column 'r' (Index 0): SNAPPY
+Row Group 5:
+  Column 'r' (Index 0): SNAPPY
+Row Group 6:
+  Column 'r' (Index 0): SNAPPY
+Row Group 7:
+  Column 'r' (Index 0): SNAPPY
+Row Group 8:
+  Column 'r' (Index 0): SNAPPY
+Row Group 9:
+  Column 'r' (Index 0): SNAPPY
+Bloom Filter Info:
+Row Group 0:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 1:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 2:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 3:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 4:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 5:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 6:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 7:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 8:
+  Column 'r' (Index 0): Has bloom filter
+Row Group 9:
+  Column 'r' (Index 0): Has bloom filter
+Compression codecs: {'SNAPPY'}
+```
+
+## Example output 
 
 ```log
 ParquetMetaModel(

iparq-0.2.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+iparq/__init__.py,sha256=sXLh7g3KC4QCFxcZGBTpG2scR7hmmBsMjq6LqRptkRg,22
+iparq/py.typed,sha256=bOHAx3O6ryp453lBypAaF78WipxsJDO9hH0PZFTAWYs,54
+iparq/source.py,sha256=qyBNysMLX0FkjZVw5dPSRuhswX3GuRXvM79v7g7emWM,8482
+iparq-0.2.0.dist-info/METADATA,sha256=TwQ7wiLtdEwh3NwY2talWs4BM-oeJw55LCzgMnlr5Gc,7163
+iparq-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+iparq-0.2.0.dist-info/entry_points.txt,sha256=vrE2lwvuheySWTOJdr_gh9AT47ck02WCHo0muRq5HS8,43
+iparq-0.2.0.dist-info/licenses/LICENSE,sha256=apqXCIYD_rrtbJVE-Ex1-1X7N0cBwZTOm4KL3TEFmYA,1067
+iparq-0.2.0.dist-info/RECORD,,

iparq-0.1.5.dist-info/RECORD

@@ -1,7 +0,0 @@
-iparq/__init__.py,sha256=sXLh7g3KC4QCFxcZGBTpG2scR7hmmBsMjq6LqRptkRg,22
-iparq/source.py,sha256=7ocTpA7j5C-oSyLkMPhDifpH3cPhqyK3LBu0CjjG83s,4851
-iparq-0.1.5.dist-info/METADATA,sha256=7kLNc40ROyYot5N37GXjSErzpp5WPDWa9_Y2BQpmr6o,5387
-iparq-0.1.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-iparq-0.1.5.dist-info/entry_points.txt,sha256=vrE2lwvuheySWTOJdr_gh9AT47ck02WCHo0muRq5HS8,43
-iparq-0.1.5.dist-info/licenses/LICENSE,sha256=apqXCIYD_rrtbJVE-Ex1-1X7N0cBwZTOm4KL3TEFmYA,1067
-iparq-0.1.5.dist-info/RECORD,,