iparq 0.2.6__tar.gz → 0.4.0__tar.gz

iparq-0.4.0/.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [MiguelElGallo]
+

{iparq-0.2.6 → iparq-0.4.0}/.github/dependabot.yml

@@ -5,7 +5,7 @@
 
 version: 2
 updates:
-  - package-ecosystem: "pip" # See documentation for possible values
+  - package-ecosystem: "uv" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"

iparq-0.4.0/.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,31 @@
+name: "Copilot Setup Steps"
+
+# Allow testing of the setup steps from your repository's "Actions" tab.
+on: workflow_dispatch
+
+jobs:
+  # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot.
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+
+    # Set the permissions to the lowest permissions possible needed for your steps.
+    # Copilot will be given its own token for its operations.
+    permissions:
+      # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete.
+      contents: read
+
+    # You can define any steps you want, and they will run before the agent starts.
+    # If you do not check out your code, Copilot will do this for you.
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install UV (Python package manager)
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          export PATH="$HOME/.cargo/bin:$PATH"
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+          uv --version
+
+      # Note: GitHub MCP server is not publicly available as npm package
+      # Remove this step until the package is officially released

{iparq-0.2.6 → iparq-0.4.0}/.gitignore

@@ -172,3 +172,4 @@ cython_debug/
 .github/.DS_Store
 yellow_tripdata_2024-01.parquet
 filter.parquet
+.DS_Store

{iparq-0.2.6 → iparq-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iparq
-Version: 0.2.6
+Version: 0.4.0
 Summary: Display version compression and bloom filter information about a parquet file
 Author-email: MiguelElGallo <miguel.zurcher@gmail.com>
 License-File: LICENSE
@@ -88,10 +88,10 @@ Read more about bloom filters in this [great article](https://duckdb.org/2025/03
 
 ## Usage
 
-iparq now supports additional options:
+iparq supports inspecting single files, multiple files, and glob patterns:
 
 ```sh
-iparq inspect <filename> [OPTIONS]
+iparq inspect <filename(s)> [OPTIONS]
 ```
 
 Options include:
@@ -100,9 +100,12 @@ Options include:
 - `--metadata-only`, `-m`: Show only file metadata without column details
 - `--column`, `-c`: Filter results to show only a specific column
 
-Examples:
+### Single File Examples:
 
 ```sh
+# Basic inspection
+iparq inspect yourfile.parquet
+
 # Output in JSON format
 iparq inspect yourfile.parquet --format json
 
@@ -113,9 +116,25 @@ iparq inspect yourfile.parquet --metadata-only
 iparq inspect yourfile.parquet --column column_name
 ```
 
-Replace `<filename>` with the path to your .parquet file. The utility will read the metadata of the file and print the compression codecs used in the parquet file.
+### Multiple Files and Glob Patterns:
+
+```sh
+# Inspect multiple specific files
+iparq inspect file1.parquet file2.parquet file3.parquet
+
+# Use glob patterns to inspect all parquet files
+iparq inspect *.parquet
+
+# Use specific patterns
+iparq inspect yellow*.parquet data_*.parquet
+
+# Combine patterns and specific files
+iparq inspect important.parquet temp_*.parquet
+```
+
+When inspecting multiple files, each file's results are displayed with a header showing the filename. The utility will read the metadata of each file and print the compression codecs used in the parquet files.
 
-## Example ouput - Bloom Filters
+## Example output - Bloom Filters
 
 ```log
 ParquetMetaModel(

{iparq-0.2.6 → iparq-0.4.0}/README.md

@@ -70,10 +70,10 @@ Read more about bloom filters in this [great article](https://duckdb.org/2025/03
 
 ## Usage
 
-iparq now supports additional options:
+iparq supports inspecting single files, multiple files, and glob patterns:
 
 ```sh
-iparq inspect <filename> [OPTIONS]
+iparq inspect <filename(s)> [OPTIONS]
 ```
 
 Options include:
@@ -82,9 +82,12 @@ Options include:
 - `--metadata-only`, `-m`: Show only file metadata without column details
 - `--column`, `-c`: Filter results to show only a specific column
 
-Examples:
+### Single File Examples:
 
 ```sh
+# Basic inspection
+iparq inspect yourfile.parquet
+
 # Output in JSON format
 iparq inspect yourfile.parquet --format json
 
@@ -95,9 +98,25 @@ iparq inspect yourfile.parquet --metadata-only
 iparq inspect yourfile.parquet --column column_name
 ```
 
-Replace `<filename>` with the path to your .parquet file. The utility will read the metadata of the file and print the compression codecs used in the parquet file.
+### Multiple Files and Glob Patterns:
+
+```sh
+# Inspect multiple specific files
+iparq inspect file1.parquet file2.parquet file3.parquet
+
+# Use glob patterns to inspect all parquet files
+iparq inspect *.parquet
+
+# Use specific patterns
+iparq inspect yellow*.parquet data_*.parquet
+
+# Combine patterns and specific files
+iparq inspect important.parquet temp_*.parquet
+```
+
+When inspecting multiple files, each file's results are displayed with a header showing the filename. The utility will read the metadata of each file and print the compression codecs used in the parquet files.
 
-## Example ouput - Bloom Filters
+## Example output - Bloom Filters
 
 ```log
 ParquetMetaModel(

{iparq-0.2.6 → iparq-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "iparq"
-version = "0.2.6"
+version = "0.4.0"
 description = "Display version compression and bloom filter information about a parquet file"
 readme = "README.md"
 authors = [
@@ -38,4 +38,9 @@ testpaths = [
 
 [[tool.mypy.overrides]]
 module = ["pyarrow.*"]
-ignore_missing_imports = true
+ignore_missing_imports = true
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+]

iparq-0.4.0/src/iparq/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

{iparq-0.2.6 → iparq-0.4.0}/src/iparq/source.py

@@ -1,3 +1,4 @@
+import glob
 import json
 from enum import Enum
 from typing import List, Optional
@@ -53,6 +54,9 @@ class ColumnInfo(BaseModel):
         column_index (int): The index of the column.
         compression_type (str): The compression type used for the column.
         has_bloom_filter (bool): Whether the column has a bloom filter.
+        has_min_max (bool): Whether min/max statistics are available.
+        min_value (Optional[str]): The minimum value in the column (as string for display).
+        max_value (Optional[str]): The maximum value in the column (as string for display).
     """
 
     row_group: int
@@ -60,6 +64,9 @@ class ColumnInfo(BaseModel):
     column_index: int
     compression_type: str
     has_bloom_filter: Optional[bool] = False
+    has_min_max: Optional[bool] = False
+    min_value: Optional[str] = None
+    max_value: Optional[str] = None
 
 
 class ParquetColumnInfo(BaseModel):
@@ -84,22 +91,16 @@ def read_parquet_metadata(filename: str):
         tuple: A tuple containing:
             - parquet_metadata (pyarrow.parquet.FileMetaData): The metadata of the Parquet file.
             - compression_codecs (set): A set of compression codecs used in the Parquet file.
-    """
-    try:
-        compression_codecs = set([])
-        parquet_metadata = pq.ParquetFile(filename).metadata
 
-        for i in range(parquet_metadata.num_row_groups):
-            for j in range(parquet_metadata.num_columns):
-                compression_codecs.add(
-                    parquet_metadata.row_group(i).column(j).compression
-                )
+    Raises:
+        FileNotFoundError: If the file cannot be found or opened.
+    """
+    compression_codecs = set([])
+    parquet_metadata = pq.ParquetFile(filename).metadata
 
-    except FileNotFoundError:
-        console.print(
-            f"Cannot open: {filename}.", style="blink bold red underline on white"
-        )
-        exit(1)
+    for i in range(parquet_metadata.num_row_groups):
+        for j in range(parquet_metadata.num_columns):
+            compression_codecs.add(parquet_metadata.row_group(i).column(j).compression)
 
     return parquet_metadata, compression_codecs
 
@@ -208,6 +209,59 @@ def print_bloom_filter_info(parquet_metadata, column_info: ParquetColumnInfo) ->
         )
 
 
+def print_min_max_statistics(parquet_metadata, column_info: ParquetColumnInfo) -> None:
+    """
+    Updates the column_info model with min/max statistics information.
+
+    Args:
+        parquet_metadata: The Parquet file metadata.
+        column_info: The ParquetColumnInfo model to update.
+    """
+    try:
+        num_row_groups = parquet_metadata.num_row_groups
+        num_columns = parquet_metadata.num_columns
+
+        for i in range(num_row_groups):
+            row_group = parquet_metadata.row_group(i)
+
+            for j in range(num_columns):
+                column_chunk = row_group.column(j)
+
+                # Find the corresponding column in our model
+                for col in column_info.columns:
+                    if col.row_group == i and col.column_index == j:
+                        # Check if this column has statistics
+                        if column_chunk.is_stats_set:
+                            stats = column_chunk.statistics
+                            col.has_min_max = stats.has_min_max
+
+                            if stats.has_min_max:
+                                # Convert values to string for display, handling potential None values
+                                try:
+                                    col.min_value = (
+                                        str(stats.min)
+                                        if stats.min is not None
+                                        else "null"
+                                    )
+                                    col.max_value = (
+                                        str(stats.max)
+                                        if stats.max is not None
+                                        else "null"
+                                    )
+                                except Exception:
+                                    # Fallback for complex types that might not stringify well
+                                    col.min_value = "<unable to display>"
+                                    col.max_value = "<unable to display>"
+                        else:
+                            col.has_min_max = False
+                        break
+    except Exception as e:
+        console.print(
+            f"Error while collecting min/max statistics: {e}",
+            style="blink bold red underline on white",
+        )
+
+
 def print_column_info_table(column_info: ParquetColumnInfo) -> None:
     """
     Prints the column information using a Rich table.
@@ -223,15 +277,27 @@ def print_column_info_table(column_info: ParquetColumnInfo) -> None:
     table.add_column("Index", justify="center")
     table.add_column("Compression", style="magenta")
     table.add_column("Bloom Filter", justify="center")
+    table.add_column("Min Value", style="yellow")
+    table.add_column("Max Value", style="yellow")
 
     # Add rows to the table
     for col in column_info.columns:
+        # Format min/max values for display
+        min_display = (
+            col.min_value if col.has_min_max and col.min_value is not None else "N/A"
+        )
+        max_display = (
+            col.max_value if col.has_min_max and col.max_value is not None else "N/A"
+        )
+
         table.add_row(
             str(col.row_group),
             col.column_name,
             str(col.column_index),
             col.compression_type,
             "✅" if col.has_bloom_filter else "❌",
+            min_display,
+            max_display,
         )
 
     # Print the table
@@ -260,27 +326,24 @@ def output_json(
     print(json.dumps(result, indent=2))
 
 
-@app.command(name="")
-@app.command(name="inspect")
-def inspect(
-    filename: str = typer.Argument(..., help="Path to the Parquet file to inspect"),
-    format: OutputFormat = typer.Option(
-        OutputFormat.RICH, "--format", "-f", help="Output format (rich or json)"
-    ),
-    metadata_only: bool = typer.Option(
-        False,
-        "--metadata-only",
-        "-m",
-        help="Show only file metadata without column details",
-    ),
-    column_filter: Optional[str] = typer.Option(
-        None, "--column", "-c", help="Filter results to show only specific column"
-    ),
-):
+def inspect_single_file(
+    filename: str,
+    format: OutputFormat,
+    metadata_only: bool,
+    column_filter: Optional[str],
+) -> None:
     """
-    Inspect a Parquet file and display its metadata, compression settings, and bloom filter information.
+    Inspect a single Parquet file and display its metadata, compression settings, and bloom filter information.
+
+    Raises:
+        Exception: If the file cannot be processed.
     """
-    (parquet_metadata, compression) = read_parquet_metadata(filename)
+    try:
+        (parquet_metadata, compression) = read_parquet_metadata(filename)
+    except FileNotFoundError:
+        raise Exception(f"Cannot open: {filename}.")
+    except Exception as e:
+        raise Exception(f"Failed to read metadata: {e}")
 
     # Create metadata model
     meta_model = ParquetMetaModel(
@@ -298,6 +361,7 @@ def inspect(
     # Collect information
     print_compression_types(parquet_metadata, column_info)
     print_bloom_filter_info(parquet_metadata, column_info)
+    print_min_max_statistics(parquet_metadata, column_info)
 
     # Filter columns if requested
     if column_filter:
@@ -322,5 +386,61 @@ def inspect(
             console.print(f"Compression codecs: {compression}")
 
 
+@app.command(name="")
+@app.command(name="inspect")
+def inspect(
+    filenames: List[str] = typer.Argument(
+        ..., help="Path(s) or pattern(s) to Parquet files to inspect"
+    ),
+    format: OutputFormat = typer.Option(
+        OutputFormat.RICH, "--format", "-f", help="Output format (rich or json)"
+    ),
+    metadata_only: bool = typer.Option(
+        False,
+        "--metadata-only",
+        "-m",
+        help="Show only file metadata without column details",
+    ),
+    column_filter: Optional[str] = typer.Option(
+        None, "--column", "-c", help="Filter results to show only specific column"
+    ),
+):
+    """
+    Inspect Parquet files and display their metadata, compression settings, and bloom filter information.
+    """
+    # Expand glob patterns and collect all matching files
+    all_files = []
+    for pattern in filenames:
+        matches = glob.glob(pattern)
+        if matches:
+            all_files.extend(matches)
+        else:
+            # If no matches found, treat as literal filename (for better error reporting)
+            all_files.append(pattern)
+
+    # Remove duplicates while preserving order
+    seen = set()
+    unique_files = []
+    for file in all_files:
+        if file not in seen:
+            seen.add(file)
+            unique_files.append(file)
+
+    # Process each file
+    for i, filename in enumerate(unique_files):
+        # For multiple files, add a header to separate results
+        if len(unique_files) > 1:
+            if i > 0:
+                console.print()  # Add blank line between files
+            console.print(f"[bold blue]File: {filename}[/bold blue]")
+            console.print("─" * (len(filename) + 6))
+
+        try:
+            inspect_single_file(filename, format, metadata_only, column_filter)
+        except Exception as e:
+            console.print(f"Error processing {filename}: {e}", style="red")
+            continue
+
+
 if __name__ == "__main__":
     app()

iparq-0.4.0/tests/test_cli.py

@@ -0,0 +1,173 @@
+import json
+from pathlib import Path
+
+from typer.testing import CliRunner
+
+from iparq.source import app
+
+# Define path to test fixtures
+FIXTURES_DIR = Path(__file__).parent
+fixture_path = FIXTURES_DIR / "dummy.parquet"
+
+
+def test_parquet_info():
+    """Test that the CLI correctly displays parquet file information."""
+    runner = CliRunner()
+    result = runner.invoke(app, ["inspect", str(fixture_path)])
+
+    assert result.exit_code == 0
+
+    # Check for key components instead of exact table format
+    assert "ParquetMetaModel" in result.stdout
+    assert "created_by='parquet-cpp-arrow version 14.0.2'" in result.stdout
+    assert "num_columns=3" in result.stdout
+    assert "num_rows=3" in result.stdout
+    assert "Parquet Column Information" in result.stdout
+    assert "Min Value" in result.stdout
+    assert (
+        "Value" in result.stdout
+    )  # This covers "Max Value" which is split across lines
+    assert "one" in result.stdout and "-1.0" in result.stdout and "2.5" in result.stdout
+    assert "two" in result.stdout and "bar" in result.stdout and "foo" in result.stdout
+    assert (
+        "three" in result.stdout
+        and "False" in result.stdout
+        and "True" in result.stdout
+    )
+    assert "Compression codecs: {'SNAPPY'}" in result.stdout
+
+
+def test_metadata_only_flag():
+    """Test that the metadata-only flag works correctly."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", "--metadata-only", str(fixture_path)])
+
+    assert result.exit_code == 0
+    assert "ParquetMetaModel" in result.stdout
+    assert "Parquet Column Information" not in result.stdout
+
+
+def test_column_filter():
+    """Test that filtering by column name works correctly."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", "--column", "one", str(fixture_path)])
+
+    assert result.exit_code == 0
+    assert "one" in result.stdout
+    assert "two" not in result.stdout
+
+
+def test_json_output():
+    """Test JSON output format."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", "--format", "json", str(fixture_path)])
+
+    assert result.exit_code == 0
+
+    # Test that output is valid JSON
+    data = json.loads(result.stdout)
+
+    # Check JSON structure
+    assert "metadata" in data
+    assert "columns" in data
+    assert "compression_codecs" in data
+    assert data["metadata"]["num_columns"] == 3
+
+    # Check that min/max statistics are included
+    for column in data["columns"]:
+        assert "has_min_max" in column
+        assert "min_value" in column
+        assert "max_value" in column
+        # For our test data, all columns should have min/max stats
+        assert column["has_min_max"] is True
+        assert column["min_value"] is not None
+        assert column["max_value"] is not None
+
+
+def test_multiple_files():
+    """Test that multiple files can be inspected in a single command."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    # Use the same file twice to test deduplication behavior
+
+    result = runner.invoke(app, ["inspect", str(fixture_path), str(fixture_path)])
+
+    assert result.exit_code == 0
+    # Since both arguments are the same file, deduplication means only one file is processed
+    # and since there's only one unique file, no file header should be shown
+    assert (
+        "File:" not in result.stdout
+    )  # No header for single file (after deduplication)
+    assert result.stdout.count("ParquetMetaModel") == 1
+
+
+def test_multiple_different_files():
+    """Test multiple different files by creating a temporary copy."""
+    import shutil
+    import tempfile
+
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+
+    # Create a temporary file copy
+    with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as tmp_file:
+        shutil.copy2(fixture_path, tmp_file.name)
+        tmp_path = tmp_file.name
+
+    try:
+        result = runner.invoke(app, ["inspect", str(fixture_path), tmp_path])
+
+        assert result.exit_code == 0
+        # Should contain file headers for both files
+        assert f"File: {fixture_path}" in result.stdout
+        assert f"File: {tmp_path}" in result.stdout
+        # Should contain metadata for both files
+        assert result.stdout.count("ParquetMetaModel") == 2
+        assert result.stdout.count("Parquet Column Information") == 2
+    finally:
+        # Clean up temporary file
+        import os
+
+        os.unlink(tmp_path)
+
+
+def test_glob_pattern():
+    """Test that glob patterns work correctly."""
+    runner = CliRunner()
+    # Test with a pattern that should match dummy files
+    result = runner.invoke(app, ["inspect", str(FIXTURES_DIR / "dummy*.parquet")])
+
+    assert result.exit_code == 0
+    # Should process at least one file
+    assert "ParquetMetaModel" in result.stdout
+
+
+def test_single_file_no_header():
+    """Test that single files don't show file headers."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", str(fixture_path)])
+
+    assert result.exit_code == 0
+    # Should not contain file header for single file
+    assert "File:" not in result.stdout
+    assert "ParquetMetaModel" in result.stdout
+
+
+def test_error_handling_with_multiple_files():
+    """Test that errors in one file don't stop processing of other files."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    nonexistent_path = FIXTURES_DIR / "nonexistent.parquet"
+
+    result = runner.invoke(app, ["inspect", str(fixture_path), str(nonexistent_path)])
+
+    assert result.exit_code == 0
+    # Should process the good file
+    assert "ParquetMetaModel" in result.stdout
+    # Should show error for bad file
+    assert "Error processing" in result.stdout
+    assert "nonexistent.parquet" in result.stdout