iparq 0.2.0__tar.gz → 0.2.6__tar.gz

{iparq-0.2.0 → iparq-0.2.6}/.github/workflows/merge.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-20.04", "windows-latest"]
+        os: ["ubuntu-latest", "windows-latest"]
         python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     env:
       UV_SYSTEM_PYTHON: 1
@@ -56,5 +56,5 @@ jobs:
           run: uvx black . --check --verbose
         - name: Run Python tests
           if: runner.os != 'Windows'
-          run: uv run pytest -s -vv
+          run: uv run pytest -vv
 

iparq-0.2.6/.github/workflows/python-package.yml

@@ -0,0 +1,50 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    permissions:
+        contents: read
+        pull-requests: write
+    name: Test ${{ matrix.os }} Python ${{ matrix.python_version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest", "windows-latest"]
+        python_version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    env:
+      UV_SYSTEM_PYTHON: 1
+    steps:
+        - uses: actions/checkout@v4
+        - name: Setup python
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python_version }}
+            architecture: x64
+        - name: Install uv
+          uses: astral-sh/setup-uv@v5
+
+        # dependencies are in uv.lock
+        - name: Install dependencies
+          run: |
+            uv sync --all-extras 
+
+        - name: Lint with ruff
+          run: uv run ruff check .
+        - name: Check types with mypy
+          run: |
+            cd src/iparq
+            uv run mypy . --config-file=../../pyproject.toml
+        - name: Check formatting with black
+          run: uvx black . --check --verbose
+        - name: Run Python tests
+          if: runner.os != 'Windows'
+          run: uv run pytest -vv

{iparq-0.2.0 → iparq-0.2.6}/.github/workflows/python-publish.yml

@@ -15,11 +15,71 @@ permissions:
   contents: read
 
 jobs:
+  test-and-validate:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+
+      - name: Update dependencies and sync
+        run: |
+          uv lock --upgrade
+          uv sync --all-extras
+
+      - name: Run linting
+        run: |
+          uv run ruff check .
+
+      - name: Run type checking
+        run: |
+          cd src/iparq
+          uv run mypy . --config-file=../../pyproject.toml
+
+      - name: Run tests
+        run: |
+          uv run pytest -v
+
+      - name: Test package build
+        run: |
+          uv build
+
+      - name: Test package installation in clean environment
+        run: |
+          # Test that the built package can be installed and imported
+          python -m venv test_install_env
+          source test_install_env/bin/activate
+          # Install the latest wheel file
+          pip install $(ls -t dist/*.whl | head -1)
+          python -c "import iparq; print(f'Successfully imported iparq version {iparq.__version__}')"
+          iparq --help
+          deactivate
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-results
+          path: |
+            .coverage
+            htmlcov/
+
   release-build:
     permissions:
       contents: read
       pull-requests: write
     runs-on: ubuntu-latest
+    needs: test-and-validate
 
     steps:
       - uses: actions/checkout@v4
@@ -32,9 +92,14 @@ jobs:
         with:
           python-version-file: "pyproject.toml"
 
+      - name: Update dependencies and sync
+        run: |
+          uv lock --upgrade
+          uv sync --all-extras
+
       - name: Build release distributions
         run: |
-            uv build
+          uv build
 
       - name: Upload distributions
         uses: actions/upload-artifact@v4
@@ -45,6 +110,7 @@ jobs:
   pypi-publish:
     runs-on: ubuntu-latest
     needs:
+      - test-and-validate
       - release-build
     permissions:
       # IMPORTANT: this permission is mandatory for trusted publishing

{iparq-0.2.0 → iparq-0.2.6}/.vscode/launch.json

@@ -11,9 +11,7 @@
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
-            "args": [
-                "${command:pickArgs}"
-            ]
+            "args": "${command:pickArgs}"
         }
     ]
 }

iparq-0.2.6/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: iparq
+Version: 0.2.6
+Summary: Display version compression and bloom filter information about a parquet file
+Author-email: MiguelElGallo <miguel.zurcher@gmail.com>
+License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: pyarrow
+Requires-Dist: pydantic
+Requires-Dist: rich
+Requires-Dist: typer
+Provides-Extra: checks
+Requires-Dist: mypy>=1.14.1; extra == 'checks'
+Requires-Dist: ruff>=0.9.3; extra == 'checks'
+Provides-Extra: test
+Requires-Dist: pytest>=7.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# iparq
+
+[![Python package](https://github.com/MiguelElGallo/iparq/actions/workflows/python-package.yml/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/python-package.yml)
+
+[![Dependabot Updates](https://github.com/MiguelElGallo/iparq/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/dependabot/dependabot-updates)
+
+[![Upload Python Package](https://github.com/MiguelElGallo/iparq/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/python-publish.yml)
+
+![alt text](media/iparq.png)
+After reading [this blog](https://duckdb.org/2025/01/22/parquet-encodings.html), I began to wonder which Parquet version and compression methods the everyday tools we rely on actually use, only to find that there's no straightforward way to determine this. That curiosity and the difficulty of quickly discovering such details motivated me to create iparq (Information Parquet). My goal with iparq is to help users easily identify the specifics of the Parquet files generated by different engines, making it clear which features—like newer encodings or certain compression algorithms—the creator of the parquet is using.
+
+***New*** Bloom filters information: Displays if there are bloom filters.
+Read more about bloom filters in this [great article](https://duckdb.org/2025/03/07/parquet-bloom-filters-in-duckdb.html).
+
+## Installation
+
+### Zero installation - Recommended
+
+1) Make sure to have Astral's UV installed by following the steps here:  
+
+    <https://docs.astral.sh/uv/getting-started/installation/>
+
+2) Execute the following command:
+
+    ```sh
+    uvx --refresh iparq inspect yourparquet.parquet
+    ```
+
+### Using pip
+
+1) Install the package using pip:
+
+    ```sh
+    pip install iparq
+    ```
+
+2) Verify the installation by running:
+
+    ```sh
+    iparq --help
+    ```
+
+### Using uv
+
+1) Make sure to have Astral's UV installed by following the steps here:  
+
+    <https://docs.astral.sh/uv/getting-started/installation/>
+
+2) Execute the following command:
+
+    ```sh
+    uv pip install iparq
+    ```
+
+3) Verify the installation by running:
+
+    ```sh
+    iparq --help
+    ```
+
+### Using Homebrew in a MAC
+
+1) Run the following:
+
+    ```sh
+    brew tap MiguelElGallo/tap https://github.com/MiguelElGallo//homebrew-iparq.git
+    brew install MiguelElGallo/tap/iparq
+    iparq --help
+    ```
+
+## Usage
+
+iparq now supports additional options:
+
+```sh
+iparq inspect <filename> [OPTIONS]
+```
+
+Options include:
+
+- `--format`, `-f`: Output format, either `rich` (default) or `json`
+- `--metadata-only`, `-m`: Show only file metadata without column details
+- `--column`, `-c`: Filter results to show only a specific column
+
+Examples:
+
+```sh
+# Output in JSON format
+iparq inspect yourfile.parquet --format json
+
+# Show only metadata
+iparq inspect yourfile.parquet --metadata-only
+
+# Filter to show only a specific column
+iparq inspect yourfile.parquet --column column_name
+```
+
+Replace `<filename>` with the path to your .parquet file. The utility will read the metadata of the file and print the compression codecs used in the parquet file.
+
+## Example ouput - Bloom Filters
+
+```log
+ParquetMetaModel(
+    created_by='DuckDB version v1.2.1 (build 8e52ec4395)',
+    num_columns=1,
+    num_rows=100000000,
+    num_row_groups=10,
+    format_version='1.0',
+    serialized_size=1196
+)
+                   Parquet Column Information                   
+┏━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
+┃ Row Group ┃ Column Name ┃ Index ┃ Compression ┃ Bloom Filter ┃
+┡━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
+│     0     │ r           │   0   │ SNAPPY      │      ✅      │
+│     1     │ r           │   0   │ SNAPPY      │      ✅      │
+│     2     │ r           │   0   │ SNAPPY      │      ✅      │
+│     3     │ r           │   0   │ SNAPPY      │      ✅      │
+│     4     │ r           │   0   │ SNAPPY      │      ✅      │
+│     5     │ r           │   0   │ SNAPPY      │      ✅      │
+│     6     │ r           │   0   │ SNAPPY      │      ✅      │
+│     7     │ r           │   0   │ SNAPPY      │      ✅      │
+│     8     │ r           │   0   │ SNAPPY      │      ✅      │
+│     9     │ r           │   0   │ SNAPPY      │      ✅      │
+└───────────┴─────────────┴───────┴─────────────┴──────────────┘
+Compression codecs: {'SNAPPY'}
+```

iparq-0.2.6/README.md

@@ -0,0 +1,127 @@
+# iparq
+
+[![Python package](https://github.com/MiguelElGallo/iparq/actions/workflows/python-package.yml/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/python-package.yml)
+
+[![Dependabot Updates](https://github.com/MiguelElGallo/iparq/actions/workflows/dependabot/dependabot-updates/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/dependabot/dependabot-updates)
+
+[![Upload Python Package](https://github.com/MiguelElGallo/iparq/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MiguelElGallo/iparq/actions/workflows/python-publish.yml)
+
+![alt text](media/iparq.png)
+After reading [this blog](https://duckdb.org/2025/01/22/parquet-encodings.html), I began to wonder which Parquet version and compression methods the everyday tools we rely on actually use, only to find that there's no straightforward way to determine this. That curiosity and the difficulty of quickly discovering such details motivated me to create iparq (Information Parquet). My goal with iparq is to help users easily identify the specifics of the Parquet files generated by different engines, making it clear which features—like newer encodings or certain compression algorithms—the creator of the parquet is using.
+
+***New*** Bloom filters information: Displays if there are bloom filters.
+Read more about bloom filters in this [great article](https://duckdb.org/2025/03/07/parquet-bloom-filters-in-duckdb.html).
+
+## Installation
+
+### Zero installation - Recommended
+
+1) Make sure to have Astral's UV installed by following the steps here:  
+
+    <https://docs.astral.sh/uv/getting-started/installation/>
+
+2) Execute the following command:
+
+    ```sh
+    uvx --refresh iparq inspect yourparquet.parquet
+    ```
+
+### Using pip
+
+1) Install the package using pip:
+
+    ```sh
+    pip install iparq
+    ```
+
+2) Verify the installation by running:
+
+    ```sh
+    iparq --help
+    ```
+
+### Using uv
+
+1) Make sure to have Astral's UV installed by following the steps here:  
+
+    <https://docs.astral.sh/uv/getting-started/installation/>
+
+2) Execute the following command:
+
+    ```sh
+    uv pip install iparq
+    ```
+
+3) Verify the installation by running:
+
+    ```sh
+    iparq --help
+    ```
+
+### Using Homebrew in a MAC
+
+1) Run the following:
+
+    ```sh
+    brew tap MiguelElGallo/tap https://github.com/MiguelElGallo//homebrew-iparq.git
+    brew install MiguelElGallo/tap/iparq
+    iparq --help
+    ```
+
+## Usage
+
+iparq now supports additional options:
+
+```sh
+iparq inspect <filename> [OPTIONS]
+```
+
+Options include:
+
+- `--format`, `-f`: Output format, either `rich` (default) or `json`
+- `--metadata-only`, `-m`: Show only file metadata without column details
+- `--column`, `-c`: Filter results to show only a specific column
+
+Examples:
+
+```sh
+# Output in JSON format
+iparq inspect yourfile.parquet --format json
+
+# Show only metadata
+iparq inspect yourfile.parquet --metadata-only
+
+# Filter to show only a specific column
+iparq inspect yourfile.parquet --column column_name
+```
+
+Replace `<filename>` with the path to your .parquet file. The utility will read the metadata of the file and print the compression codecs used in the parquet file.
+
+## Example ouput - Bloom Filters
+
+```log
+ParquetMetaModel(
+    created_by='DuckDB version v1.2.1 (build 8e52ec4395)',
+    num_columns=1,
+    num_rows=100000000,
+    num_row_groups=10,
+    format_version='1.0',
+    serialized_size=1196
+)
+                   Parquet Column Information                   
+┏━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
+┃ Row Group ┃ Column Name ┃ Index ┃ Compression ┃ Bloom Filter ┃
+┡━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
+│     0     │ r           │   0   │ SNAPPY      │      ✅      │
+│     1     │ r           │   0   │ SNAPPY      │      ✅      │
+│     2     │ r           │   0   │ SNAPPY      │      ✅      │
+│     3     │ r           │   0   │ SNAPPY      │      ✅      │
+│     4     │ r           │   0   │ SNAPPY      │      ✅      │
+│     5     │ r           │   0   │ SNAPPY      │      ✅      │
+│     6     │ r           │   0   │ SNAPPY      │      ✅      │
+│     7     │ r           │   0   │ SNAPPY      │      ✅      │
+│     8     │ r           │   0   │ SNAPPY      │      ✅      │
+│     9     │ r           │   0   │ SNAPPY      │      ✅      │
+└───────────┴─────────────┴───────┴─────────────┴──────────────┘
+Compression codecs: {'SNAPPY'}
+```

{iparq-0.2.0 → iparq-0.2.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "iparq"
-version = "0.2.0"
+version = "0.2.6"
 description = "Display version compression and bloom filter information about a parquet file"
 readme = "README.md"
 authors = [
@@ -9,7 +9,7 @@ authors = [
 requires-python = ">=3.9"
 dependencies = [
     "pyarrow",
-    "typer[all]",
+    "typer",
     "pydantic",
     "rich",
 ]
@@ -31,7 +31,7 @@ requires = ["hatchling"]
 build-backend = "hatchling.build"
 
 [tool.pytest.ini_options]
-addopts = "-ra -q"
+addopts = ["-ra", "-q"]
 testpaths = [
     "tests",
 ]

iparq-0.2.6/src/iparq/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.6"

{iparq-0.2.0 → iparq-0.2.6}/src/iparq/source.py

@@ -1,3 +1,5 @@
+import json
+from enum import Enum
 from typing import List, Optional
 
 import pyarrow.parquet as pq
@@ -7,10 +9,19 @@ from rich import print
 from rich.console import Console
 from rich.table import Table
 
-app = typer.Typer()
+app = typer.Typer(
+    help="Inspect Parquet files for metadata, compression, and bloom filters"
+)
 console = Console()
 
 
+class OutputFormat(str, Enum):
+    """Enum for output format options."""
+
+    RICH = "rich"
+    JSON = "json"
+
+
 class ParquetMetaModel(BaseModel):
     """
     ParquetMetaModel is a data model representing metadata for a Parquet file.
@@ -227,20 +238,59 @@ def print_column_info_table(column_info: ParquetColumnInfo) -> None:
     console.print(table)
 
 
-@app.command()
-def main(filename: str):
+def output_json(
+    meta_model: ParquetMetaModel,
+    column_info: ParquetColumnInfo,
+    compression_codecs: set,
+) -> None:
     """
-    Main function to read and print Parquet file metadata.
+    Outputs the parquet information in JSON format.
 
     Args:
-        filename (str): The path to the Parquet file.
-
-    Returns:
-        Metadata of the Parquet file and the compression codecs used.
+        meta_model: The Parquet metadata model
+        column_info: The column information model
+        compression_codecs: Set of compression codecs used
+    """
+    result = {
+        "metadata": meta_model.model_dump(),
+        "columns": [column.model_dump() for column in column_info.columns],
+        "compression_codecs": list(compression_codecs),
+    }
+
+    print(json.dumps(result, indent=2))
+
+
+@app.command(name="")
+@app.command(name="inspect")
+def inspect(
+    filename: str = typer.Argument(..., help="Path to the Parquet file to inspect"),
+    format: OutputFormat = typer.Option(
+        OutputFormat.RICH, "--format", "-f", help="Output format (rich or json)"
+    ),
+    metadata_only: bool = typer.Option(
+        False,
+        "--metadata-only",
+        "-m",
+        help="Show only file metadata without column details",
+    ),
+    column_filter: Optional[str] = typer.Option(
+        None, "--column", "-c", help="Filter results to show only specific column"
+    ),
+):
+    """
+    Inspect a Parquet file and display its metadata, compression settings, and bloom filter information.
     """
     (parquet_metadata, compression) = read_parquet_metadata(filename)
 
-    print_parquet_metadata(parquet_metadata)
+    # Create metadata model
+    meta_model = ParquetMetaModel(
+        created_by=parquet_metadata.created_by,
+        num_columns=parquet_metadata.num_columns,
+        num_rows=parquet_metadata.num_rows,
+        num_row_groups=parquet_metadata.num_row_groups,
+        format_version=str(parquet_metadata.format_version),
+        serialized_size=parquet_metadata.serialized_size,
+    )
 
     # Create a model to store column information
     column_info = ParquetColumnInfo()
@@ -249,10 +299,27 @@ def main(filename: str):
     print_compression_types(parquet_metadata, column_info)
     print_bloom_filter_info(parquet_metadata, column_info)
 
-    # Print the information as a table
-    print_column_info_table(column_info)
-
-    print(f"Compression codecs: {compression}")
+    # Filter columns if requested
+    if column_filter:
+        column_info.columns = [
+            col for col in column_info.columns if col.column_name == column_filter
+        ]
+        if not column_info.columns:
+            console.print(
+                f"No columns match the filter: {column_filter}", style="yellow"
+            )
+
+    # Output based on format selection
+    if format == OutputFormat.JSON:
+        output_json(meta_model, column_info, compression)
+    else:  # Rich format
+        # Print the metadata
+        console.print(meta_model)
+
+        # Print column details if not metadata only
+        if not metadata_only:
+            print_column_info_table(column_info)
+            console.print(f"Compression codecs: {compression}")
 
 
 if __name__ == "__main__":

iparq-0.2.6/tests/test_cli.py

@@ -0,0 +1,78 @@
+import json
+from pathlib import Path
+
+from typer.testing import CliRunner
+
+from iparq.source import app
+
+# Define path to test fixtures
+FIXTURES_DIR = Path(__file__).parent
+fixture_path = FIXTURES_DIR / "dummy.parquet"
+
+
+def test_parquet_info():
+    """Test that the CLI correctly displays parquet file information."""
+    runner = CliRunner()
+    result = runner.invoke(app, ["inspect", str(fixture_path)])
+
+    assert result.exit_code == 0
+
+    expected_output = """ParquetMetaModel(
+    created_by='parquet-cpp-arrow version 14.0.2',
+    num_columns=3,
+    num_rows=3,
+    num_row_groups=1,
+    format_version='2.6',
+    serialized_size=2223
+)
+                   Parquet Column Information                   
+┏━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓
+┃ Row Group ┃ Column Name ┃ Index ┃ Compression ┃ Bloom Filter ┃
+┡━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩
+│     0     │ one         │   0   │ SNAPPY      │      ✅      │
+│     0     │ two         │   1   │ SNAPPY      │      ✅      │
+│     0     │ three       │   2   │ SNAPPY      │      ✅      │
+└───────────┴─────────────┴───────┴─────────────┴──────────────┘
+Compression codecs: {'SNAPPY'}"""
+
+    assert expected_output in result.stdout
+
+
+def test_metadata_only_flag():
+    """Test that the metadata-only flag works correctly."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", "--metadata-only", str(fixture_path)])
+
+    assert result.exit_code == 0
+    assert "ParquetMetaModel" in result.stdout
+    assert "Parquet Column Information" not in result.stdout
+
+
+def test_column_filter():
+    """Test that filtering by column name works correctly."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", "--column", "one", str(fixture_path)])
+
+    assert result.exit_code == 0
+    assert "one" in result.stdout
+    assert "two" not in result.stdout
+
+
+def test_json_output():
+    """Test JSON output format."""
+    runner = CliRunner()
+    fixture_path = FIXTURES_DIR / "dummy.parquet"
+    result = runner.invoke(app, ["inspect", "--format", "json", str(fixture_path)])
+
+    assert result.exit_code == 0
+
+    # Test that output is valid JSON
+    data = json.loads(result.stdout)
+
+    # Check JSON structure
+    assert "metadata" in data
+    assert "columns" in data
+    assert "compression_codecs" in data
+    assert data["metadata"]["num_columns"] == 3