polars-bio 0.14.0__tar.gz → 0.15.0__tar.gz

{polars_bio-0.14.0 → polars_bio-0.15.0}/Cargo.lock

@@ -1404,7 +1404,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-bam"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -1425,7 +1425,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-bed"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-compression",
  "async-stream",
@@ -1449,7 +1449,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-core"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-compression",
  "bytes",
@@ -1468,7 +1468,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-fasta"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-compression",
  "async-stream",
@@ -1490,7 +1490,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-fastq"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-compression",
  "async-stream",
@@ -1514,7 +1514,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-gff"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-compression",
  "async-stream",
@@ -1539,7 +1539,7 @@ dependencies = [
 [[package]]
 name = "datafusion-bio-format-vcf"
 version = "0.1.0"
-source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=aecd7b316c1c498bff14c2cbb4ac0bd04d21612f#aecd7b316c1c498bff14c2cbb4ac0bd04d21612f"
+source = "git+https://github.com/biodatageeks/datafusion-bio-formats.git?rev=99e811d1eeb5f73b67b0a7ce5ea300f0bf557913#99e811d1eeb5f73b67b0a7ce5ea300f0bf557913"
 dependencies = [
  "async-compression",
  "async-stream",
@@ -1554,9 +1554,11 @@ dependencies = [
  "noodles 0.100.0",
  "noodles-bgzf 0.36.0",
  "noodles-vcf 0.80.0",
+ "num_cpus",
  "opendal",
  "tokio",
  "tokio-util",
+ "tracing",
 ]
 
 [[package]]
@@ -2777,6 +2779,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -4184,6 +4192,16 @@ dependencies = [
  "libm",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "numpy"
 version = "0.24.0"
@@ -5225,7 +5243,7 @@ dependencies = [
 
 [[package]]
 name = "polars_bio"
-version = "0.14.0"
+version = "0.15.0"
 dependencies = [
  "arrow",
  "arrow-array",

{polars_bio-0.14.0 → polars_bio-0.15.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars_bio"
-version = "0.14.0"
+version = "0.15.0"
 edition = "2021"
 readme = "README.md"
 
@@ -36,13 +36,13 @@ polars-arrow =  { git = "https://github.com/mwiewior/polars.git" , rev = "da42ae
 polars-python = { git = "https://github.com/mwiewior/polars.git" , rev = "da42ae21ca9c25bc14562e36e07cf02eafd620ee"}
 
 
-datafusion-bio-format-vcf = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
-datafusion-bio-format-core = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
-datafusion-bio-format-gff = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
-datafusion-bio-format-fastq = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
-datafusion-bio-format-bam = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
-datafusion-bio-format-bed = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
-datafusion-bio-format-fasta = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "aecd7b316c1c498bff14c2cbb4ac0bd04d21612f" }
+datafusion-bio-format-vcf = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
+datafusion-bio-format-core = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
+datafusion-bio-format-gff = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
+datafusion-bio-format-fastq = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
+datafusion-bio-format-bam = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
+datafusion-bio-format-bed = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
+datafusion-bio-format-fasta = { git = "https://github.com/biodatageeks/datafusion-bio-formats.git", rev = "99e811d1eeb5f73b67b0a7ce5ea300f0bf557913" }
 
 
 

{polars_bio-0.14.0 → polars_bio-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polars-bio
-Version: 0.14.0
+Version: 0.15.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

polars_bio-0.15.0/benchmarks/01_general_performance.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""
+Benchmark script for general performance comparison between Pandas, Polars, and polars-bio.
+Tests separate read-only and read+filter operations on compressed GFF files.
+
+Features:
+- Includes GFF attributes parsing for fair comparison (all libraries use List[Struct{key, value}])
+- Configurable number of runs per benchmark (NUM_RUNS variable)
+- Single-threaded execution for fair comparison
+
+Split into two benchmark types:
+1. Reading only (no filtering) - measures raw I/O performance (single config per library)
+2. Reading with filtering applied - measures combined I/O + query performance
+
+Notes:
+- Polars has projection/predicate pushdown optimizations enabled by default
+- polars-bio explicitly enables both optimizations for best performance comparison
+- pandas doesn't have equivalent optimization concepts (eager evaluation)
+- Detailed optimization testing is available in separate dedicated benchmark scripts
+"""
+
+import csv
+import os
+import time
+from pathlib import Path
+from typing import Union
+
+import polars as pl
+from gff_parsers import pandas_read_gff, polars_scan_gff
+
+import polars_bio as pb
+
+# Configuration
+GFF_FILE = "/tmp/gencode.v49.annotation.gff3.bgz"
+NUM_RUNS = 1  # Number of runs per benchmark configuration
+
+
+def benchmark_pandas_read_only():
+    """Benchmark pandas reading only (no filtering)"""
+    start_time = time.time()
+    df = pandas_read_gff(GFF_FILE)
+    read_time = time.time() - start_time
+    return read_time, len(df)
+
+
+def benchmark_pandas_read_with_filter():
+    """Benchmark pandas reading with filtering applied"""
+    start_time = time.time()
+    df = pandas_read_gff(GFF_FILE)
+    filtered = df[
+        (df["seqid"] == "chrY") & (df["start"] < 500000) & (df["end"] > 510000)
+    ]
+    result = filtered[["seqid", "start", "end", "type"]]
+    total_time = time.time() - start_time
+    return total_time, len(result)
+
+
+def benchmark_polars_read_only():
+    """Benchmark vanilla Polars reading only (no filtering)"""
+    start_time = time.time()
+    lf = polars_scan_gff(GFF_FILE)
+    result = lf.collect()
+    read_time = time.time() - start_time
+    return read_time, len(result)
+
+
+def benchmark_polars_read_with_filter():
+    """Benchmark vanilla Polars reading with filtering applied"""
+    start_time = time.time()
+    lf = polars_scan_gff(GFF_FILE)
+    result = (
+        lf.filter(
+            (pl.col("seqid") == "chrY")
+            & (pl.col("start") < 500000)
+            & (pl.col("end") > 510000)
+        )
+        .select(["seqid", "start", "end", "type"])
+        .collect()
+    )
+    total_time = time.time() - start_time
+    return total_time, len(result)
+
+
+def benchmark_polars_bio_read_only(
+    projection_pushdown: bool = False, predicate_pushdown: bool = False
+):
+    """Benchmark polars-bio reading only (no filtering)"""
+    pb.set_option("datafusion.execution.target_partitions", "1")
+    start_time = time.time()
+    lf = pb.scan_gff(
+        GFF_FILE,
+        projection_pushdown=projection_pushdown,
+        predicate_pushdown=predicate_pushdown,
+    )
+    result = lf.collect()
+    read_time = time.time() - start_time
+    return read_time, len(result)
+
+
+def benchmark_polars_bio_read_with_filter(
+    projection_pushdown: bool = False, predicate_pushdown: bool = False
+):
+    """Benchmark polars-bio reading with filtering applied"""
+    os.environ["POLARS_MAX_THREADS"] = "1"
+    pb.set_option("datafusion.execution.target_partitions", "1")
+    start_time = time.time()
+    # Use select().filter() order (not filter().select()) to avoid optimization bug
+    lf = pb.scan_gff(
+        GFF_FILE,
+        projection_pushdown=projection_pushdown,
+        predicate_pushdown=predicate_pushdown,
+    )
+    result = (
+        lf.select(["chrom", "start", "end", "type"])
+        .filter(
+            (pl.col("chrom") == "chrY")
+            & (pl.col("start") < 500000)
+            & (pl.col("end") > 510000)
+        )
+        .collect()
+    )
+    total_time = time.time() - start_time
+    return total_time, len(result)
+
+
+def main():
+    """Run benchmarks and save results"""
+    results = []
+
+    # Set single thread for fair comparison
+    os.environ["POLARS_MAX_THREADS"] = "1"
+    pb.set_option("datafusion.execution.target_partitions", "1")
+
+    print("Running general performance benchmarks...")
+
+    # Test cases: read only and read with filter
+    test_cases = [
+        ("read_only", "Reading only (no filtering)"),
+        ("read_with_filter", "Reading with filtering applied"),
+    ]
+
+    for test_type, description in test_cases:
+        print(f"\n=== {description} ===")
+
+        if test_type == "read_only":
+            # Benchmark pandas read only
+            print("Benchmarking Pandas (read only)...")
+            for i in range(NUM_RUNS):
+                total_time, result_count = benchmark_pandas_read_only()
+                results.append(
+                    {
+                        "library": "pandas",
+                        "test_type": test_type,
+                        "projection_pushdown": False,
+                        "predicate_pushdown": False,
+                        "run": i + 1,
+                        "total_time": total_time,
+                        "result_count": result_count,
+                        "threads": 1,
+                    }
+                )
+                print(f"  Run {i+1}: {total_time:.3f}s ({result_count} rows)")
+
+            # Benchmark vanilla Polars read only
+            print("Benchmarking Polars (read only)...")
+            for i in range(NUM_RUNS):
+                total_time, result_count = benchmark_polars_read_only()
+                results.append(
+                    {
+                        "library": "polars",
+                        "test_type": test_type,
+                        "projection_pushdown": False,
+                        "predicate_pushdown": False,
+                        "run": i + 1,
+                        "total_time": total_time,
+                        "result_count": result_count,
+                        "threads": 1,
+                    }
+                )
+                print(f"  Run {i+1}: {total_time:.3f}s ({result_count} rows)")
+
+            # Benchmark polars-bio read only (single configuration - optimizations don't apply)
+            print("Benchmarking polars-bio (read only, no optimizations needed)...")
+            for i in range(NUM_RUNS):
+                total_time, result_count = benchmark_polars_bio_read_only(False, False)
+                results.append(
+                    {
+                        "library": "polars-bio",
+                        "test_type": test_type,
+                        "projection_pushdown": False,
+                        "predicate_pushdown": False,
+                        "run": i + 1,
+                        "total_time": total_time,
+                        "result_count": result_count,
+                        "threads": 1,
+                    }
+                )
+                print(f"  Run {i+1}: {total_time:.3f}s ({result_count} rows)")
+
+        else:  # read_with_filter
+            # Benchmark pandas read with filter
+            print("Benchmarking Pandas (read with filter)...")
+            for i in range(NUM_RUNS):
+                total_time, result_count = benchmark_pandas_read_with_filter()
+                results.append(
+                    {
+                        "library": "pandas",
+                        "test_type": test_type,
+                        "projection_pushdown": False,
+                        "predicate_pushdown": False,
+                        "run": i + 1,
+                        "total_time": total_time,
+                        "result_count": result_count,
+                        "threads": 1,
+                    }
+                )
+                print(f"  Run {i+1}: {total_time:.3f}s ({result_count} filtered rows)")
+
+            # Benchmark vanilla Polars read with filter
+            print("Benchmarking Polars (read with filter)...")
+            for i in range(NUM_RUNS):
+                total_time, result_count = benchmark_polars_read_with_filter()
+                results.append(
+                    {
+                        "library": "polars",
+                        "test_type": test_type,
+                        "projection_pushdown": False,
+                        "predicate_pushdown": False,
+                        "run": i + 1,
+                        "total_time": total_time,
+                        "result_count": result_count,
+                        "threads": 1,
+                    }
+                )
+                print(f"  Run {i+1}: {total_time:.3f}s ({result_count} filtered rows)")
+
+            # Benchmark polars-bio read with filter (both optimizations enabled)
+            print("Benchmarking polars-bio (read with filter, both optimizations)...")
+            for i in range(NUM_RUNS):
+                total_time, result_count = benchmark_polars_bio_read_with_filter(
+                    True, True
+                )
+                results.append(
+                    {
+                        "library": "polars-bio",
+                        "test_type": test_type,
+                        "projection_pushdown": True,
+                        "predicate_pushdown": True,
+                        "run": i + 1,
+                        "total_time": total_time,
+                        "result_count": result_count,
+                        "threads": 1,
+                    }
+                )
+                print(f"  Run {i+1}: {total_time:.3f}s ({result_count} filtered rows)")
+
+    # Save results
+    Path("results").mkdir(exist_ok=True)
+    with open("results/general_performance.csv", "w", newline="") as f:
+        fieldnames = [
+            "library",
+            "test_type",
+            "projection_pushdown",
+            "predicate_pushdown",
+            "run",
+            "total_time",
+            "result_count",
+            "threads",
+        ]
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(results)
+
+    print("\nResults saved to results/general_performance.csv")
+
+    # Print summary statistics
+    print("\n=== Summary Statistics ===")
+    print("Library\t\t\tTest Type\t\tProj PD\tPred PD\tAvg Time")
+    print("-" * 75)
+
+    for test_type, _ in test_cases:
+        for library in ["pandas", "polars", "polars-bio"]:
+            lib_results = [
+                r
+                for r in results
+                if r["library"] == library and r["test_type"] == test_type
+            ]
+            if lib_results:
+                avg_time = sum(r["total_time"] for r in lib_results) / len(lib_results)
+                if library == "pandas":
+                    print(f"{library}\t\t\t{test_type}\t\tN/A\tN/A\t{avg_time:.3f}s")
+                elif library == "polars":
+                    if test_type == "read_with_filter":
+                        print(
+                            f"{library}\t\t\t{test_type}\t\tYes\tYes\t{avg_time:.3f}s (default)"
+                        )
+                    else:
+                        print(
+                            f"{library}\t\t\t{test_type}\t\tN/A\tN/A\t{avg_time:.3f}s"
+                        )
+                else:  # polars-bio
+                    if test_type == "read_with_filter":
+                        print(f"{library}\t\t{test_type}\t\tYes\tYes\t{avg_time:.3f}s")
+                    else:
+                        print(
+                            f"{library}\t\t\t{test_type}\t\tN/A\tN/A\t{avg_time:.3f}s"
+                        )
+
+
+if __name__ == "__main__":
+    main()