pycanopy 0.2.0__tar.gz → 0.2.2__tar.gz

{pycanopy-0.2.0 → pycanopy-0.2.2}/.github/workflows/CI.yml

@@ -14,7 +14,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo test
+      - uses: taiki-e/install-action@cargo-nextest
+      - run: cargo nextest run
       - run: cargo fmt --check
       - run: cargo clippy -- -D warnings
 

{pycanopy-0.2.0 → pycanopy-0.2.2}/Cargo.lock

@@ -274,9 +274,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
 
 [[package]]
 name = "log"
-version = "0.4.31"
+version = "0.4.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f"
+checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
 
 [[package]]
 name = "matrixmultiply"
@@ -396,7 +396,7 @@ dependencies = [
 
 [[package]]
 name = "pycanopy"
-version = "0.1.0"
+version = "0.2.2"
 dependencies = [
  "geo",
  "geo-index",

{pycanopy-0.2.0 → pycanopy-0.2.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pycanopy"
-version = "0.1.0"
+version = "0.2.2"
 edition = "2021"
 readme = "README.md"
 
@@ -19,7 +19,13 @@ ordered-float = "4"
 
 [profile.release]
 opt-level = 3
-lto = true
+lto = "thin"
 codegen-units = 1
 
-[dev-dependencies]
+# Local dev builds: same opt-level as release but skips LTO and uses more
+# codegen units to stay within 8GB RAM. Use for make check / make build.
+[profile.dev-release]
+inherits = "release"
+lto = false
+codegen-units = 8
+

pycanopy-0.2.2/Makefile

@@ -0,0 +1,60 @@
+.DEFAULT_GOAL := check
+
+sources = python/ tests/python/ benchmarks/
+
+# Preserve colour in cargo output when running from a tty.
+export CARGO_TERM_COLOR=$(shell (test -t 0 && echo "always") || echo "auto")
+
+.PHONY: format ## Auto-format Rust and Python source files
+format:
+	cargo fmt
+	.venv/bin/ruff check --fix $(sources)
+	.venv/bin/ruff format $(sources)
+
+.PHONY: lint-python ## Lint Python source files
+lint-python:
+	.venv/bin/ruff check $(sources)
+	.venv/bin/ruff format --check $(sources)
+
+.PHONY: lint-rust ## Lint Rust source files (fmt check + clippy over all code incl. tests)
+lint-rust:
+	cargo fmt --all -- --check
+	cargo clippy --tests -- -D warnings
+
+.PHONY: lint ## Lint Rust and Python source files
+lint: lint-python lint-rust
+
+.PHONY: build ## Debug build — fast compile, use for local iteration
+build:
+	@rm -f python/pycanopy/*.so
+	maturin develop
+
+.PHONY: build-prod ## Optimised build — use for benchmarks and profiling
+build-prod:
+	@rm -f python/pycanopy/*.so
+	maturin develop --release
+
+# Build first so clippy and cargo nextest reuse compiled objects from maturin
+# instead of each triggering a second Rust compile pass.
+# sccache (via RUSTC_WRAPPER in .cargo/config.toml) caches across runs.
+.PHONY: check ## Format, build, lint, and test — run before every commit
+check: format build lint
+	cargo nextest run
+	.venv/bin/pytest tests/python/ --durations=5
+
+.PHONY: test ## Build and run all tests without formatting or linting
+test: build
+	cargo nextest run
+	.venv/bin/pytest tests/python/
+
+.PHONY: clean ## Remove build artifacts and caches
+clean:
+	rm -rf `find . -name __pycache__`
+	rm -f `find . -type f -name '*.py[co]'`
+	rm -rf .pytest_cache .ruff_cache
+	rm -f python/pycanopy/*.so
+
+.PHONY: help ## Display this help message
+help:
+	@grep -E '^\.PHONY: .*?## .*$$' $(MAKEFILE_LIST) | \
+	awk 'BEGIN {FS = ".PHONY: |## "}; {printf "\033[36m%-15s\033[0m %s\n", $$2, $$3}'

pycanopy-0.2.2/PKG-INFO

@@ -0,0 +1,349 @@
+Metadata-Version: 2.4
+Name: pycanopy
+Version: 0.2.2
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: GIS
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Requires-Dist: pyarrow>=12.0
+Requires-Dist: numpy>=1.24
+Requires-Dist: polars>=0.20
+License-File: LICENSE
+Summary: Declarative spatial query layer for Polars
+Keywords: geospatial,spatial-index,rtree,kdtree,knn,geoarrow
+Author-email: Pranav Walimbe <pranav1077@gmail.com>
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Issues, https://github.com/pranavwalimbe/pycanopy/issues
+Project-URL: Repository, https://github.com/pranavwalimbe/pycanopy
+
+<p align="center">
+  <img src="assets/pycanopy_logo3.png" alt="PyCanopy" width="800"/>
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/pycanopy/"><img src="https://img.shields.io/pypi/v/pycanopy" alt="PyPI version"/></a>
+  <a href="https://pypi.org/project/pycanopy/"><img src="https://img.shields.io/pypi/pyversions/pycanopy" alt="Python versions"/></a>
+  <a href="https://github.com/pranav-walimbe/pycanopy/actions/workflows/CI.yml"><img src="https://img.shields.io/github/actions/workflow/status/pranav-walimbe/pycanopy/CI.yml?branch=main&label=tests" alt="CI"/></a>
+  <a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License: MIT"/></a>
+</p>
+
+<p align="center">A spatial query layer for Polars. Rust core, Python API.</p>
+
+---
+
+> [!NOTE]
+> Up to **155x** on range queries · up to **1,949x** on kNN · up to **1,521x** on polygon contains · up to **8,522x** on within joins · [Full benchmarks](#benchmarks)
+
+---
+
+## Installation
+
+```bash
+pip install pycanopy
+```
+
+> Pre-built wheels for Linux, macOS, and Windows. No Rust toolchain required.
+
+```python
+import polars as pl
+from pycanopy import SpatialFrame
+
+sf = SpatialFrame(pl.read_parquet("cities.parquet"), x_col="lon", y_col="lat")
+result = sf.lazy().filter(pl.col("population") > 100_000).range_query(-10.0, 35.0, 40.0, 70.0).collect()
+```
+
+---
+
+## Why PyCanopy
+
+Polars has no native spatial query support. Getting bounding-box filters, k-nearest neighbours, or point-in-polygon tests typically means converting to GeoPandas, managing an index manually, or scanning every row in Python. GeoPandas applies linear scans by default for containment and range tests; its STRtree requires explicit opt-in via `.sindex` and is the only available index type regardless of data distribution. KNN has no built-in path at all.
+
+PyCanopy adds a declarative lazy query layer directly on Polars DataFrames. You describe the operations you want, and PyCanopy decides which index to build, in what order to run each operation, and what to hand off to Polars to execute. It is designed for in-memory workloads at the moment. 
+
+|                             | PyCanopy      | GeoPandas        | Manual STRtree |
+|:----------------------------|:-------------:|:----------------:|:--------------:|
+| Works natively in Polars    | ✓             | ✗                | ✗              |
+| Lazy / declarative API      | ✓             | ✗                | ✗              |
+| Auto index selection        | ✓             | ✗ (STR only)     | ✗              |
+| KNN join built-in           | ✓             | ✗                | ✗              |
+| Delta buffer (live append)  | ✓             | ✗                | ✗              |
+
+- **Lazy planning** -- declare ops, the optimizer reorders and fuses them before any index is built
+- **Auto index selection** -- KD-tree, R-tree, uniform grid, or brute force chosen per query
+- **Native Polars** -- results are `pl.DataFrame`, no round-trip through GeoPandas
+- **Rust hot paths** -- zero-copy at the Python boundary, loop-level parallelism via Rayon
+- **Delta buffer** -- append new points and query immediately without rebuilding the index
+
+---
+
+## Usage
+
+### Point dataset: range and KNN
+
+```python
+import polars as pl
+from pycanopy import SpatialFrame
+
+df = pl.read_parquet("cities.parquet")
+sf = SpatialFrame(df, x_col="lon", y_col="lat")
+
+# Bounding-box filter combined with a scalar predicate.
+# Optimizer places the scalar filter first, then runs the range query
+# on the reduced row set.
+result = (
+    sf.lazy()
+    .filter(pl.col("population") > 100_000)
+    .range_query(min_x=-10.0, min_y=35.0, max_x=40.0, max_y=70.0)
+    .collect()
+)
+
+# k-nearest neighbours
+nearest = sf.lazy().knn(x=2.35, y=48.85, k=5).collect()
+```
+
+<details>
+<summary>More examples -- KNN join, polygon contains, within-distance join, branching, delta buffer</summary>
+
+### Chaining multiple spatial predicates
+
+```python
+# Two range predicates are fused into a single index build on large datasets.
+result = (
+    sf.lazy()
+    .range_query(0.0, 0.0, 50.0, 50.0)
+    .range_query(10.0, 10.0, 40.0, 40.0)
+    .collect()
+)
+```
+
+### Inspecting the optimizer plan
+
+```python
+# Declare ops in any order — explain() shows what the optimizer will actually run.
+lf = (
+    sf.lazy()
+    .range_query(min_x=-10.0, min_y=35.0, max_x=40.0, max_y=70.0)
+    .filter(pl.col("population") > 100_000)
+)
+
+print(lf.explain())
+# RANGE_QUERY [(-10, 35) → (40, 70)]
+# FROM
+#   FILTER [(col("population")) > (dyn int: 100000)]
+#   FROM
+#     DF [N=100,000; path: EXPR]
+
+print(lf.explain(optimized=False))
+# FILTER [(col("population")) > (dyn int: 100000)]
+# FROM
+#   RANGE_QUERY [(-10, 35) → (40, 70)]
+#   FROM
+#     DF [N=100,000]
+```
+
+Follows Polars' FROM-chain convention: bottom = runs first, top = outermost result. In the optimized plan, FILTER appears below RANGE_QUERY — the scalar filter runs first on raw data, and RANGE_QUERY receives the already-filtered subset. `explain(optimized=False)` shows declaration order for comparison.
+
+### KNN join
+
+```python
+query_df = pl.DataFrame({"qx": [2.35, 13.4], "qy": [48.85, 52.5]})
+
+# For each row in query_df, find the 3 nearest rows in sf.
+result = sf.lazy().knn_join(query_df, x_col="qx", y_col="qy", k=3).collect()
+```
+
+### Polygon dataset: contains and range
+
+```python
+from shapely.geometry import box
+from pycanopy import SpatialFrame
+
+polygons = [box(i, 0, i + 0.9, 0.9) for i in range(100_000)]
+df = pl.DataFrame({"id": list(range(100_000)), "geom": polygons})
+sf = SpatialFrame.from_polygons(df, geometry_col="geom")
+
+# Which polygons contain this point?
+containing = sf.lazy().contains(x=5.5, y=0.5).collect()
+
+# Which polygon MBRs intersect this bbox?
+intersecting = sf.lazy().range_query(0.0, 0.0, 10.0, 1.0).collect()
+```
+
+### Polygon holes
+
+```python
+from shapely.geometry import Polygon
+
+# Interior rings (holes) are fully supported.
+outer = [(0, 0), (10, 0), (10, 10), (0, 10)]
+hole  = [(2, 2), (8, 2),  (8, 8),  (2, 8)]
+donut = Polygon(outer, [hole])
+
+sf = SpatialFrame.from_polygons(pl.DataFrame({"id": [0], "geom": [donut]}), geometry_col="geom")
+
+# Point inside the hole is NOT contained.
+sf.lazy().contains(x=5.0, y=5.0).collect()   # empty
+
+# Point outside the hole but inside the outer ring IS contained.
+sf.lazy().contains(x=1.0, y=1.0).collect()   # returns the polygon row
+```
+
+### Within join
+
+```python
+# For each query point, find which polygons in sf contain it.
+query_df = pl.DataFrame({"qx": [5.5, 12.3], "qy": [0.5, 0.5]})
+result = sf.lazy().within_join(query_df, x_col="qx", y_col="qy").collect()
+```
+
+### Within-distance join
+
+```python
+# For each query point, find all sf points within 50 km.
+query_df = pl.DataFrame({"qx": [2.35, 13.4], "qy": [48.85, 52.5]})
+result = sf.lazy().within_distance_join(query_df, x_col="qx", y_col="qy", distance=50.0).collect()
+```
+
+### Branching from a shared base
+
+```python
+from pycanopy import SpatialFrame, SpatialLazyFrame
+
+# Expensive filter applied once; two queries branch from the result.
+base = sf.lazy().filter(pl.col("population") > 100_000).range_query(-10.0, 35.0, 40.0, 70.0)
+
+major = base.filter(pl.col("population") > 1_000_000)
+minor = base.filter(pl.col("population") <= 1_000_000)
+
+# collect_all detects the shared prefix, caches it in Polars,
+# and executes both branches in a single pass.
+results = SpatialLazyFrame.collect_all([major, minor])
+df_major, df_minor = results
+```
+
+### Live updates via delta buffer
+
+```python
+# Append new points -- visible to queries immediately, no index rebuild yet.
+import numpy as np
+sf.engine.append_delta(np.array([2.5]), np.array([48.9]))
+
+# Queries probe the main index and scan the delta in parallel.
+result = sf.lazy().range_query(-10.0, 35.0, 40.0, 70.0).collect()
+
+# The buffer flushes automatically when accumulated query cost exceeds
+# the estimated index rebuild cost, or when it exceeds 10% of N.
+# Force a flush manually if needed.
+sf.engine.flush()
+```
+
+</details>
+
+---
+
+## Benchmarks
+
+Apple M-series used for benchmarking. **Warm** = cached index, second call. **Index build** = one-time cost, amortised across queries. Uniform distribution; clustered note below.
+
+### Single-query operations
+
+| Operation              |       N | Index build |    Warm |   Naive | Speedup    | Idx mem |
+|:-----------------------|--------:|------------:|--------:|--------:|-----------:|--------:|
+| Range query (points)   | 100,000 |      1.3 ms |   29 µs |  4.4 ms |   **155x** | 783 KB  |
+| kNN k=10               | 100,000 |      9.3 ms |    3 µs |  5.4 ms | **1,949x** | 1.9 MB  |
+| Polygon contains       | 100,000 |      6.2 ms |    5 µs |  7.0 ms | **1,521x** | 3.7 MB  |
+| Polygon range          | 100,000 |      5.6 ms |    8 µs |  3.3 ms |   **391x** | 3.7 MB  |
+| kNN join k=5           |  10,000 |      7.3 ms |  2.1 ms |   5.4 s | **2,601x** | 180 KB  |
+| Within-distance join   |  10,000 |      0.5 ms | 12.6 ms |   1.3 s |   **102x** | —       |
+| Within join (polygons) |  10,000 |      1.6 ms | 0.52 ms |   4.4 s | **8,522x** | 354 KB  |
+
+### Chained lazy queries (N = 100,000, uniform)
+
+The optimizer reorders scalars before spatial ops regardless of declared order, and fuses consecutive wide spatial predicates into one index pass.
+
+| Chain                                        | Optimizer action | Index build |    Warm | GeoPandas | Speedup  |
+|:---------------------------------------------|:-----------------|------------:|--------:|----------:|---------:|
+| circ\_scalar → range³                        | scalar first     |      2.5 ms | 0.19 ms |    9.2 ms |  **50x** |
+| range² → 3× scalar (spatial declared first)  | scalars first    |      1.0 ms | 0.23 ms |    6.0 ms |  **26x** |
+| range⁴ at 10% selectivity                   | fused            |      1.0 ms | 0.92 ms |     13 ms |  **14x** |
+| wide\_scalar (95%) → tight\_range (1%)       | scalar first     |      4.1 ms | 0.30 ms |    3.1 ms |  **11x** |
+| circ\_scalar + diag\_scalar → kNN k=50       | scalar first     |       15 ms | 1.25 ms |    3.6 ms |   **3x** |
+
+---
+
+## How It Works
+
+### Query flow
+
+```
+  sf.lazy().filter(...).range_query(...).knn_join(...).collect()
+                            |
+               +------------+------------+
+               |     SpatialOptimizer    |
+               |  * reorder ops by cost  |
+               |  * fuse spatial preds   |
+               |  * select index type    |
+               |  * spatial join order   |
+               +------------+------------+
+                            |
+               +------------+------------+
+               |      Polars executes    |
+               |  scalar filters first   |
+               |  then spatial queries   |
+               +------------+------------+
+                            |
+                      pl.DataFrame
+```
+
+### Optimizer decisions
+
+- **Predicate pushdown:** scalar predicates are placed before spatial ones and sorted cheapest-first using AST cost estimation. They cost nothing extra and shrink the row count before any index is touched.
+- **Fusion:** consecutive spatial predicates on large datasets are merged into a single index build and one pass over the data.
+- **Index type:** selected per query based on geometry type, data distribution, and selectivity.
+- **Join order:** for symmetric joins (`within_join`, `within_distance_join`), the optimizer indexes the smaller side when it is less than half the size of the other. `knn_join` is asymmetric and always indexes the engine side.
+
+### Index management
+
+Indexes are built lazily. Nothing is constructed at load time; stats (extent, point distribution, a 32x32 histogram) are computed eagerly and drive selection at the first query. The selected index is then cached for all subsequent queries.
+
+| Condition                                     | Index        |
+|:----------------------------------------------|:-------------|
+| N < 500, selectivity > 50%, or k/N > 10%     | Brute force  |
+| Point range, uniform distribution             | Uniform grid |
+| Point range, clustered distribution           | KD-tree      |
+| Point KNN or contains                         | KD-tree      |
+| Polygons, any query                           | R-tree       |
+
+All index types share the same underlying coordinate arrays with no duplication.
+
+### Why Rust
+
+The hot paths need packed immutable index structures, zero-copy array slices at the Python boundary, and loop-level parallelism. C++ would require a separate FFI layer and loses the native Polars plugin integration that PyO3/Maturin provides for free.
+
+---
+
+## Accepted input formats
+
+| Format                             | Example                                    |
+|:-----------------------------------|:-------------------------------------------|
+| numpy `(N, 2)` array               | `np.array([[x, y], ...])`                  |
+| GeoArrow PyArrow array             | `pa.StructArray` or `FixedSizeList<2>`     |
+| geopandas `GeoSeries`              | `gdf.geometry`                             |
+| list of shapely Points or Polygons | `[Point(x, y), ...]`                       |
+| list of `(x, y)` tuples            | `[(x, y), ...]`                            |
+| Separate coordinate sequences      | `Engine.from_coords(xs, ys)`               |
+
+---
+
+## License
+
+MIT
+