ppgrid 0.2.2__tar.gz → 0.4.0__tar.gz

ppgrid-0.4.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  # PRs: full interpreter sweep on GitHub-hosted runners, pinned versions
+  # (requires-python >=3.11, classifiers 3.11-3.14; review m6).
+  test:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --all-extras --locked
+      - run: uv run --extra dev ruff check .
+      - run: uv run --extra dev ruff format --check .
+      - run: uv run pytest
+
+  # Pushes to main: GitHub-hosted, single leg (review m6: matrix is PR-only).
+  # 2026-10-08: was runs-on 'nf' — that label points at the
+  # nestfinderau/nestfinder REPO runners (label 'nestfinder-ci'); nothing
+  # named 'nf' is registered on the marzukia account, so the first main
+  # push (#39 merge) queued 9.5h starved. Wire real self-hosted runners
+  # to this account before switching back.
+  test-main:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --all-extras --locked
+      - run: uv run --extra dev ruff check .
+      - run: uv run --extra dev ruff format --check .
+      - run: uv run pytest

ppgrid-0.4.0/.github/workflows/publish.yml

@@ -0,0 +1,43 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Build and check the upload, but do not publish"
+        type: boolean
+        default: false
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+      - run: uv sync --locked -q
+      # Fail fast when a release tag does not match the pyproject version
+      # (before: an opaque uv publish "File already exists", or a
+      # workflow_dispatch publishing whatever version is checked out).
+      - name: Guard release tag matches pyproject version
+        if: github.event_name == 'release'
+        env:
+          TAG_NAME: ${{ github.event.release.tag_name }}
+        run: |
+          ver=$(uv run python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
+          if [ "$TAG_NAME" != "v$ver" ]; then
+            echo "::error::release tag '$TAG_NAME' does not match pyproject version '$ver' (expected 'v$ver'); aborting before upload"
+            exit 1
+          fi
+          echo "release tag $TAG_NAME matches pyproject version $ver"
+      - run: uv build
+      - name: Publish
+        env:
+          DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+        run: |
+          if [ "$DRY_RUN" = "true" ]; then
+            uv publish --dry-run --token ${{ secrets.PYPI_TOKEN }}
+          else
+            uv publish --token ${{ secrets.PYPI_TOKEN }}
+          fi

{ppgrid-0.2.2 → ppgrid-0.4.0}/.gitignore

@@ -8,3 +8,5 @@ __pycache__/
 .DS_Store
 **/.DS_Store
 out/
+# Full-AU sparse point set (53 MB, generated; not shipped in the sdist either)
+data/au_gcc_sparse.csv

ppgrid-0.4.0/AGENTS.md

@@ -0,0 +1,165 @@
+# AGENTS.md
+
+Working notes + rules for AI agents (and humans) touching this repo. Read this before editing.
+
+## What this is
+
+`ppgrid` — pull-push mipmap scattered-data interpolation. Turns `N` geolocated points
+into a pair of int16 GeoTIFFs (value percentile + support-km) at continent scale in
+minutes, no GPU. It is an **IDW approximation**: fast, visually clean, and
+deliberately *not* a spatially exact interpolator (the output is for visualisation,
+not downstream calculation).
+
+- Python 3.11+, `uv` managed. Entry point `ppgrid` (`ppgrid.pipeline:main`; `ppgrid.idwgrid` is a back-compat shim).
+- Deps: numpy, pandas, pyproj, rasterio. Optional `[parquet]` (pyarrow).
+- Tests: `uv run pytest` (190 tests). Lint/format: `ruff` (line-length 120).
+
+## Repo layout
+
+- `ppgrid/pipeline.py` — the Pipeline + CLI. All the logic lives here. `ppgrid/idwgrid.py` is a thin back-compat shim (module alias to `ppgrid.pipeline`).
+- `ppgrid/calibrate.py` — transform selection + spatially-blocked CV fill-cap.
+- `data/` — example inputs: `melb_houses.csv` (13,580 pts), `all_equakes.csv` (44,376 pts).
+- `examples/` — committed outputs + repro scripts + README images (see "Images" below).
+- `tests/` — pytest. `examples/run_melb.py` regenerates the Melbourne example rasters.
+
+## Versioning / release
+
+- `pyproject.toml` `version` is the source of truth; `__version__` reads it via
+  `importlib.metadata`. Keep `CHANGELOG.md` in step with every version bump.
+- Releasing: bump `version` + `uv.lock`, update CHANGELOG, push a tag, the
+  `publish.yml` workflow uploads to PyPI. The README PyPI badge shows the *published*
+  version, so it lags the repo until a tag is cut.
+
+## Rules — do not fuck these up
+
+### Images in the README (this bit bites)
+- The README is the PyPI `long_description`. **Every image MUST be an absolute
+  `https://raw.githubusercontent.com/marzukia/ppgrid/main/examples/...` URL.**
+  Relative `examples/...` paths render on GitHub but 404 on PyPI's static host.
+- README images are committed under `examples/`. They resolve from `main`, so a
+  new image is invisible on the repo's README **until the branch merges to main**.
+- The benchmark charts (`bench_headtohead.png`, `bench_scaling.png`) are generated
+  with **`charted`** (github.com/marzukia/charted — zero-dep Python SVG/PNG). Regenerate,
+  don't hand-edit:
+  ```bash
+  uvx --from "charted[png]" python  # build BarChart / LineChart, .to_png_bytes()
+  ```
+  Data for those charts lives in the blog post
+  (`mrzk.io` ppgrid post, `fig-bar-benchmark.json` / `fig-line-scaling.json`).
+
+### The `--percentile-step` guard
+- `pipeline.py` uses `if pct_step is not None:` (NOT `if pct_step:`). A step value of
+  `0` is invalid (validated to `(0, 100]`) but the `is not None` form is the correct
+  guard and matches the docstring. Do not "simplify" it back to a truthiness check.
+
+### CRS defaults
+- Working CRS `EPSG:6933` (Wagner VII, equal-area, metres are true) for interpolation.
+- Output CRS `EPSG:3857` (Web Mercator) for the final GeoTIFF.
+- Input `EPSG:4326`. All three are CLI-overridable (`--work-crs`, `--out-crs`,
+  `--src-crs`). Don't hardcode a different default.
+
+### Output encoding is opinionated — leave it
+- Value band = **percentile, not raw value**: `percentile = DN / scale`, default
+  scale 100 so DN 5000 = 50th percentile. int16 fits 0-10000. Raw values are restored
+  via the `calibration.json` LUT. This is deliberate (clean vectorisation/MVT, rounds
+  to whole numbers). If you "fix" it to store raw values you break the int16 design
+  and the decoding example in the README.
+
+### Data provenance
+- `melb_houses.csv`: Kaggle Melbourne Housing Snapshot, **CC BY-NC-SA 4.0**.
+- `all_equakes.csv`: USGS FDSN, **Public Domain**.
+- Keep the Data Lineage table in the README accurate if the equake download window
+  or filters change.
+
+## Git / pushing
+
+- The repo lives under the `marzukia` GitHub org. The `monkytheluffy` bot identity is
+  NOT a collaborator (push 403s). Push with the `marzukia` PAT:
+  ```bash
+  PAT=$(cat ~/.config/marzukia-pat)
+  git push "https://x-access-token:${PAT}@github.com/marzukia/ppgrid" <branch>
+  ```
+- Branches go to a PR, not main, per the fleet merge gate.
+
+## What changed recently (2026-10-08, sparse-output branch reconciled onto main post-#50)
+
+- PR #27 (`sparse-output-lowram`) merged `origin/main` (0.3.0, turbo) and reconciled: **main's turbo behaviour is the baseline; #27's low-RAM optimisations layer on top.**
+  - `_reproject_band` gained `task_info` (work-grid block geometry + populated-block set from `grid()`): a coarse 2048px dst tile whose source footprint intersects no task block is written as a plain NoData buffer without a GDAL warp (conservative round-out, so never a false empty). File-source serial reproject only; the turbo array reproject (`_reproject_band_array`) is untouched.
+  - `_descent_banded` (main's structure: `_thread_bands` + `_descent_band_body` per-thread buffers) gained the #27 per-level RAM gate: pool clamped to `min(n_threads, nbands, 8)`, and single-thread when `/proc/meminfo` MemAvailable < peak RSS + two bands of f32 buffers. `nworkers` is an alias of `n_threads` (both test suites call it).
+  - Non-turbo (low-RAM) shared descent is now parallel by design: `--max-band-parallel` caps the pool, default `workers` (main's baseline for this path was serial `n_threads = 1`). Turbo regime A still never calls `_descent_banded` (full in-RAM descent) — unchanged.
+  - Bit-identity holds at every thread count (S3 guarantee; `tests/test_pullpush_mt.py` + the #27 bit-identical/pool/RAM-gate tests all green). 180 (main) + 10 (#27) = 190 tests.
+
+## What changed recently (2026-10-06→08, turbo speed passes 2/3 + 0.3.0 release)
+
+- **Speed passes 2/3 (0a6fe90 + 941f4e5, both in the #39 merge history): full-AU 50:46 → 2:33, byte-identical** (regime A, `--turbo 64`, 43 GB; per-box 8.5 GB: 3:20; all outputs sha256-identical to the serial anchors). Pass 2 (0a6fe90) is the dnfill/write-path base; pass 3 (941f4e5) hit the wall target.
+  - Regime A keeps the shared full fields + quantised DN fields on tmpfs memmaps (reclaimable clean file cache) instead of anon RAM; `_pull_push_descent(out_val=/out_sup=)` writes the final descent level straight into the shared fields (bit-exact, threaded and serial branches).
+  - Parallel stock writer (`_turbo_write_stock_parallel`): fixed three silent serial-fallback bugs — TID scratch collision (`get_ident() & 0xFFFF`), block-major vs global 512 raster-scan tile order (608 vs 616 tiles), 3 GB zero-placeholder reference head (now one 512² tile).
+  - Reproject: one no-copy `MemoryDataset(copy=False)` + MultiBand tuple form, shared read-only across warp threads (rasterio's ndarray form copied 3.1 GB per warp call per thread; 8 in-flight copies OOM-killed the run).
+  - `_quantize` in-place (same IEEE ops/order, bit-identical), per-thread f32/f64 scratch, owned int16 outputs (the shared thread-local scratch raced the `ex.map` prefetch and corrupted ~10% of blocks).
+  - Support-blend fix: `om` buffer reused across blends — computed `(1-a)*pv*ps` instead of `(1-a)*ps`. Correctness fix on the default multi-level path: `support_km.tif` for multi-level grids may differ vs 0.2.2 (now correct), `value.tif` unaffected.
+  - `MADV_NOHUGEPAGE` on large fresh arrays (THP=always + sync defrag made first-touch faults 20-40 us/page).
+  - Known state: on this GDAL stack the zstd oracle mismatches libtiff streaming, so turbo runs take the byte-exact stock-parallel write fallback; the parallel ZSTD path is unit-tested and activates where the stack matches.
+- **Budget semantics (measured):** `--ram-gb` caps the shared-field budget, NOT process RSS — regime-C runs peaked at 39.4 GB under a 20 GB budget (write/descent buffers sit outside Bt sizing). per_box is the only genuine low-RAM mode.
+- **0.3.0 released 2026-10-08** (tag v0.3.0 @ f9e0380): CHANGELOG consolidated into one [0.3.0] entry; release notes live on the GitHub release; `publish.yml` publishes to PyPI on `release: published`. Issues #29-#33 closed as landed in #39.
+- Review findings from the #39 adversarial review are filed as #40-#45 (all minor, post-release): stale-output warning gap (#40), odd `band_rows` off-by-one (#41), `_turbo_zstd_ok` docstring (#42), dead `downsample_sum(out=)` param (#43), WIP-labelled 0a6fe90 in history (#44), DN temp-file hygiene vs `_open_fresh_memmap` guard (#45).
+
+## What changed recently (2026-10-07, PR #39 review fixes: M-1..M-3, Y-1)
+
+- M-1: equakes anchors regenerated under the rasterio 1.5.1 lock (the committed
+  files were cut on 1.5.0; the vendored GDAL emits different BigTIFF ZSTD tile
+  bytes — arrays equal, tile offsets moved). New shas in the A.6 table:
+  `examples/equakes/value.tif` `99c8a01c…`, `support_km.tif` `5e545742…`.
+  `bench/compare_tier2.py` re-run: bit-identical (melb anchors unchanged).
+- M-2: `turbop.resolve_cap` clamps a preset against `min(physical, cgroup)`
+  (a capped slice OOMs at its own limit); the pipeline wires both reads via
+  `Pipeline._resolve_turbo_cap` (before: preset 32 on a 25.8 GB slice planned
+  32 GB; now clamps to 17.8 GB).
+- M-3: `turbop.per_box_peak_bytes` (64 B/cell bottom-up + 2 GB fixed, x
+  in-flight blocks + points/DN extra) + `precheck(per_box_peak_bytes=…)`;
+  the pipeline computes it from the worst snapped box after `grid()`.
+  Non-strict per-box over budget now warns "best-effort, OOM risk, output may
+  be partial" instead of promising exit-0 correct output; strict names the
+  per-box infeasibility. Verified: melb @1m --max-ram 8 warns 189.7 GB est >
+  6.8 GB budget, then dies 137.
+- Y-1: BigTIFF slot widths in the parallel ZSTD write. `_tif_parse_ifd` sizes
+  tags per TIFF type (type 16 LONG8 added; unknown type = hard error) and
+  returns `sz324/sz325/fmt324/fmt325`; `_turbo_write_parallel` patches 324
+  (8-byte slots) and 325 (4-byte slots) at their real strides, shared
+  `_TIFF_TYPE_SIZES` with the overlap guard. Pre-fix, one 8-byte stride
+  corrupted both arrays on every real BigTIFF output (7888 tiles: 325 sizes
+  read `2000, 0, 2001, 0, …`). Pinned: BigTIFF parallel byte-identity test +
+  IFD parse value asserts.
+- Tests: 170 -> 176 (resolve_cap cgroup clamp, per-box peak model + gate,
+  pipeline wiring, BigTIFF parallel, IFD value pins).
+
+## What changed recently (2026-10-07, turbo write path + CLI, issue #33)
+
+- CLI: `--turbo` (optional preset `16|32|64|128`; bare = auto cap), `--max-ram <GB>` (implies turbo), `--ram-gb` (alias of `--max-ram`), `--turbo-strict`. Pre-check after `grid()` prints cap/budget/regime used; shared infeasible (per-box regime) -> strict: stderr error + exit 3, else `[warn]` + continue on per-box with turbo warp/write (exit 0, correct output).
+- `_write_rasters` turbo path: no work-CRS intermediate files. Quantised int16 DN fields stay in RAM (budget-gated: `nx*ny*4 > budget//2` -> file-based reproject path with MT warp) and are reprojected tile-by-tile through the same GDAL warp kernel as the serial path (`_reproject_band_array`; same 2048px tiles, same 512px serial flush, same metadata) -> byte-identical output, sha256-tested.
+- Parallel per-tile ZSTD (`_turbo_write_parallel`): GDAL zero-filled reference head serialises the exact IFD/tag layout; the pipeline patches TileOffsets/TileByteCounts + oracle-verified zstd frames. Oracle mismatch (e.g. CPL zstd pfn vs libtiff streaming params differ on a stack) -> byte-exact stock-codec parallel write (the serial per-band write is the last resort only if that writer's layout guards fail).
+- Budget-derived shared cap (S3.3): turbo mode derives the cells cap from `Bt = 0.85*C - 4 GB`; non-turbo keeps `_SHARED_MAX_CELLS = 2.5e8`; the `1e8` in-RAM/memmap tier is unchanged.
+- `_prepare_shared` / `box_count_banded` run threaded under the budgeted worker count when turbo (`plan.workers`); `n_threads == 1` keeps the exact serial path (A8).
+- `turbop._read_cgroup_max_gb` fixed: walks the process cgroup hierarchy to the root and takes the smallest finite `memory.max` (a slice limit above the service cgroup binds). Previously only the root cgroup was read, so a memory-capped user slice saw full physical RAM and auto `--turbo` could plan a regime that OOMs the slice.
+- `examples/melb/10m/value.tif` regenerated: the committed anchor predated a pipeline change; current `main` and this branch produce the new bytes (verified against pre-turbo HEAD).
+- Tests: `tests/test_turbo.py` now 22 (A9 regime pins, strict exit 3, non-strict continue, CLI flags/wiring, array-vs-file reproject sha, parallel-zstd byte-identity, 4GiB guard, IFD classic+BigTIFF parse, e2e in-RAM identity, budget-gate fallback, out==work CRS, `box_count_banded` MT, `test_cgroup_reader` walk-up).
+
+## What changed recently (2026-10-06, turbo read-side MT, 0.3.0)
+
+- New `Pipeline(n_threads=...)` kwarg (default `1`, serial path unchanged, byte-identical output). NOT a CLI flag.
+  - `_reproject_band`: 2048px warp tiles per row band run on a `ThreadPoolExecutor` via the module-level `_warp_dst_tile` helper (per-tile source open: GDAL handles are not thread-safe). The 512px flush loop STAYS serial raster-scan (file bytes depend on tile order). Do not "parallelise" the flush.
+  - `ingest`: CSVs >= 100k rows per chunk split into complete-record byte ranges (`_csv_chunk_tasks`, newline scan + quote-parity guards, `None` on any doubt -> serial parse) and parsed in a `ProcessPoolExecutor` (`_read_csv_chunk`, explicit float64 dtypes + per-chunk dtype assert). Chunks concat in original row order (`bin_points` is order-sensitive); row-count mismatch re-runs the serial parse.
+  - rasterio's benign `NotGeoreferencedWarning` is ignored at import (CPython warning filters are process-global, not thread-safe; the library's nested suppression leaks under concurrent warps). A matching `filterwarnings` entry in `[tool.pytest.ini_options]` restates it for pytest.
+  - Tests: `tests/test_turbo.py` (6, sha256/bit-identity + fallbacks). Bench: `bench/turbo_bench.py`, `bench/turbo_decomp.py`.
+  - Design doc in-repo: `docs/turbo-design.md` (accepted criteria A4-A8).
+
+## What changed recently (2026-09-22, the "polish" pass)
+
+- Bumped to **0.2.1**.
+- `--percentile-step` flag (round output percentiles to a step, e.g. 5 → 90/95/100)
+  + the `is not None` guard fix.
+- README: added hero image (`examples/melb_4panel.jpg`), replaced the old single
+  wall-time-vs-resolution chart with two `charted` benchmark charts (head-to-head vs
+  gdal at 100K pts + wall-time-vs-point-count scaling), added side-by-side/zoom
+  comparison images, fixed the clone URL (`pullpush` → `ppgrid`). Removed stale
+  `bench_combined/time/size.png`.
+- All README image refs converted to absolute raw.githubusercontent URLs for PyPI.

ppgrid-0.4.0/CHANGELOG.md

@@ -0,0 +1,101 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.4.0] - 2026-10-08
+- Performance (sparse output, low-RAM scope): `_reproject_band` skips the reproject warp for coarse output tiles whose work-CRS source footprint (conservatively rounded out) intersects no task block, writing a plain NoData buffer instead — RAM-free, and a no-op when no tile is skippable (the auto-cap case). The skip is guarded end-to-end against a no-skip reference with vertically asymmetric tasks (bit-identical; `tests/test_reproject_skip.py`). `_descent_banded` row-bands within a level now run on a thread pool (levels stay sequential): bit-identical output, `--max-band-parallel` flag (default `min(workers, nbands, 8)`), and a per-level `/proc/meminfo` gate that falls back to single-threaded when available RAM is below peak RSS plus two bands of float32 buffers.
+- Write-path hardening (production polish round):
+  - Atomic finals: `value.tif`/`support_km.tif` publish only via `.tif.tmp` rename (serial + turbo) — a mid-write crash leaves the previous run's finals byte-identical and warns (#40).
+  - Stale-output warning on any non-zero exit over pre-existing rasters in the out dir; reviewer repro pinned as a test (#40).
+  - All 4 unguarded `w+` memmap sites routed through `_open_fresh_memmap` (new shared `ppgrid/_tempfiles.py`) (#45).
+  - `_pull_push_descent` `_band_bufs` off-by-one for odd `band_rows` (+4 -> +5) with bit-exact MT regression test (#41).
+  - Dead `downsample_sum(out=)` parameter removed (#43).
+  - ZSTD oracle mismatch warning unambiguous ("bytes differ at tile N"), names the stock PARALLEL writer as the fallback.
+- CI: PRs on a pinned 3.11-3.14 matrix (ubuntu-latest); main single leg; publish.yml guard — release tag must match pyproject version before upload; `workflow_dispatch` dry_run input -> `uv publish --dry-run`.
+- Tests: 176 -> 190 (stale/mid-write/odd-band regressions + low-RAM bit-identity guards incl. the end-to-end asymmetric-tasks repro).
+
+## [0.3.0] - 2026-10-08
+- Full-AU (16M points, `--res 100`): 30:45 (quiet) to 50:46 (loaded) with the same pre-PR turbo code, load-dependent -> 2:33 wall time (~12-20x) with sha256-identical outputs (marzuki-hydrogen, `--turbo 64`, regime A).
+- Turbo: write path + budget cap + CLI (issue #33):
+  - `--turbo` (optional preset `16|32|64|128`; bare = auto `min(physical, cgroup) - 8 GB`), `--max-ram <GB>` (implies turbo), `--ram-gb` alias, `--turbo-strict`.
+  - Pre-check after `grid()`: prints the cap/budget/regime actually used; shared infeasible (per-box regime) -> strict: error + exit 3, else `[warn]` + continue on the per-box path with turbo warp/write (exit 0, correct output).
+  - `_write_rasters` turbo: skips the work-CRS intermediate files, keeps the quantised int16 DN fields in RAM, and reprojects tile-by-tile from arrays through the same GDAL warp kernel (byte-identical to the file-based path, sha256-tested). Optional parallel per-tile ZSTD via `zstdmt.compress_tiles`, gated by the compression oracle (mismatch -> byte-exact stock-codec parallel write; the serial per-band write is the last resort only if that writer's layout guards fail). In-RAM DN is budget-gated; oversized grids fall back to the file-based reproject path with MT warp.
+  - Budget-derived shared cap (S3.3) replaces the hard 2.5e8 cell cap in turbo mode; non-turbo keeps 2.5e8; the 1e8 in-RAM/memmap tier is unchanged.
+  - `_prepare_shared` / `box_count_banded` run threaded under the budgeted worker count when turbo (serial at `n_threads == 1`, A8 preserved).
+  - Fixed `turbop._read_cgroup_max_gb`: now walks the process cgroup hierarchy to the root and takes the smallest finite `memory.max` (a slice limit above the service cgroup binds; previously only the root cgroup was read, so capped user slices saw the full physical RAM and the auto cap could plan a regime that OOMs the slice).
+  - Regenerated stale anchor `examples/melb/10m/value.tif` (byte-identical to current `main` output; the committed file predated a pipeline change).
+  - RAM budget `Bt = 0.85 * C - 4 GB` (`C` = effective RAM cap) drives regime selection: A = full in-RAM shared, B = input in RAM + banded val/sup memmap, C = shared all-memmap banded descent (input + val/sup file-backed memmap, 8 GB page-cache floor in its peak accounting), per_box = per-box warp/write fallback taken when regime C has < 2 in-flight units (the only genuine low-RAM mode). `--ram-gb`/`--max-ram` cap this shared-field budget, not process RSS.
+- Turbo: speed pass 3 — regime-A in-RAM shared path + byte-exact parallel writer (issue #39):
+  - Regime A keeps the shared full fields (val_full/sup_full) and the quantised DN fields (val_dn/sup_dn) on tmpfs memmaps (reclaimable as clean file cache) instead of anon RAM; `_pull_push_descent` gained optional `out_val/out_sup` and writes the final descent level straight into the shared fields (bit-exact in both the threaded and serial branches).
+  - Byte-exact parallel tile writer: full-thread-id TID scratch paths (low-16-bit id collisions truncated a live scratch file mid-run), tile frames reassembled into global 512 raster-scan (block-major assembly was wrong), reference head written as one 512^2 tile. sha256-identical to the committed serial anchors.
+  - Reproject: the source band is wrapped once in a no-copy `MemoryDataset` — rasterio's ndarray source form copied the whole band into a fresh MEM dataset per warp call (3 GB per 2048^2 block at full-AU; 8 in-flight copies OOM-killed the run).
+  - `_quantize` rewritten in-place (same IEEE ops/order, bit-identical) with per-thread f32+f64 scratch; the int16 outputs are owned per call (thread-local scratch raced the `ex.map` prefetch and corrupted ~10% of blocks).
+  - Fixed the support blend reusing the value `om` buffer across blends (computed `(1-a)*pv*ps` instead of `(1-a)*ps`). User-visible: `support_km.tif` for multi-level grids may differ vs 0.2.2 (now correct); `value.tif` unchanged. `MADV_NOHUGEPAGE` on large fresh arrays (THP=always with sync defrag made first-touch faults 20-40 us/page).
+- Turbo: pipeline read-side MT (issue #32) behind a new `Pipeline(n_threads=...)` kwarg (default `1`: the serial code path, byte-identical output):
+  - `_reproject_band`: the 2048px warp tiles of each row band run on a `ThreadPoolExecutor` (the GDAL warp runs under `nogil`; each tile opens its own source handle, GDAL dataset handles are not thread-safe). The 512px flush loop stays serial raster-scan, so the on-disk bytes never change (sha256-equal to the serial run is test-covered).
+  - `ingest`: CSVs with enough rows (>= 100k per chunk) are split into complete-record byte ranges (newline scan with quote-parity guards; any doubt falls back to the serial parse) and parsed in a `ProcessPoolExecutor` (`read_csv` holds the GIL, so threads cannot overlap the parse). Each chunk parses the wanted columns with explicit float64 dtypes and asserts per-chunk dtype equality; chunks concatenate in original row order (`bin_points` is order-sensitive). A row-count mismatch re-runs the serial parse.
+  - New `tests/test_turbo.py` (6 tests): MT reproject sha256-equality vs serial, MT ingest bit-identity vs the single whole-file parse on a 500k-row edge-row fixture (2**53 +/- 1, empty/inf rows, int-only column, quoted fields, non-adjacent columns), serial-parse fallbacks (small file, quoted embedded newline), end-to-end full-run sha256 equality `n_threads=1` vs `4`, and `n_threads` validation.
+  - Ignores rasterio's benign `NotGeoreferencedWarning` (MemoryDataset initial transform read): CPython warning filters are process-global and not thread-safe, so the library's nested suppression can leak under concurrent warps.
+  - Bench harnesses: `bench/turbo_bench.py` (full paths), `bench/turbo_decomp.py` (parallelisable part in isolation).
+
+## [0.2.2] - 2026-09-25
+- Code hygiene (no behavior change; output rasters remain byte-identical) — issues #4/#5:
+  - Renamed `ppgrid/idwgrid.py` -> `ppgrid/pipeline.py`; `ppgrid/idwgrid` is now a thin backwards-compat shim, `ppgrid.Pipeline` is exported from the package root, and the `ppgrid` console script points at `ppgrid.pipeline:main`.
+  - Extracted the nested `_reproject_band` closure to a module-level function.
+  - Magic numbers in `pipeline.py`, `calibrate.py`, `pullpush.py` promoted to named constants (CRS defaults, int16 support-band encode/decode pair, tile size, memmap band indices, CLI defaults, float32 exact-integer limit, count epsilon, unresolved-support sentinel).
+  - Worker context: `_CTX` now holds the `_WorkerConfig` dataclass object (no per-field dict copy); adding a field is one edit.
+  - Deduplicated the 3x3-block neighbourhood / window / snapped-halo-box logic into shared helpers used by the per-block, shared-field, and test paths.
+  - Dropped the unused 4th point-memmap band.
+  - CLI: `--out` now defaults to `out/` (was `examples/`, which wrote build artifacts into the docs directory); argparse validation errors tested per flag.
+  - `ruff`: removed the `global-statement` ignore (no global statements left); COM812 remains ignored (pre-existing, conflicts with the formatter).
+- Performance (issue #10): vectorized the output-band reproject warp; output byte-identical.
+- Performance/correctness: shared full-field pyramid descent + fast box-count, calibrate hoisted out of the block loop; fixed `_CTX` not being cleared between runs; capped the shared path at 2.5e8 cells; stopped memmap descent at level `min(2, levels)` for levels=1 grids.
+- Release/docs pass (issues #17/#19/#20/#22): slimmed the sdist (generated example rasters/images and the 8.3 MB `data/all_equakes.csv` no longer ship; ~87 MB -> <1 MB, `data/melb_houses.csv` still ships), added the ruff check + format gate to CI, documented the Python 3.11+ requirement in the README install section, fixed stale docs (test count, module names, closed PR ref), removed stray `examples/melb/value.tif.aux.xml`, untracked `.vscode/settings.json`, added `out/` to `.gitignore`.
+
+## [0.2.1] - 2026-09-22
+- Added `--percentile-step` CLI flag: rounds output percentiles to the nearest step (e.g. `5` -> 90/95/100), clamped to 0-100. Recorded as a `percentile_step` tag on the value GeoTIFF.
+- Fixed `percentile_step` guard: `if pct_step:` -> `if pct_step is not None:` so a falsy step is handled correctly and the validation is consistent with the docstring.
+- README: added hero image (multi-resolution output), replaced the single wall-time-vs-resolution benchmark with a head-to-head vs `gdal_grid`/`gdal_rasterize` (100K points) and a wall-time-vs-point-count scaling chart (generated with `charted`), plus side-by-side comparison images. Fixed the source-clone URL (`pullpush` -> `ppgrid`).
+
+## [0.2.0] - 2026-08-08
+- Added `__version__` to package.
+- Moved `pyarrow` to optional `[parquet]` dependency.
+- Added `CHANGELOG.md`.
+- Added `SECURITY.md` and `SUPPORT.md`.
+- Added `CODEOWNERS` and pre-commit config.
+- Added `examples/run_melb.py` reproducible example script.
+- Added CI and PyPI badges to README.
+- Stated benchmark hardware in README.
+- Fixed README project structure and CLI table.
+- Added `python` keyword and Python 3.14 classifier.
+- Fixed emdash in pyproject.toml description.
+- Regenerated `uv.lock`.
+
+## [0.1.4] - 2026-08-08
+- Fixed PyPI project URLs to point to the renamed GitHub repository (`ppgrid`).
+- Added homepage link between PyPI and GitHub.
+
+## [0.1.3] - 2026-08-08
+- Fixed error message tuple (trailing comma in `pullpush.py`).
+- Removed dead `_init_worker` function.
+- Added index validation to `bin_points`.
+- Added 4 acceptance tests (blocked=whole, coverage, georef, validity).
+- Fixed README grammar, install section, structure.
+- Removed `.DS_Store` files from git.
+- Fixed all ruff warnings and errors.
+
+## [0.1.2] - 2026-08-08
+- Re-published after making repository public for README images.
+
+## [0.1.1] - 2026-08-08
+- Fixed README image paths for PyPI rendering.
+
+## [0.1.0] - 2026-08-08
+- Initial release of ppgrid.
+- Fast continent-scale raster interpolation for scattered point data.
+- IDW interpolation with pull-push mipmap pipeline.
+- Windowed reprojection (no OOM on large datasets).
+- CLI with validation and --version flag.
+- Calibrate module for transform selection + cross-validation.
+- 17 tests across 3 test files.
+- Melbourne housing example (13.5k points at 6 resolutions).

{ppgrid-0.2.2 → ppgrid-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ppgrid
-Version: 0.2.2
+Version: 0.4.0
 Summary: Fast continent-scale raster interpolation for scattered point data using pull-push mipmap algorithms. Turns millions of geolocated points into gapless GeoTIFF in minutes. No GPU required.
 Project-URL: Homepage, https://github.com/marzukia/ppgrid
 Project-URL: Repository, https://github.com/marzukia/ppgrid
@@ -49,7 +49,7 @@ A single 100K-point dataset rendered at four resolutions. The adaptive mipmap py
 
 You have `N` points with `(longitude, latitude, value)`. You want a raster where every cell within a specified distance of real data carries an interpolated value, and everything else is nodata.
 
-Standard IDW in QGIS or ArcGIS is `O(N*M)` (~14 hours for 16M points). This tool uses pull-push mipmap interpolation to reduce cost to `O(M)`, independent of N.
+Standard IDW in QGIS or ArcGIS is `O(N*M)` (hours to days at 16M points, depending on the IDW implementation and machine). This tool uses pull-push mipmap interpolation to reduce cost to `O(M)`, independent of N.
 
 ## How it works
 
@@ -160,6 +160,25 @@ ppgrid data.csv --value-col premium --res 100 --cap-km 25 --percentile-step 5
 | `--work-crs` | `6933` | Working CRS for interpolation (equal-area) |
 | `--out-crs` | `3857` | Output CRS for final GeoTIFF |
 | `--skip-calibration` | False | Skip calibration, use defaults |
+| `--turbo` | (none) | Turbo mode: multithreaded pipeline sized to a RAM cap. Optional preset `16|32|64|128` (GB); bare `--turbo` = auto `min(physical, cgroup) - 8 GB` |
+| `--max-ram` | (none) | Explicit RAM cap in GB (overrides the `--turbo` preset). Implies `--turbo` |
+| `--ram-gb` | (none) | Alias of `--max-ram` |
+| `--turbo-strict` | False | Shared path infeasible under the budget: error + exit 3 instead of `[warn]` + per-box fallback (exit 0) |
+
+### Turbo mode
+
+`--turbo` enables the multithreaded pipeline. The RAM cap `C` (a preset, `--max-ram`, or bare `--turbo` auto-detect) drives a planning budget **`Bt = 0.85 * C - 4 GB`** and a regime, printed by a pre-check after gridding. None of the turbo settings change the output bytes.
+
+- **A** — full in-RAM shared fields. Fastest; needs the most RAM.
+- **B** — input in RAM, output bands file-backed with a limited number of descent bands in flight.
+- **C** — shared all-memmap banded descent: input and output bands file-backed (reclaimable memmap), with an 8 GB page-cache floor in its peak accounting.
+- **per_box** — per-box warp/write fallback, taken when regime C has fewer than 2 in-flight units. This is the only genuine low-RAM mode.
+
+When the shared path is infeasible under the budget, the run prints `[warn]` and continues on the per-box path (exit 0, correct output). `--turbo-strict` makes the infeasibility an error (exit 3).
+
+**`--ram-gb` caps the shared-field budget, not process RSS.** Write/descent buffers sit outside the `Bt` sizing, so a regime C run under `--ram-gb 20` can peak well above 20 GB RSS (measured: 39.4 GB). Only the per-box path is genuinely low-RAM (measured: 8.5 GB peak).
+
+**Byte identity.** Turbo output is sha256-identical to the serial path. The optional parallel per-tile ZSTD write is gated by a compression oracle: where the oracle mismatches (e.g. the reference GDAL 3.12.4 / libtiff 6.2 stack), the run uses the stock-codec **parallel** writer instead, which is byte-exact; the serial per-band write is the last resort only, reached if that writer's layout guards fail.
 
 ## Decoding the output
 
@@ -202,6 +221,19 @@ Melbourne Housing dataset (13,580 points) at various resolutions on the same mac
 | 250m | 0.6s | 159 KB |
 | 500m | 0.6s | 49 KB |
 
+### Turbo benchmarks (full AU)
+
+Full-Australia grid, 16M points, `--res 100 --cap-km 25 --skip-calibration`, on marzuki-hydrogen (Threadripper 24C/48T, 125 GB, CPU-only). All outputs sha256-identical across the pre- and post-turbo-code runs.
+
+| Mode | Wall | Peak RSS |
+|------|------|----------|
+| Pre-PR turbo code, regime A (`--turbo 64`, same code) | 30:45 (quiet) to 50:46 (loaded) | 41.7 GB |
+| Regime A (`--turbo 64`) | 2:33 | 43 GB |
+| per-box (24 GB cgroup slice) | 3:20 | 8.5 GB |
+| Regime C (`--ram-gb 20`) | 6:25 pre-PR, 3:08 post | 39.4 GB |
+
+The pre-PR regime-A wall varies 30:45-50:46 with machine load (two measured runs of the same code on the same box), so the speedup range is ~12-20x rather than a single ratio. Note the `--ram-gb 20` row: the cap bounds the shared-field budget, not RSS — see the caveat above; per-box is the only genuinely low-RAM mode.
+
 ## Example Outputs
 
 `ppgrid` against `gdal_grid` + IDW and `gdal_rasterize` on the same data. Left is `ppgrid`, middle is `gdal_grid` with IDW, right is `gdal_rasterize`:

{ppgrid-0.2.2 → ppgrid-0.4.0}/README.md

@@ -13,7 +13,7 @@ A single 100K-point dataset rendered at four resolutions. The adaptive mipmap py
 
 You have `N` points with `(longitude, latitude, value)`. You want a raster where every cell within a specified distance of real data carries an interpolated value, and everything else is nodata.
 
-Standard IDW in QGIS or ArcGIS is `O(N*M)` (~14 hours for 16M points). This tool uses pull-push mipmap interpolation to reduce cost to `O(M)`, independent of N.
+Standard IDW in QGIS or ArcGIS is `O(N*M)` (hours to days at 16M points, depending on the IDW implementation and machine). This tool uses pull-push mipmap interpolation to reduce cost to `O(M)`, independent of N.
 
 ## How it works
 
@@ -124,6 +124,25 @@ ppgrid data.csv --value-col premium --res 100 --cap-km 25 --percentile-step 5
 | `--work-crs` | `6933` | Working CRS for interpolation (equal-area) |
 | `--out-crs` | `3857` | Output CRS for final GeoTIFF |
 | `--skip-calibration` | False | Skip calibration, use defaults |
+| `--turbo` | (none) | Turbo mode: multithreaded pipeline sized to a RAM cap. Optional preset `16|32|64|128` (GB); bare `--turbo` = auto `min(physical, cgroup) - 8 GB` |
+| `--max-ram` | (none) | Explicit RAM cap in GB (overrides the `--turbo` preset). Implies `--turbo` |
+| `--ram-gb` | (none) | Alias of `--max-ram` |
+| `--turbo-strict` | False | Shared path infeasible under the budget: error + exit 3 instead of `[warn]` + per-box fallback (exit 0) |
+
+### Turbo mode
+
+`--turbo` enables the multithreaded pipeline. The RAM cap `C` (a preset, `--max-ram`, or bare `--turbo` auto-detect) drives a planning budget **`Bt = 0.85 * C - 4 GB`** and a regime, printed by a pre-check after gridding. None of the turbo settings change the output bytes.
+
+- **A** — full in-RAM shared fields. Fastest; needs the most RAM.
+- **B** — input in RAM, output bands file-backed with a limited number of descent bands in flight.
+- **C** — shared all-memmap banded descent: input and output bands file-backed (reclaimable memmap), with an 8 GB page-cache floor in its peak accounting.
+- **per_box** — per-box warp/write fallback, taken when regime C has fewer than 2 in-flight units. This is the only genuine low-RAM mode.
+
+When the shared path is infeasible under the budget, the run prints `[warn]` and continues on the per-box path (exit 0, correct output). `--turbo-strict` makes the infeasibility an error (exit 3).
+
+**`--ram-gb` caps the shared-field budget, not process RSS.** Write/descent buffers sit outside the `Bt` sizing, so a regime C run under `--ram-gb 20` can peak well above 20 GB RSS (measured: 39.4 GB). Only the per-box path is genuinely low-RAM (measured: 8.5 GB peak).
+
+**Byte identity.** Turbo output is sha256-identical to the serial path. The optional parallel per-tile ZSTD write is gated by a compression oracle: where the oracle mismatches (e.g. the reference GDAL 3.12.4 / libtiff 6.2 stack), the run uses the stock-codec **parallel** writer instead, which is byte-exact; the serial per-band write is the last resort only, reached if that writer's layout guards fail.
 
 ## Decoding the output
 
@@ -166,6 +185,19 @@ Melbourne Housing dataset (13,580 points) at various resolutions on the same mac
 | 250m | 0.6s | 159 KB |
 | 500m | 0.6s | 49 KB |
 
+### Turbo benchmarks (full AU)
+
+Full-Australia grid, 16M points, `--res 100 --cap-km 25 --skip-calibration`, on marzuki-hydrogen (Threadripper 24C/48T, 125 GB, CPU-only). All outputs sha256-identical across the pre- and post-turbo-code runs.
+
+| Mode | Wall | Peak RSS |
+|------|------|----------|
+| Pre-PR turbo code, regime A (`--turbo 64`, same code) | 30:45 (quiet) to 50:46 (loaded) | 41.7 GB |
+| Regime A (`--turbo 64`) | 2:33 | 43 GB |
+| per-box (24 GB cgroup slice) | 3:20 | 8.5 GB |
+| Regime C (`--ram-gb 20`) | 6:25 pre-PR, 3:08 post | 39.4 GB |
+
+The pre-PR regime-A wall varies 30:45-50:46 with machine load (two measured runs of the same code on the same box), so the speedup range is ~12-20x rather than a single ratio. Note the `--ram-gb 20` row: the cap bounds the shared-field budget, not RSS — see the caveat above; per-box is the only genuinely low-RAM mode.
+
 ## Example Outputs
 
 `ppgrid` against `gdal_grid` + IDW and `gdal_rasterize` on the same data. Left is `ppgrid`, middle is `gdal_grid` with IDW, right is `gdal_rasterize`:

ppgrid-0.4.0/bench/turbo_bench.py

@@ -0,0 +1,69 @@
+"""Turbo read-side MT (issue #32): speed evidence for reproject warp + ingest.
+
+Repro: uv run python -m bench.turbo_bench
+"""
+
+import tempfile
+import time
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import rasterio
+from rasterio.warp import calculate_default_transform
+
+from ppgrid.pipeline import NODATA, TILE_PX, Pipeline, _reproject_band
+from tests.test_turbo import _write_src_tif
+
+TMP = Path(tempfile.mkdtemp(prefix="turbo_bench"))
+
+
+def median(f, runs=3):
+    ts = []
+    for _ in range(runs):
+        t0 = time.perf_counter()
+        f()
+        ts.append(time.perf_counter() - t0)
+    return float(np.median(ts))
+
+
+def bench_reproject():
+    src = TMP / "src.tif"
+    _write_src_tif(src, 5000, 4000)
+    with rasterio.open(src) as ds:
+        t, w, h = calculate_default_transform(ds.crs, "EPSG:3857", ds.width, ds.height, *ds.bounds)
+    profile = {
+        "driver": "GTiff",
+        "dtype": "int16",
+        "nodata": NODATA,
+        "compress": "ZSTD",
+        "tiled": True,
+        "blockxsize": TILE_PX,
+        "blockysize": TILE_PX,
+        "predictor": 2,
+        "count": 1,
+    }
+    out = TMP / "out.tif"
+    t1 = median(lambda: _reproject_band(str(src), str(out), profile, "EPSG:3857", t, w, h))
+    t4 = median(lambda: _reproject_band(str(src), str(out), profile, "EPSG:3857", t, w, h, n_threads=4))
+    print(f"reproject band {w}x{h}: serial {t1:.3f}s, 4t {t4:.3f}s, speedup {t1 / t4:.2f}x")
+
+
+def bench_ingest(n_rows=1_000_000):
+    rng = np.random.default_rng(0)
+    csv = TMP / "big.csv"
+    pd.DataFrame(
+        {
+            "value": rng.normal(0.0, 100.0, n_rows),
+            "longitude": rng.uniform(110.0, 155.0, n_rows),
+            "latitude": rng.uniform(-50.0, -10.0, n_rows),
+        },
+    ).to_csv(csv, index=False)
+    t1 = median(lambda: Pipeline(str(csv), "value", "longitude", "latitude", str(TMP / "o1"), n_threads=1).ingest())
+    t4 = median(lambda: Pipeline(str(csv), "value", "longitude", "latitude", str(TMP / "o4"), n_threads=4).ingest())
+    print(f"ingest {n_rows} rows: serial {t1:.3f}s, 4t {t4:.3f}s, speedup {t1 / t4:.2f}x")
+
+
+if __name__ == "__main__":
+    bench_reproject()
+    bench_ingest()