datashader-browser 0.1.0a2__tar.gz

datashader_browser-0.1.0a2/.github/workflows/pages.yml

@@ -0,0 +1,49 @@
+name: Build browser demo
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v2
+        with:
+          init-shell: none
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - run: pip install -r requirements-build.txt
+      - name: Build JupyterLite site
+        run: >-
+          jupyter lite build
+          --XeusAddon.environment_file=environment.yml
+          --contents content
+          --output-dir dist
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: dist
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

datashader_browser-0.1.0a2/.github/workflows/release.yml

@@ -0,0 +1,48 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build release distributions
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - name: Verify tag matches package version
+        run: test "$(uv run python -c 'from datashader_browser import __version__; print(__version__)')" = "${GITHUB_REF_NAME#v}"
+      - name: Build wheel and source distribution
+        run: uv build --out-dir release-dist
+      - name: Validate package metadata
+        run: uv run --with twine twine check release-dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-distributions
+          path: release-dist/
+
+  publish:
+    name: Publish release to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datashader-browser
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: python-distributions
+          path: release-dist/
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: release-dist/

datashader_browser-0.1.0a2/.github/workflows/test.yml

@@ -0,0 +1,16 @@
+name: Test
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+      - run: uv sync --extra test --extra data
+      - run: uv run pytest

datashader_browser-0.1.0a2/.gitignore

@@ -0,0 +1,10 @@
+.venv/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.cache/
+dist/
+_output/
+.jupyterlite.doit.db
+.jupyterlite.doit.db.*

datashader_browser-0.1.0a2/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Sumaiya Islam
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

datashader_browser-0.1.0a2/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.5
+Name: datashader-browser
+Version: 0.1.0a2
+Summary: Bounded-memory Datashader pipelines for Python running in the browser
+Project-URL: Homepage, https://github.com/SuMayaBee/datashader-browser
+Project-URL: Issues, https://github.com/SuMayaBee/datashader-browser/issues
+Project-URL: Repository, https://github.com/SuMayaBee/datashader-browser
+Author: Sumaiya Islam
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Requires-Python: >=3.10
+Requires-Dist: datashader<0.20,>=0.19.1
+Requires-Dist: pandas>=2.2
+Provides-Extra: data
+Requires-Dist: pillow>=11; extra == 'data'
+Requires-Dist: pyarrow>=21; extra == 'data'
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# datashader-browser
+
+Run Datashader-powered visualizations entirely in a browser with Python and
+WebAssembly—without assuming that the complete dataset fits in browser memory.
+
+This is an early, independent project. It is not yet an official HoloViz
+project and is not affiliated with or endorsed by the Datashader maintainers.
+
+[Try the live browser demo](https://sumayabee.github.io/datashader-browser/).
+Open `streaming-points.ipynb` and run all cells.
+
+## What works now
+
+The first vertical slice is a bounded-memory point aggregator:
+
+```python
+import datashader as ds
+import datashader.transfer_functions as tf
+
+from datashader_browser import StreamingCanvas, iter_csv_chunks, iter_parquet_batches
+
+source = iter_csv_chunks(
+    "points.csv",
+    chunksize=250_000,
+    columns=["longitude", "latitude", "value"],
+)
+
+canvas = StreamingCanvas(
+    plot_width=1000,
+    plot_height=600,
+    x_range=(-180, 180),
+    y_range=(-90, 90),
+)
+aggregate = canvas.points(source, "longitude", "latitude", agg=ds.mean("value"))
+image = tf.shade(aggregate)
+
+# A local Parquet file can yield projected Arrow record batches directly.
+parquet_source = iter_parquet_batches(
+    "points.parquet", columns=["longitude", "latitude"], batch_size=250_000
+)
+aggregate = canvas.points(parquet_source, "longitude", "latitude", agg=ds.count())
+```
+
+Only one input chunk and Datashader's fixed-size aggregate buffers are live at
+once. The implementation calls Datashader's own compiler, so point binning and
+reductions match native Datashader instead of being reimplemented.
+
+Currently verified reductions are `count`, `sum`, `mean`, `min`, and `max`.
+Explicit `x_range` and `y_range` values are required to preserve a true
+single-pass pipeline.
+
+## NYC Taxi real-data verification
+
+The streaming accumulator successfully rendered all **11,842,094 pickup
+locations** in HoloViz's transformed
+[January 2015 NYC Taxi dataset](https://examples.holoviz.org/gallery/nyc_taxi/nyc_taxi.html).
+The 385 MB Parquet file was decoded as 48 Arrow/Pandas batches of at most
+250,000 rows; the complete table was never materialized as one DataFrame.
+
+![Datashader rendering of 11.8 million January 2015 NYC taxi pickups](docs/assets/nyc-taxi-pickups.png)
+
+One warm-cache Linux run produced the 1200-by-700 image in 3.24 seconds with
+514 MiB peak process RSS. Runtime and memory will vary by machine. Reproduce it
+with:
+
+```bash
+uv sync --extra data
+uv run python examples/nyc_taxi.py --download
+```
+
+This benchmark validates `StreamingCanvas` against a real large dataset on
+native Python. The browser build now includes an experimental PyArrow Parquet
+reader: open `nyc-taxi-browser.ipynb`, place the source file in the browser
+filesystem, and it streams projected Arrow record batches through the same
+accumulator. Full 385 MB end-to-end browser timing and memory results are still
+an explicit validation task, not a claim made by this repository.
+
+## Browser runtime
+
+The initial deployment target is JupyterLite with the Xeus-Python WebAssembly
+kernel. Its environment is described in [`environment.yml`](environment.yml)
+and uses the experimental Numba and llvmlite packages from emscripten-forge.
+The kernel executes in a Web Worker, keeping aggregation off the browser UI
+thread. The Parquet-capable build is currently about 256 MB because PyArrow
+brings compiled Arrow and compression dependencies; this is an engineering
+baseline, not the final delivery architecture. The first visit therefore needs
+a substantial download, while subsequent use benefits from the browser cache.
+
+Browser verification: `streaming-parquet.ipynb` has been run end-to-end in the
+Xeus-Python WebAssembly kernel. It created a 57.2 MB, two-million-row Parquet
+file in the browser filesystem, streamed it through 20 Arrow batches, checked
+the final row count, and rendered the shaded aggregate with no browser console
+errors. The full 385 MB NYC Taxi browser benchmark remains a separate task.
+
+```bash
+uv sync --extra test
+uv run pytest
+
+# Build the browser site (requires the build dependencies below).
+uv pip install -r requirements-build.txt
+uv run jupyter lite build \
+  --XeusAddon.environment_file=environment.yml \
+  --contents content \
+  --output-dir dist
+```
+
+## Large-data strategy
+
+“In the browser” does not mean that an unlimited dataset can be copied into a
+Pandas DataFrame. The project uses three execution levels:
+
+1. **Stream:** scan CSV or local Parquet/Arrow batches through a bounded-memory Datashader
+   accumulator. Total row count affects time, not peak aggregate memory.
+2. **Prune:** use Parquet row groups, spatial partitions, and DuckDB-Wasm to
+   read only the viewport and required columns.
+3. **Tile:** for billion-row interactive datasets, fetch spatial partitions or
+   precomputed multiresolution aggregates instead of rescanning the source on
+   every zoom.
+
+See [Architecture](docs/architecture.md), [Research](docs/research.md), and the
+[roadmap](ROADMAP.md) for the engineering decisions and current limitations.
+
+## Status
+
+This repository is an alpha-stage research implementation. The WebAssembly
+Numba/llvmlite stack works in current demonstrations but its patches have not
+yet been fully upstreamed. Production use should wait for reproducible browser
+builds, compatibility CI, and measured browser memory limits.
+
+## License
+
+BSD-3-Clause. Datashader is a separate BSD-3-Clause project owned by its
+respective copyright holders and contributors.

datashader_browser-0.1.0a2/README.md

@@ -0,0 +1,133 @@
+# datashader-browser
+
+Run Datashader-powered visualizations entirely in a browser with Python and
+WebAssembly—without assuming that the complete dataset fits in browser memory.
+
+This is an early, independent project. It is not yet an official HoloViz
+project and is not affiliated with or endorsed by the Datashader maintainers.
+
+[Try the live browser demo](https://sumayabee.github.io/datashader-browser/).
+Open `streaming-points.ipynb` and run all cells.
+
+## What works now
+
+The first vertical slice is a bounded-memory point aggregator:
+
+```python
+import datashader as ds
+import datashader.transfer_functions as tf
+
+from datashader_browser import StreamingCanvas, iter_csv_chunks, iter_parquet_batches
+
+source = iter_csv_chunks(
+    "points.csv",
+    chunksize=250_000,
+    columns=["longitude", "latitude", "value"],
+)
+
+canvas = StreamingCanvas(
+    plot_width=1000,
+    plot_height=600,
+    x_range=(-180, 180),
+    y_range=(-90, 90),
+)
+aggregate = canvas.points(source, "longitude", "latitude", agg=ds.mean("value"))
+image = tf.shade(aggregate)
+
+# A local Parquet file can yield projected Arrow record batches directly.
+parquet_source = iter_parquet_batches(
+    "points.parquet", columns=["longitude", "latitude"], batch_size=250_000
+)
+aggregate = canvas.points(parquet_source, "longitude", "latitude", agg=ds.count())
+```
+
+Only one input chunk and Datashader's fixed-size aggregate buffers are live at
+once. The implementation calls Datashader's own compiler, so point binning and
+reductions match native Datashader instead of being reimplemented.
+
+Currently verified reductions are `count`, `sum`, `mean`, `min`, and `max`.
+Explicit `x_range` and `y_range` values are required to preserve a true
+single-pass pipeline.
+
+## NYC Taxi real-data verification
+
+The streaming accumulator successfully rendered all **11,842,094 pickup
+locations** in HoloViz's transformed
+[January 2015 NYC Taxi dataset](https://examples.holoviz.org/gallery/nyc_taxi/nyc_taxi.html).
+The 385 MB Parquet file was decoded as 48 Arrow/Pandas batches of at most
+250,000 rows; the complete table was never materialized as one DataFrame.
+
+![Datashader rendering of 11.8 million January 2015 NYC taxi pickups](docs/assets/nyc-taxi-pickups.png)
+
+One warm-cache Linux run produced the 1200-by-700 image in 3.24 seconds with
+514 MiB peak process RSS. Runtime and memory will vary by machine. Reproduce it
+with:
+
+```bash
+uv sync --extra data
+uv run python examples/nyc_taxi.py --download
+```
+
+This benchmark validates `StreamingCanvas` against a real large dataset on
+native Python. The browser build now includes an experimental PyArrow Parquet
+reader: open `nyc-taxi-browser.ipynb`, place the source file in the browser
+filesystem, and it streams projected Arrow record batches through the same
+accumulator. Full 385 MB end-to-end browser timing and memory results are still
+an explicit validation task, not a claim made by this repository.
+
+## Browser runtime
+
+The initial deployment target is JupyterLite with the Xeus-Python WebAssembly
+kernel. Its environment is described in [`environment.yml`](environment.yml)
+and uses the experimental Numba and llvmlite packages from emscripten-forge.
+The kernel executes in a Web Worker, keeping aggregation off the browser UI
+thread. The Parquet-capable build is currently about 256 MB because PyArrow
+brings compiled Arrow and compression dependencies; this is an engineering
+baseline, not the final delivery architecture. The first visit therefore needs
+a substantial download, while subsequent use benefits from the browser cache.
+
+Browser verification: `streaming-parquet.ipynb` has been run end-to-end in the
+Xeus-Python WebAssembly kernel. It created a 57.2 MB, two-million-row Parquet
+file in the browser filesystem, streamed it through 20 Arrow batches, checked
+the final row count, and rendered the shaded aggregate with no browser console
+errors. The full 385 MB NYC Taxi browser benchmark remains a separate task.
+
+```bash
+uv sync --extra test
+uv run pytest
+
+# Build the browser site (requires the build dependencies below).
+uv pip install -r requirements-build.txt
+uv run jupyter lite build \
+  --XeusAddon.environment_file=environment.yml \
+  --contents content \
+  --output-dir dist
+```
+
+## Large-data strategy
+
+“In the browser” does not mean that an unlimited dataset can be copied into a
+Pandas DataFrame. The project uses three execution levels:
+
+1. **Stream:** scan CSV or local Parquet/Arrow batches through a bounded-memory Datashader
+   accumulator. Total row count affects time, not peak aggregate memory.
+2. **Prune:** use Parquet row groups, spatial partitions, and DuckDB-Wasm to
+   read only the viewport and required columns.
+3. **Tile:** for billion-row interactive datasets, fetch spatial partitions or
+   precomputed multiresolution aggregates instead of rescanning the source on
+   every zoom.
+
+See [Architecture](docs/architecture.md), [Research](docs/research.md), and the
+[roadmap](ROADMAP.md) for the engineering decisions and current limitations.
+
+## Status
+
+This repository is an alpha-stage research implementation. The WebAssembly
+Numba/llvmlite stack works in current demonstrations but its patches have not
+yet been fully upstreamed. Production use should wait for reproducible browser
+builds, compatibility CI, and measured browser memory limits.
+
+## License
+
+BSD-3-Clause. Datashader is a separate BSD-3-Clause project owned by its
+respective copyright holders and contributors.

datashader_browser-0.1.0a2/ROADMAP.md

@@ -0,0 +1,35 @@
+# Roadmap
+
+## 0.1: bounded-memory correctness
+
+- Point glyphs over iterables of Pandas chunks.
+- Count, sum, mean, min, and max parity with Datashader 0.19.1.
+- Progress and cancellation hooks.
+- Xeus-Python/JupyterLite environment using emscripten-forge Numba.
+- Browser build and deployment CI.
+
+## 0.2: browser data plane
+
+- Local Parquet record-batch source via PyArrow in the Xeus-Python worker.
+- DuckDB-Wasm worker for local and remote Parquet.
+- Column projection and viewport predicates.
+- Transferable buffers between the data worker and Python worker.
+- Generation IDs so obsolete pan/zoom jobs are cancelled or ignored.
+
+## 0.3: indexed exploration
+
+- Spatially partitioned Parquet manifest format.
+- Partition pruning by canvas bounds.
+- Cache recent partitions in OPFS.
+- Progressive low-resolution then full-resolution rendering.
+
+## Later
+
+- Lines with one-row partition overlap.
+- Categorical aggregation with explicit category budgets.
+- Panel/Bokeh integration outside the JupyterLab shell.
+- Optional GPU aggregation experiments after CPU parity and memory behavior are
+  characterized.
+
+Polygons, trimesh, quadmesh, Dask, CUDA, and arbitrary row-index reductions are
+outside the initial scope.

datashader_browser-0.1.0a2/content/README.md

@@ -0,0 +1,10 @@
+# Datashader Browser Lab
+
+Start with `streaming-parquet.ipynb` to create, stream, and render a Parquet
+dataset entirely in this browser tab. `streaming-points.ipynb` is the smaller
+in-memory-generated example. `nyc-taxi-browser.ipynb` contains the full
+11.8-million-row HoloViz NYC Taxi workflow and instructions for supplying its
+385 MB source file.
+
+The Xeus-Python kernel and all Python packages are WebAssembly assets served by
+the static site; no Python server is involved.

datashader_browser-0.1.0a2/content/nyc-taxi-browser.ipynb

@@ -0,0 +1,119 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# NYC Taxi: 11.8 million points in browser Python\n",
+    "\n",
+    "This notebook renders all January 2015 NYC Taxi pickup locations from the 385 MB HoloViz Parquet dataset. Upload or mount `nyc_taxi_wide.parq` in JupyterLite, or enable the download cell below. Parquet decoding and Datashader aggregation are bounded by the selected batch size; the complete table is never constructed as one DataFrame."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "from urllib.request import urlopen\n",
+    "\n",
+    "import pyarrow.parquet as pq\n",
+    "import datashader.transfer_functions as tf\n",
+    "\n",
+    "from datashader_browser import StreamingCanvas, iter_parquet_batches\n",
+    "\n",
+    "DATA_URL = 'https://datasets.holoviz.org/nyc_taxi/v1/nyc_taxi_wide.parq'\n",
+    "DATA_PATH = Path('nyc_taxi_wide.parq')\n",
+    "X_RANGE = (-8_256_000, -8_209_000)\n",
+    "Y_RANGE = (4_964_000, 4_990_000)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Set `DOWNLOAD = True` to fetch the public dataset. Mounting or uploading it through JupyterLite avoids repeating this 385 MB download on subsequent sessions."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DOWNLOAD = False\n",
+    "\n",
+    "if DOWNLOAD and not DATA_PATH.exists():\n",
+    "    with urlopen(DATA_URL) as response, DATA_PATH.open('wb') as output:\n",
+    "        while block := response.read(1024 * 1024):\n",
+    "            output.write(block)\n",
+    "\n",
+    "if not DATA_PATH.exists():\n",
+    "    raise FileNotFoundError(\n",
+    "        f'{DATA_PATH} is missing. Upload/mount it in JupyterLite or set DOWNLOAD=True.'\n",
+    "    )\n",
+    "\n",
+    "parquet = pq.ParquetFile(DATA_PATH)\n",
+    "print(f'{parquet.metadata.num_rows:,} rows; {DATA_PATH.stat().st_size / 1_000_000:.1f} MB')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def report(progress):\n",
+    "    if progress.chunks % 10 == 0 or progress.rows == parquet.metadata.num_rows:\n",
+    "        print(f'aggregated {progress.rows:,}/{parquet.metadata.num_rows:,} rows')\n",
+    "\n",
+    "canvas = StreamingCanvas(\n",
+    "    plot_width=1_200,\n",
+    "    plot_height=700,\n",
+    "    x_range=X_RANGE,\n",
+    "    y_range=Y_RANGE,\n",
+    ")\n",
+    "aggregate = canvas.points(\n",
+    "    iter_parquet_batches(\n",
+    "        DATA_PATH,\n",
+    "        batch_size=250_000,\n",
+    "        columns=['pickup_x', 'pickup_y'],\n",
+    "    ),\n",
+    "    'pickup_x',\n",
+    "    'pickup_y',\n",
+    "    on_progress=report,\n",
+    ")\n",
+    "aggregate.attrs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert aggregate.attrs['source_rows'] == parquet.metadata.num_rows\n",
+    "image = tf.shade(\n",
+    "    aggregate,\n",
+    "    cmap=['#111827', '#1d4ed8', '#06b6d4', '#facc15', '#fff7ed'],\n",
+    "    how='eq_hist',\n",
+    ")\n",
+    "tf.set_background(image, '#050814').to_pil()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (XPython)",
+   "language": "python",
+   "name": "xpython"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}