datashader-browser 0.1.0a2__py3-none-any.whl

datashader_browser/__init__.py

@@ -0,0 +1,11 @@
+"""Run Datashader over bounded-memory streams in a browser Python runtime."""
+
+from .streaming import Progress, StreamingCanvas, iter_csv_chunks, iter_parquet_batches
+
+__all__ = [
+    "Progress",
+    "StreamingCanvas",
+    "iter_csv_chunks",
+    "iter_parquet_batches",
+]
+__version__ = "0.1.0a2"

datashader_browser/streaming.py

@@ -0,0 +1,272 @@
+"""Single-pass, bounded-memory Datashader aggregation.
+
+The browser cannot assume that the complete dataset fits in its WebAssembly
+heap.  This module keeps one input chunk and Datashader's fixed-size aggregate
+state resident at a time.  It intentionally starts with point glyphs: they do
+not require overlap between adjacent chunks and therefore provide a sound
+foundation for later Arrow and DuckDB-Wasm sources.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+import sys
+from typing import Any
+
+import pandas as pd
+
+import datashader as ds
+from datashader.compiler import compile_components
+from datashader.core import _bypixel_sanitise
+from datashader.glyphs import Point
+
+
+@dataclass(frozen=True, slots=True)
+class Progress:
+    """Progress reported after an input chunk has been aggregated."""
+
+    chunks: int
+    rows: int
+
+
+ProgressCallback = Callable[[Progress], None]
+CancelCallback = Callable[[], bool]
+
+
+def iter_csv_chunks(
+    path: str | Path,
+    *,
+    chunksize: int = 250_000,
+    columns: Iterable[str] | None = None,
+    **read_csv_kwargs: Any,
+) -> Iterator[pd.DataFrame]:
+    """Yield bounded-size DataFrames from a CSV in the browser filesystem.
+
+    Parameters mirror :func:`pandas.read_csv`. Only requested columns are
+    decoded, which is important in memory-constrained WebAssembly runtimes.
+    """
+
+    if chunksize <= 0:
+        raise ValueError("chunksize must be greater than zero")
+    usecols = None if columns is None else list(columns)
+    yield from pd.read_csv(
+        path,
+        chunksize=chunksize,
+        usecols=usecols,
+        **read_csv_kwargs,
+    )
+
+
+def iter_parquet_batches(
+    source: Any,
+    *,
+    batch_size: int = 250_000,
+    columns: Sequence[str] | None = None,
+    row_groups: Sequence[int] | None = None,
+    use_threads: bool = False,
+) -> Iterator[Any]:
+    """Yield bounded-size Arrow record batches from a Parquet source.
+
+    ``source`` may be a path or any seekable file object accepted by
+    :class:`pyarrow.parquet.ParquetFile`.  In JupyterLite/Xeus-Python this
+    includes files uploaded or mounted in the browser filesystem.  PyArrow is
+    imported lazily so CSV-only users do not need the optional dependency.
+
+    Arrow batches are intentionally returned without first materializing a
+    complete Pandas DataFrame. :meth:`StreamingCanvas.points` converts one
+    batch at a time immediately before aggregation.
+    """
+
+    if batch_size <= 0:
+        raise ValueError("batch_size must be greater than zero")
+
+    try:
+        import pyarrow.parquet as pq
+    except ImportError as exc:  # pragma: no cover - depends on optional install
+        raise ImportError(
+            "Parquet streaming requires PyArrow; install "
+            "datashader-browser[data]"
+        ) from exc
+
+    parquet = pq.ParquetFile(source)
+    yield from parquet.iter_batches(
+        batch_size=batch_size,
+        row_groups=None if row_groups is None else list(row_groups),
+        columns=None if columns is None else list(columns),
+        use_threads=use_threads,
+    )
+
+
+class StreamingCanvas:
+    """A Datashader canvas that consumes Pandas or Arrow chunks.
+
+    Unlike ``datashader.Canvas.points``, the full source is never retained.
+    The current implementation requires explicit ranges so aggregation stays
+    single-pass. Use source metadata, a cheap bounds query, or a preparatory
+    pass to determine those ranges.
+    """
+
+    def __init__(
+        self,
+        *,
+        plot_width: int = 600,
+        plot_height: int = 600,
+        x_range: tuple[float, float],
+        y_range: tuple[float, float],
+        x_axis_type: str = "linear",
+        y_axis_type: str = "linear",
+    ) -> None:
+        self._canvas = ds.Canvas(
+            plot_width=plot_width,
+            plot_height=plot_height,
+            x_range=x_range,
+            y_range=y_range,
+            x_axis_type=x_axis_type,
+            y_axis_type=y_axis_type,
+        )
+
+    @property
+    def canvas(self) -> ds.Canvas:
+        """The underlying Datashader canvas."""
+
+        return self._canvas
+
+    def points(
+        self,
+        chunks: Iterable[Any],
+        x: str,
+        y: str,
+        *,
+        agg: Any | None = None,
+        on_progress: ProgressCallback | None = None,
+        cancelled: CancelCallback | None = None,
+    ):
+        """Aggregate point chunks while retaining only fixed-size state.
+
+        The aggregation uses Datashader's compiler directly, so binning and
+        reduction behavior match the native package rather than a separate
+        reimplementation.
+        """
+
+        reduction = ds.count() if agg is None else agg
+        glyph = Point(x, y)
+        iterator = iter(chunks)
+
+        try:
+            first = next(iterator)
+        except StopIteration as exc:
+            raise ValueError("chunks must contain at least one DataFrame") from exc
+
+        first, dshape = _prepare_chunk(first, glyph, reduction)
+        schema = dshape.measure
+        glyph.validate(schema)
+        reduction.validate(schema)
+        self._canvas.validate()
+
+        if reduction.uses_row_index(cuda=False, partitioned=False):
+            raise NotImplementedError(
+                "row-index reductions are not supported by the streaming points MVP"
+            )
+
+        (
+            create,
+            info,
+            append,
+            _combine,
+            finalize,
+            antialias_stage_2,
+            antialias_stage_2_funcs,
+            _column_names,
+        ) = compile_components(
+            reduction,
+            schema,
+            glyph,
+            antialias=False,
+            cuda=False,
+            partitioned=False,
+        )
+
+        extend = glyph._build_extend(  # noqa: SLF001 - Datashader extension API
+            self._canvas.x_axis.mapper,
+            self._canvas.y_axis.mapper,
+            info,
+            append,
+            antialias_stage_2,
+            antialias_stage_2_funcs,
+        )
+
+        x_range = self._canvas.x_range
+        y_range = self._canvas.y_range
+        assert x_range is not None and y_range is not None
+        self._canvas.validate_ranges(x_range, y_range)
+
+        width = self._canvas.plot_width
+        height = self._canvas.plot_height
+        x_st = self._canvas.x_axis.compute_scale_and_translate(x_range, width)
+        y_st = self._canvas.y_axis.compute_scale_and_translate(y_range, height)
+        transform = x_st + y_st
+        bounds = x_range + y_range
+        bases = create((height, width))
+
+        processed_chunks = 0
+        processed_rows = 0
+
+        for chunk in _prepend(first, iterator):
+            if cancelled is not None and cancelled():
+                raise RuntimeError("aggregation cancelled")
+            prepared, chunk_dshape = _prepare_chunk(chunk, glyph, reduction)
+            if chunk_dshape.measure != schema:
+                raise TypeError(
+                    "all chunks must have matching Datashader schemas; "
+                    f"expected {schema}, received {chunk_dshape.measure}"
+                )
+            extend(bases, prepared, transform, bounds)
+            processed_chunks += 1
+            processed_rows += len(prepared)
+            if on_progress is not None:
+                on_progress(Progress(processed_chunks, processed_rows))
+
+        x_axis = self._canvas.x_axis.compute_index(x_st, width)
+        y_axis = self._canvas.y_axis.compute_index(y_st, height)
+        return finalize(
+            bases,
+            cuda=False,
+            coords={glyph.x_label: x_axis, glyph.y_label: y_axis},
+            dims=[glyph.y_label, glyph.x_label],
+            attrs={
+                "x_range": x_range,
+                "y_range": y_range,
+                "source_chunks": processed_chunks,
+                "source_rows": processed_rows,
+            },
+        )
+
+
+def _prepare_chunk(chunk: Any, glyph: Point, reduction: Any):
+    if not isinstance(chunk, pd.DataFrame):
+        to_pandas = getattr(chunk, "to_pandas", None)
+        if to_pandas is None or not callable(to_pandas):
+            raise TypeError(
+                "each chunk must be a pandas DataFrame or Arrow-compatible "
+                f"record batch, received {type(chunk)!r}"
+            )
+        # PyArrow's direct Arrow-to-Pandas bridge can stall in the current
+        # Emscripten runtime.  Building the bounded DataFrame from Python
+        # column values avoids that bridge while preserving the batch limit.
+        if sys.platform == "emscripten" and callable(getattr(chunk, "to_pydict", None)):
+            chunk = pd.DataFrame(chunk.to_pydict())
+        else:
+            chunk = to_pandas()
+        if not isinstance(chunk, pd.DataFrame):
+            raise TypeError(
+                "chunk.to_pandas() must return a pandas DataFrame, received "
+                f"{type(chunk)!r}"
+            )
+    return _bypixel_sanitise(chunk, glyph, reduction)
+
+
+def _prepend(first: Any, rest: Iterator[Any]) -> Iterator[Any]:
+    yield first
+    yield from rest

datashader_browser-0.1.0a2.dist-info/METADATA

@@ -0,0 +1,162 @@
+Metadata-Version: 2.5
+Name: datashader-browser
+Version: 0.1.0a2
+Summary: Bounded-memory Datashader pipelines for Python running in the browser
+Project-URL: Homepage, https://github.com/SuMayaBee/datashader-browser
+Project-URL: Issues, https://github.com/SuMayaBee/datashader-browser/issues
+Project-URL: Repository, https://github.com/SuMayaBee/datashader-browser
+Author: Sumaiya Islam
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Requires-Python: >=3.10
+Requires-Dist: datashader<0.20,>=0.19.1
+Requires-Dist: pandas>=2.2
+Provides-Extra: data
+Requires-Dist: pillow>=11; extra == 'data'
+Requires-Dist: pyarrow>=21; extra == 'data'
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# datashader-browser
+
+Run Datashader-powered visualizations entirely in a browser with Python and
+WebAssembly—without assuming that the complete dataset fits in browser memory.
+
+This is an early, independent project. It is not yet an official HoloViz
+project and is not affiliated with or endorsed by the Datashader maintainers.
+
+[Try the live browser demo](https://sumayabee.github.io/datashader-browser/).
+Open `streaming-points.ipynb` and run all cells.
+
+## What works now
+
+The first vertical slice is a bounded-memory point aggregator:
+
+```python
+import datashader as ds
+import datashader.transfer_functions as tf
+
+from datashader_browser import StreamingCanvas, iter_csv_chunks, iter_parquet_batches
+
+source = iter_csv_chunks(
+    "points.csv",
+    chunksize=250_000,
+    columns=["longitude", "latitude", "value"],
+)
+
+canvas = StreamingCanvas(
+    plot_width=1000,
+    plot_height=600,
+    x_range=(-180, 180),
+    y_range=(-90, 90),
+)
+aggregate = canvas.points(source, "longitude", "latitude", agg=ds.mean("value"))
+image = tf.shade(aggregate)
+
+# A local Parquet file can yield projected Arrow record batches directly.
+parquet_source = iter_parquet_batches(
+    "points.parquet", columns=["longitude", "latitude"], batch_size=250_000
+)
+aggregate = canvas.points(parquet_source, "longitude", "latitude", agg=ds.count())
+```
+
+Only one input chunk and Datashader's fixed-size aggregate buffers are live at
+once. The implementation calls Datashader's own compiler, so point binning and
+reductions match native Datashader instead of being reimplemented.
+
+Currently verified reductions are `count`, `sum`, `mean`, `min`, and `max`.
+Explicit `x_range` and `y_range` values are required to preserve a true
+single-pass pipeline.
+
+## NYC Taxi real-data verification
+
+The streaming accumulator successfully rendered all **11,842,094 pickup
+locations** in HoloViz's transformed
+[January 2015 NYC Taxi dataset](https://examples.holoviz.org/gallery/nyc_taxi/nyc_taxi.html).
+The 385 MB Parquet file was decoded as 48 Arrow/Pandas batches of at most
+250,000 rows; the complete table was never materialized as one DataFrame.
+
+![Datashader rendering of 11.8 million January 2015 NYC taxi pickups](docs/assets/nyc-taxi-pickups.png)
+
+One warm-cache Linux run produced the 1200-by-700 image in 3.24 seconds with
+514 MiB peak process RSS. Runtime and memory will vary by machine. Reproduce it
+with:
+
+```bash
+uv sync --extra data
+uv run python examples/nyc_taxi.py --download
+```
+
+This benchmark validates `StreamingCanvas` against a real large dataset on
+native Python. The browser build now includes an experimental PyArrow Parquet
+reader: open `nyc-taxi-browser.ipynb`, place the source file in the browser
+filesystem, and it streams projected Arrow record batches through the same
+accumulator. Full 385 MB end-to-end browser timing and memory results are still
+an explicit validation task, not a claim made by this repository.
+
+## Browser runtime
+
+The initial deployment target is JupyterLite with the Xeus-Python WebAssembly
+kernel. Its environment is described in [`environment.yml`](environment.yml)
+and uses the experimental Numba and llvmlite packages from emscripten-forge.
+The kernel executes in a Web Worker, keeping aggregation off the browser UI
+thread. The Parquet-capable build is currently about 256 MB because PyArrow
+brings compiled Arrow and compression dependencies; this is an engineering
+baseline, not the final delivery architecture. The first visit therefore needs
+a substantial download, while subsequent use benefits from the browser cache.
+
+Browser verification: `streaming-parquet.ipynb` has been run end-to-end in the
+Xeus-Python WebAssembly kernel. It created a 57.2 MB, two-million-row Parquet
+file in the browser filesystem, streamed it through 20 Arrow batches, checked
+the final row count, and rendered the shaded aggregate with no browser console
+errors. The full 385 MB NYC Taxi browser benchmark remains a separate task.
+
+```bash
+uv sync --extra test
+uv run pytest
+
+# Build the browser site (requires the build dependencies below).
+uv pip install -r requirements-build.txt
+uv run jupyter lite build \
+  --XeusAddon.environment_file=environment.yml \
+  --contents content \
+  --output-dir dist
+```
+
+## Large-data strategy
+
+“In the browser” does not mean that an unlimited dataset can be copied into a
+Pandas DataFrame. The project uses three execution levels:
+
+1. **Stream:** scan CSV or local Parquet/Arrow batches through a bounded-memory Datashader
+   accumulator. Total row count affects time, not peak aggregate memory.
+2. **Prune:** use Parquet row groups, spatial partitions, and DuckDB-Wasm to
+   read only the viewport and required columns.
+3. **Tile:** for billion-row interactive datasets, fetch spatial partitions or
+   precomputed multiresolution aggregates instead of rescanning the source on
+   every zoom.
+
+See [Architecture](docs/architecture.md), [Research](docs/research.md), and the
+[roadmap](ROADMAP.md) for the engineering decisions and current limitations.
+
+## Status
+
+This repository is an alpha-stage research implementation. The WebAssembly
+Numba/llvmlite stack works in current demonstrations but its patches have not
+yet been fully upstreamed. Production use should wait for reproducible browser
+builds, compatibility CI, and measured browser memory limits.
+
+## License
+
+BSD-3-Clause. Datashader is a separate BSD-3-Clause project owned by its
+respective copyright holders and contributors.

datashader_browser-0.1.0a2.dist-info/RECORD

@@ -0,0 +1,6 @@
+datashader_browser/__init__.py,sha256=2xa8RV0wxXtsyGjW0gJI-dGI4mQvcsKfGDR4yOR0oY0,296
+datashader_browser/streaming.py,sha256=0Zy2C7798MGpFHNeA_sdoikL8m9pZdKKrlrPAOlCAaQ,9070
+datashader_browser-0.1.0a2.dist-info/METADATA,sha256=dl4huRF2j2d5KlRLCQxsCrF-_RVZDDvI50ZorTaShz0,6648
+datashader_browser-0.1.0a2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+datashader_browser-0.1.0a2.dist-info/licenses/LICENSE,sha256=hTBVAxwAZaXkBLTae5MjdcULGYL7gclAjqJcCs0lHmE,1500
+datashader_browser-0.1.0a2.dist-info/RECORD,,

datashader_browser-0.1.0a2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

datashader_browser-0.1.0a2.dist-info/licenses/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Sumaiya Islam
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.