narwhals-datafusion 0.1.0__tar.gz

narwhals_datafusion-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+__pycache__/
+*.py[cod]
+*.so
+.venv/
+dist/
+build/
+*.egg-info/
+.ruff_cache/
+.pytest_cache/
+.hypothesis/
+tmp/
+
+# Rust (extra-functions-ffi)
+target/
+
+# IDEs
+.vscode/
+.idea/
+
+# IPython / Jupyter
+.ipynb_checkpoints/
+profile_default/
+ipython_config.py
+*.ipynb
+
+# Claude Code
+.claude/
+
+# Local-only docs (kept out of git and any publishing)
+docs/

narwhals_datafusion-0.1.0/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "narwhals"]
+	path = narwhals
+	url = https://github.com/narwhals-dev/narwhals

narwhals_datafusion-0.1.0/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 to present, Dmitrii Bugakov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

narwhals_datafusion-0.1.0/PKG-INFO

@@ -0,0 +1,149 @@
+Metadata-Version: 2.5
+Name: narwhals-datafusion
+Version: 0.1.0
+Summary: Apache DataFusion backend for Narwhals, via the Narwhals plugin system
+Project-URL: Repository, https://github.com/s5dsn-eqee/narwhals-datafusion
+License-Expression: MIT
+License-File: LICENSE.md
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Requires-Dist: datafusion-extra-functions-ffi<0.2,>=0.1
+Requires-Dist: datafusion<55,>=54
+Requires-Dist: narwhals<2.26,>=2.25
+Requires-Dist: pyarrow>=14.0.0
+Description-Content-Type: text/markdown
+
+# narwhals-datafusion
+
+[![CI](https://github.com/s5dsn-eqee/narwhals-datafusion/actions/workflows/ci.yml/badge.svg)](https://github.com/s5dsn-eqee/narwhals-datafusion/actions/workflows/ci.yml)
+[![PyPI version](https://badge.fury.io/py/narwhals-datafusion.svg)](https://badge.fury.io/py/narwhals-datafusion)
+[![Downloads](https://static.pepy.tech/badge/narwhals-datafusion/month)](https://pepy.tech/project/narwhals-datafusion)
+[![Trusted publishing](https://img.shields.io/badge/Trusted_publishing-Provides_attestations-bright_green)](https://peps.python.org/pep-0740/)
+[![PYPI - Types](https://img.shields.io/pypi/types/narwhals-datafusion)](https://pypi.org/project/narwhals-datafusion)
+
+[Apache DataFusion](https://datafusion.apache.org/python/) backend for
+[Narwhals](https://github.com/narwhals-dev/narwhals), implemented as an
+out-of-tree plugin via the `narwhals.plugins` entry-point system.
+
+## Usage
+
+```python
+import narwhals as nw
+import pyarrow as pa
+from datafusion import SessionContext
+
+ctx = SessionContext()
+df = ctx.from_arrow(pa.table({"a": [1, 2, 3], "b": ["x", "y", "x"]}))
+
+lf = nw.from_native(df)  # -> nw.LazyFrame, dispatched to this plugin
+result = (
+    lf.group_by("b")
+    .agg(nw.col("a").sum())
+    .sort("b")
+    .collect(backend="pyarrow")
+)
+```
+
+Everything stays lazy until `.collect()`: narwhals expressions are translated to
+`datafusion.Expr`, frame verbs to `datafusion.DataFrame` methods, and the plan
+executes in DataFusion's Rust engine. Dtypes are pyarrow end-to-end.
+
+## Architecture
+
+The backend sits on narwhals' shared SQL layer (`narwhals._sql`), the same
+abstraction DuckDB, Ibis, and Spark use. It targets `narwhals==2.25` internals
+(`narwhals._sql`, `narwhals._compliant`), vendored as the `narwhals/` git
+submodule pinned to that release. The plugin provides:
+
+| Module | Class |
+|---|---|
+| `dataframe.py` | `DataFusionLazyFrame` — frame verbs over `datafusion.DataFrame` |
+| `expr.py` | `DataFusionExpr` — the six `SQLExpr` hooks + backend specifics |
+| `namespace.py` | `DataFusionNamespace` — the four `SQLNamespace` primitives, IO, horizontal fns |
+| `group_by.py`, `selectors.py`, `expr_str/dt/list/struct.py` | supporting surface |
+| `utils.py` | function-name remapping, window/sort builders, dtype bridge (delegates to `narwhals._arrow`) |
+
+## `mode`, `skew`, `kurtosis` via `datafusion-extra-functions`
+
+DataFusion core deliberately keeps its function library lean, so aggregates
+like `mode`/`skewness`/`kurtosis` live in the contrib
+[`datafusion-extra-functions`](https://github.com/datafusion-contrib/datafusion-extra-functions)
+crate (Rust-only, no wheel on PyPI). The
+[`datafusion-extra-functions-ffi`](https://github.com/s5dsn-eqee/datafusion-extra-functions-ffi)
+package — a required dependency, installed as a prebuilt wheel — exposes its
+aggregate UDFs to datafusion-python via the `__datafusion_aggregate_udf__`
+PyCapsule protocol; they back `Expr.mode`, `Expr.skew`, and `Expr.kurtosis`.
+Its FFI ABI is tied to the `datafusion` major it was compiled against
+(currently 54), which the wheel's own dependency pin enforces.
+
+## API coverage
+
+Status of the narwhals public API on this backend (`narwhals==2.25`,
+`datafusion==54`). ⚠️ entries work with the caveat in parentheses; details in
+[Known limitations](#known-limitations-as-of-datafusion-54).
+
+| Namespace | ✅ Supported | ⚠️ Partial | ❌ Not supported |
+|---|---|---|---|
+| `Expr` | `abs` `alias` `all` `any` `any_value` `ceil` `clip` `cos` `count` `cum_count` `cum_max` `cum_min` `cum_sum` `diff` `exp` `fill_nan` `first` `floor` `is_between` `is_close` `is_duplicated` `is_finite` `is_first_distinct` `is_in` `is_last_distinct` `is_nan` `is_null` `is_unique` `kurtosis` `last` `len` `log` `max` `mean` `median` `min` `null_count` `over` `pipe` `rank` `rolling_mean` `rolling_std` `rolling_sum` `rolling_var` `round` `shift` `sin` `skew` `sqrt` `std` `sum` `var` | `cast` (no `Enum`) · `fill_null` (no `strategy` + `limit`) · `mode` (`keep="any"` only) · `n_unique` (not over windows) · `replace_strict` (explicit `default` required) | `cum_prod` `quantile` |
+| `Expr.str` | `contains` `ends_with` `head` `len_chars` `pad_end` `pad_start` `replace_all` `slice` `split` `starts_with` `strip_chars` `strip_chars_end` `strip_chars_start` `tail` `to_lowercase` `to_uppercase` `zfill` | `to_date`/`to_datetime` (explicit `format` required) · `to_time` (`"HH:MM:SS"`-style only) · `to_titlecase` (no word breaks on digits) | `replace` (use `replace_all`) |
+| `Expr.dt` | `convert_time_zone` `date` `day` `hour` `microsecond` `millisecond` `minute` `month` `nanosecond` `ordinal_day` `second` `to_string` `truncate` `weekday` `year` | `replace_time_zone` (`None`/`"UTC"` only) | `offset_by` `timestamp` `total_microseconds` `total_milliseconds` `total_minutes` `total_nanoseconds` `total_seconds` |
+| `Expr.list` | `contains` `get` `len` `max` `min` `sort` | `unique` (`maintain_order=False` only) | `mean` `median` `sum` |
+| `Expr.struct` | `field` | | |
+| `LazyFrame` | `collect` `collect_schema` `drop` `drop_nulls` `filter` `group_by` `head` `join` `rename` `select` `sort` `top_k` `unique` `unpivot` `with_columns` `with_row_index` | `explode` (single column) · `sink_parquet` (file path only) | `join_asof` |
+
+Not listed: methods narwhals itself doesn't support on *any* lazy/SQL backend
+(`Expr.filter`, `Expr.drop_nulls`, `Expr.unique`, `Expr.map_batches`,
+`Expr.ewm_mean`, `LazyFrame.tail`, `LazyFrame.gather_every`).
+
+## Known limitations (as of datafusion 54)
+
+- `join_asof`, exact `quantile`,
+  `cum_prod`, `list.sum/mean/median`, `dt.total_*`, `dt.offset_by`,
+  `dt.timestamp`, `str.replace`, `Enum` casts — no engine support;
+  raise `NotImplementedError`.
+- `n_unique().over(...)` raises: DataFusion silently ignores `DISTINCT` inside
+  window aggregates, which would return wrong results. (The same engine quirk
+  drops `ORDER BY`/`IGNORE NULLS` declared inside window aggregates — this
+  backend moves those modifiers onto the window itself.)
+- `fill_null(strategy=..., limit=n)` raises: bounded window frames with
+  `first_value`/`last_value` need `retract_batch`, unimplemented engine-side.
+- `replace_time_zone` supports `None` (strip) and `"UTC"` only; use
+  `convert_time_zone` for instant-preserving conversions.
+- `str.to_datetime`/`to_date` require an explicit `format`;
+  `str.to_time` parses `"HH:MM:SS"`-style strings via Arrow's cast, ignoring
+  custom formats.
+- `replace_strict` requires an explicit `default`.
+- `str.to_titlecase` uses `initcap`, which doesn't break words on digits.
+- No row-order guarantees except after `sort` (standard for SQL engines);
+  backward `fill_null` may reorder rows.
+
+## Development
+
+```sh
+git submodule update --init      # narwhals, pinned to the targeted release
+uv sync --group tests
+```
+
+## Running narwhals' own test suite against this backend
+
+The narwhals repo is vendored as a git submodule pinned to the targeted
+release:
+
+```sh
+git submodule update --init
+uv run --group tests python run_tests.py    # known failures deselected
+```
+
+For the full, unfiltered run:
+
+```sh
+uv run --group tests pytest narwhals/tests -c narwhals/pyproject.toml \
+    -p narwhals_datafusion.testing -p env --use-external-constructor
+```
+
+Regenerate the deselect list after fixing tests or bumping the submodule with
+`uv run --group tests python update_run_tests.py`.

narwhals_datafusion-0.1.0/README.md

@@ -0,0 +1,130 @@
+# narwhals-datafusion
+
+[![CI](https://github.com/s5dsn-eqee/narwhals-datafusion/actions/workflows/ci.yml/badge.svg)](https://github.com/s5dsn-eqee/narwhals-datafusion/actions/workflows/ci.yml)
+[![PyPI version](https://badge.fury.io/py/narwhals-datafusion.svg)](https://badge.fury.io/py/narwhals-datafusion)
+[![Downloads](https://static.pepy.tech/badge/narwhals-datafusion/month)](https://pepy.tech/project/narwhals-datafusion)
+[![Trusted publishing](https://img.shields.io/badge/Trusted_publishing-Provides_attestations-bright_green)](https://peps.python.org/pep-0740/)
+[![PYPI - Types](https://img.shields.io/pypi/types/narwhals-datafusion)](https://pypi.org/project/narwhals-datafusion)
+
+[Apache DataFusion](https://datafusion.apache.org/python/) backend for
+[Narwhals](https://github.com/narwhals-dev/narwhals), implemented as an
+out-of-tree plugin via the `narwhals.plugins` entry-point system.
+
+## Usage
+
+```python
+import narwhals as nw
+import pyarrow as pa
+from datafusion import SessionContext
+
+ctx = SessionContext()
+df = ctx.from_arrow(pa.table({"a": [1, 2, 3], "b": ["x", "y", "x"]}))
+
+lf = nw.from_native(df)  # -> nw.LazyFrame, dispatched to this plugin
+result = (
+    lf.group_by("b")
+    .agg(nw.col("a").sum())
+    .sort("b")
+    .collect(backend="pyarrow")
+)
+```
+
+Everything stays lazy until `.collect()`: narwhals expressions are translated to
+`datafusion.Expr`, frame verbs to `datafusion.DataFrame` methods, and the plan
+executes in DataFusion's Rust engine. Dtypes are pyarrow end-to-end.
+
+## Architecture
+
+The backend sits on narwhals' shared SQL layer (`narwhals._sql`), the same
+abstraction DuckDB, Ibis, and Spark use. It targets `narwhals==2.25` internals
+(`narwhals._sql`, `narwhals._compliant`), vendored as the `narwhals/` git
+submodule pinned to that release. The plugin provides:
+
+| Module | Class |
+|---|---|
+| `dataframe.py` | `DataFusionLazyFrame` — frame verbs over `datafusion.DataFrame` |
+| `expr.py` | `DataFusionExpr` — the six `SQLExpr` hooks + backend specifics |
+| `namespace.py` | `DataFusionNamespace` — the four `SQLNamespace` primitives, IO, horizontal fns |
+| `group_by.py`, `selectors.py`, `expr_str/dt/list/struct.py` | supporting surface |
+| `utils.py` | function-name remapping, window/sort builders, dtype bridge (delegates to `narwhals._arrow`) |
+
+## `mode`, `skew`, `kurtosis` via `datafusion-extra-functions`
+
+DataFusion core deliberately keeps its function library lean, so aggregates
+like `mode`/`skewness`/`kurtosis` live in the contrib
+[`datafusion-extra-functions`](https://github.com/datafusion-contrib/datafusion-extra-functions)
+crate (Rust-only, no wheel on PyPI). The
+[`datafusion-extra-functions-ffi`](https://github.com/s5dsn-eqee/datafusion-extra-functions-ffi)
+package — a required dependency, installed as a prebuilt wheel — exposes its
+aggregate UDFs to datafusion-python via the `__datafusion_aggregate_udf__`
+PyCapsule protocol; they back `Expr.mode`, `Expr.skew`, and `Expr.kurtosis`.
+Its FFI ABI is tied to the `datafusion` major it was compiled against
+(currently 54), which the wheel's own dependency pin enforces.
+
+## API coverage
+
+Status of the narwhals public API on this backend (`narwhals==2.25`,
+`datafusion==54`). ⚠️ entries work with the caveat in parentheses; details in
+[Known limitations](#known-limitations-as-of-datafusion-54).
+
+| Namespace | ✅ Supported | ⚠️ Partial | ❌ Not supported |
+|---|---|---|---|
+| `Expr` | `abs` `alias` `all` `any` `any_value` `ceil` `clip` `cos` `count` `cum_count` `cum_max` `cum_min` `cum_sum` `diff` `exp` `fill_nan` `first` `floor` `is_between` `is_close` `is_duplicated` `is_finite` `is_first_distinct` `is_in` `is_last_distinct` `is_nan` `is_null` `is_unique` `kurtosis` `last` `len` `log` `max` `mean` `median` `min` `null_count` `over` `pipe` `rank` `rolling_mean` `rolling_std` `rolling_sum` `rolling_var` `round` `shift` `sin` `skew` `sqrt` `std` `sum` `var` | `cast` (no `Enum`) · `fill_null` (no `strategy` + `limit`) · `mode` (`keep="any"` only) · `n_unique` (not over windows) · `replace_strict` (explicit `default` required) | `cum_prod` `quantile` |
+| `Expr.str` | `contains` `ends_with` `head` `len_chars` `pad_end` `pad_start` `replace_all` `slice` `split` `starts_with` `strip_chars` `strip_chars_end` `strip_chars_start` `tail` `to_lowercase` `to_uppercase` `zfill` | `to_date`/`to_datetime` (explicit `format` required) · `to_time` (`"HH:MM:SS"`-style only) · `to_titlecase` (no word breaks on digits) | `replace` (use `replace_all`) |
+| `Expr.dt` | `convert_time_zone` `date` `day` `hour` `microsecond` `millisecond` `minute` `month` `nanosecond` `ordinal_day` `second` `to_string` `truncate` `weekday` `year` | `replace_time_zone` (`None`/`"UTC"` only) | `offset_by` `timestamp` `total_microseconds` `total_milliseconds` `total_minutes` `total_nanoseconds` `total_seconds` |
+| `Expr.list` | `contains` `get` `len` `max` `min` `sort` | `unique` (`maintain_order=False` only) | `mean` `median` `sum` |
+| `Expr.struct` | `field` | | |
+| `LazyFrame` | `collect` `collect_schema` `drop` `drop_nulls` `filter` `group_by` `head` `join` `rename` `select` `sort` `top_k` `unique` `unpivot` `with_columns` `with_row_index` | `explode` (single column) · `sink_parquet` (file path only) | `join_asof` |
+
+Not listed: methods narwhals itself doesn't support on *any* lazy/SQL backend
+(`Expr.filter`, `Expr.drop_nulls`, `Expr.unique`, `Expr.map_batches`,
+`Expr.ewm_mean`, `LazyFrame.tail`, `LazyFrame.gather_every`).
+
+## Known limitations (as of datafusion 54)
+
+- `join_asof`, exact `quantile`,
+  `cum_prod`, `list.sum/mean/median`, `dt.total_*`, `dt.offset_by`,
+  `dt.timestamp`, `str.replace`, `Enum` casts — no engine support;
+  raise `NotImplementedError`.
+- `n_unique().over(...)` raises: DataFusion silently ignores `DISTINCT` inside
+  window aggregates, which would return wrong results. (The same engine quirk
+  drops `ORDER BY`/`IGNORE NULLS` declared inside window aggregates — this
+  backend moves those modifiers onto the window itself.)
+- `fill_null(strategy=..., limit=n)` raises: bounded window frames with
+  `first_value`/`last_value` need `retract_batch`, unimplemented engine-side.
+- `replace_time_zone` supports `None` (strip) and `"UTC"` only; use
+  `convert_time_zone` for instant-preserving conversions.
+- `str.to_datetime`/`to_date` require an explicit `format`;
+  `str.to_time` parses `"HH:MM:SS"`-style strings via Arrow's cast, ignoring
+  custom formats.
+- `replace_strict` requires an explicit `default`.
+- `str.to_titlecase` uses `initcap`, which doesn't break words on digits.
+- No row-order guarantees except after `sort` (standard for SQL engines);
+  backward `fill_null` may reorder rows.
+
+## Development
+
+```sh
+git submodule update --init      # narwhals, pinned to the targeted release
+uv sync --group tests
+```
+
+## Running narwhals' own test suite against this backend
+
+The narwhals repo is vendored as a git submodule pinned to the targeted
+release:
+
+```sh
+git submodule update --init
+uv run --group tests python run_tests.py    # known failures deselected
+```
+
+For the full, unfiltered run:
+
+```sh
+uv run --group tests pytest narwhals/tests -c narwhals/pyproject.toml \
+    -p narwhals_datafusion.testing -p env --use-external-constructor
+```
+
+Regenerate the deselect list after fixing tests or bumping the submodule with
+`uv run --group tests python update_run_tests.py`.

narwhals_datafusion-0.1.0/pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "narwhals-datafusion"
+version = "0.1.0"
+description = "Apache DataFusion backend for Narwhals, via the Narwhals plugin system"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+dependencies = [
+  # targets narwhals 2.25 internals (`narwhals._sql`); widen only after testing
+  "narwhals>=2.25,<2.26",
+  # must match the FFI shim's compiled ABI (see extra-functions-ffi dependency)
+  "datafusion>=54,<55",
+  "pyarrow>=14.0.0",
+  # FFI shim for `mode`/`skew`/`kurtosis`; prebuilt wheels from
+  # https://github.com/s5dsn-eqee/datafusion-extra-functions-ffi
+  "datafusion-extra-functions-ffi>=0.1,<0.2",
+]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+]
+
+[project.urls]
+Repository = "https://github.com/s5dsn-eqee/narwhals-datafusion"
+
+[project.entry-points.'narwhals.plugins']
+narwhals-datafusion = 'narwhals_datafusion'
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/narwhals_datafusion"]
+
+[tool.hatch.build.targets.sdist]
+# ship only what's needed to build the wheel; `narwhals/` is the dev submodule
+exclude = ["docs", "narwhals", "tests", "tmp", ".github", ".hypothesis", "run_tests.py", "update_run_tests.py", "uv.lock"]
+
+[dependency-groups]
+dev = ["pytest>=8"]
+# Deps for running narwhals' own suite from the `narwhals/` submodule
+# (daft-style): `uv run --group tests python run_tests.py`.
+tests = [
+  "pytest>=9.0.3",
+  "pytest-env",
+  "pytest-randomly",
+  "pytest-xdist",
+  "hypothesis>=6.0.0",
+]
+
+[tool.uv.sources]
+# narwhals comes editable from the pinned submodule (run
+# `git submodule update --init` after cloning), so the plugin is always
+# developed and tested against the exact internals it targets.
+narwhals = { path = "narwhals", editable = true }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+# Standalone lint config (decoupled from the narwhals repo's root settings).
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+extend-exclude = ["narwhals"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W", "UP", "B", "SIM", "TC"]
+ignore = ["SIM108"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["narwhals_datafusion"]

narwhals_datafusion-0.1.0/src/narwhals_datafusion/__init__.py

@@ -0,0 +1,28 @@
+"""Apache DataFusion backend for Narwhals, registered via the `narwhals.plugins` entry point."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import datafusion
+    from narwhals._utils import Version
+    from typing_extensions import TypeIs
+
+    from narwhals_datafusion.namespace import DataFusionNamespace
+
+__version__ = "0.1.0"
+
+NATIVE_PACKAGE = "datafusion"
+
+
+def __narwhals_namespace__(version: Version) -> DataFusionNamespace:  # noqa: N807
+    from narwhals_datafusion.namespace import DataFusionNamespace
+
+    return DataFusionNamespace(version=version)
+
+
+def is_native(native_object: object) -> TypeIs[datafusion.DataFrame]:
+    import datafusion
+
+    return isinstance(native_object, datafusion.DataFrame)