betterframe 0.1.0__py3-none-any.whl

betterframe/__init__.py

@@ -0,0 +1,38 @@
+"""BetterFrame: write a dataframe pipeline once, run it on Dask or pandas.
+
+Not a DataFrame implementation. It adapts the *execution semantics* that differ
+between engines -- per-partition application, meta schemas, aggregation
+keywords, persistence -- so a pipeline that must work on both does not fill up
+with ``if is_dask:`` branches.
+
+Public API:
+- :class:`BetterFrame` -- a frame bound to the operations its engine needs
+- :class:`DaskOps` / :class:`PandasOps` -- subclass to add engine-specific
+  behaviour of your own, and pass to :class:`BetterFrame`
+- :func:`is_dask_frame` -- engine test that does not import Dask needlessly
+
+Example::
+
+    from betterframe import BetterFrame
+
+    frame = BetterFrame(records).mutable()
+    frame = frame.apply(normalise, meta=lambda: build_meta(frame.meta_source()))
+    result = frame.pipe(lambda df: df.groupby("key").agg(aggs, **frame.agg_kwargs()))
+    return result.finalize()
+"""
+
+from __future__ import annotations
+
+from .frame import BetterFrame
+from .ops import DaskOps, DataFrameOps, MetaSource, PandasOps, is_dask_frame
+
+__all__ = [
+    "BetterFrame",
+    "DaskOps",
+    "DataFrameOps",
+    "MetaSource",
+    "PandasOps",
+    "is_dask_frame",
+]
+
+__version__ = "0.1.0"

betterframe/frame.py

@@ -0,0 +1,113 @@
+"""``BetterFrame`` -- a frame bound to the operations its engine needs.
+
+Every operation that differs between Dask and pandas takes the frame as its
+first argument, so binding the frame removes that argument from every call and
+lets a pipeline chain instead of repeating itself.
+
+The wrapper is deliberately thin. It does not proxy the dataframe API: real
+work still happens on the frame itself, reached through :attr:`native` or
+:meth:`pipe`. Wrapping exists to answer engine questions, not to replace pandas.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from .ops import DaskOps, DataFrameOps, PandasOps, is_dask_frame
+
+__all__ = ["BetterFrame"]
+
+
+class BetterFrame:
+    """A dataframe together with the operations its engine requires.
+
+    Args:
+        frame: A Dask or pandas DataFrame.
+        dask_ops: Operations class to use for Dask frames. Subclass
+            :class:`~betterframe.DaskOps` to add behaviour of your own.
+        pandas_ops: Operations class to use for pandas frames.
+
+    Methods that produce a frame return a new ``BetterFrame``, so a pipeline
+    chains. :meth:`finalize` ends the chain and hands back a native frame.
+    """
+
+    __slots__ = ("_frame", "_ops")
+
+    def __init__(
+        self,
+        frame: Any,
+        *,
+        dask_ops: type[DataFrameOps] = DaskOps,
+        pandas_ops: type[DataFrameOps] = PandasOps,
+    ) -> None:
+        self._frame = frame
+        self._ops = dask_ops() if is_dask_frame(frame) else pandas_ops()
+
+    # --- access ---------------------------------------------------------
+
+    @property
+    def native(self) -> Any:
+        """The underlying dataframe, for operations that need no adaptation."""
+        return self._frame
+
+    @property
+    def ops(self) -> DataFrameOps:
+        """The operations object, for engine questions that take no frame."""
+        return self._ops
+
+    @property
+    def is_dask(self) -> bool:
+        return self._ops.is_dask
+
+    def _wrap(self, frame: Any) -> BetterFrame:
+        new = object.__new__(BetterFrame)
+        new._frame = frame
+        new._ops = self._ops
+        return new
+
+    # --- operations -----------------------------------------------------
+
+    def apply(
+        self, fn: Callable, *args: Any, meta: Callable | None = None, **kwargs: Any
+    ) -> BetterFrame:
+        """Apply a per-partition function.
+
+        ``meta`` is a thunk rather than a value so Dask-only meta builders are
+        never invoked on a pandas frame.
+        """
+        return self._wrap(self._ops.apply(self._frame, fn, *args, meta=meta, **kwargs))
+
+    def pipe(self, fn: Callable, *args: Any, **kwargs: Any) -> BetterFrame:
+        """Run a function against the whole native frame and stay wrapped.
+
+        For operations that need no adaptation -- ``groupby``, ``rename`` and
+        the like -- where dropping to :attr:`native` and re-wrapping would only
+        add noise.
+        """
+        return self._wrap(fn(self._frame, *args, **kwargs))
+
+    def mutable(self) -> BetterFrame:
+        """A frame whose columns may be assigned without affecting the caller."""
+        return self._wrap(self._ops.mutable(self._frame))
+
+    def index_names(self) -> Any:
+        """Names of the frame's index levels."""
+        return self._ops.index_names(self._frame)
+
+    def agg_kwargs(self) -> dict[str, Any]:
+        """Extra keyword arguments for ``groupby().agg()`` on this engine."""
+        return self._ops.agg_kwargs(self._frame)
+
+    def meta_source(self) -> Any:
+        """Something a Dask meta builder can read ``.columns`` and ``._meta`` from."""
+        return self._ops.meta_source(self._frame)
+
+    def finalize(self) -> Any:
+        """Make the result concrete and return the native frame."""
+        return self._ops.finalize(self._frame)
+
+    # --- niceties -------------------------------------------------------
+
+    def __repr__(self) -> str:
+        engine = "dask" if self.is_dask else "pandas"
+        return f"BetterFrame({engine}, {type(self._frame).__name__})"

betterframe/ops.py

@@ -0,0 +1,190 @@
+"""Engine differences between Dask and pandas frames, isolated in one place.
+
+An aggregation pipeline that must run on either engine otherwise fills up with
+``if is_dask:`` branches. These classes hold the differences so the pipeline can
+be written once. Most callers do not use them directly -- they go through
+:class:`~betterframe.BetterFrame`, which binds a frame to the right one -- but
+they are the extension point for adding engine-specific behaviour of your own.
+
+Most of the surface is mechanical -- ``map_partitions(fn)`` versus ``fn(df)``,
+``persist()`` versus nothing. The part worth reading is how the pandas side
+emulates Dask's ``meta`` semantics, because getting it wrong produces frames
+that differ only in dtype, only for empty inputs, and only sometimes:
+
+* Dask coerces **every** partition to a meta schema. A function that
+  short-circuits on an empty frame and returns it unchanged therefore still
+  comes out with the populated schema, because Dask repairs it afterwards.
+* Where no explicit meta is given, Dask *infers* one by running the function
+  against a synthetic non-empty frame. So even without a meta, an empty
+  partition ends up with the schema the function would have produced had there
+  been rows.
+
+pandas does neither. :class:`PandasOps` reproduces both, so a frame that is
+empty on one engine is not silently a different dtype on the other.
+
+This is not a DataFrame implementation. It adapts execution semantics for
+frames that already exist.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+__all__ = ["DaskOps", "DataFrameOps", "MetaSource", "PandasOps", "is_dask_frame"]
+
+
+def is_dask_frame(df: Any) -> bool:
+    """True if ``df`` is a Dask DataFrame, without importing Dask unnecessarily.
+
+    Dask is an optional dependency: a caller working purely in pandas should not
+    pay an import for a question whose answer is no.
+    """
+    module = type(df).__module__
+    if not module.startswith("dask"):
+        return False
+    try:
+        import dask.dataframe as dd
+    except ImportError:  # pragma: no cover - only when dask is half-installed
+        return False
+    return isinstance(df, dd.DataFrame)
+
+
+class DataFrameOps:
+    """Dispatch table for the operations that differ between engines.
+
+    Subclass :class:`DaskOps` or :class:`PandasOps` to add engine-specific
+    behaviour of your own -- custom aggregations, for instance -- and pass the
+    subclasses to :class:`~betterframe.BetterFrame`.
+    """
+
+    #: Whether this implementation drives a Dask frame.
+    is_dask: bool
+
+    def apply(
+        self,
+        df: Any,
+        fn: Callable,
+        *args: Any,
+        meta: Callable | None = None,
+        **kwargs: Any,
+    ) -> Any:
+        """Apply a per-partition function.
+
+        ``meta`` is a thunk rather than a value so Dask-only meta builders are
+        never invoked on a pandas frame.
+        """
+        raise NotImplementedError
+
+    def index_names(self, df: Any) -> Any:
+        """Names of the frame's index levels."""
+        raise NotImplementedError
+
+    def agg_kwargs(self, df: Any) -> dict[str, Any]:
+        """Extra keyword arguments for ``groupby().agg()`` on this engine."""
+        raise NotImplementedError
+
+    def finalize(self, df: Any) -> Any:
+        """Make the result concrete, if the engine has such a notion."""
+        raise NotImplementedError
+
+    def mutable(self, df: Any) -> Any:
+        """A frame whose columns may be assigned without affecting the caller."""
+        raise NotImplementedError
+
+    def meta_source(self, df: Any) -> Any:
+        """Something a Dask meta builder can read ``.columns`` and ``._meta`` from.
+
+        Lets meta builders written against Dask be reused unchanged on pandas
+        frames, rather than duplicated per engine.
+        """
+        raise NotImplementedError
+
+
+class DaskOps(DataFrameOps):
+    """Operations against a Dask DataFrame."""
+
+    is_dask = True
+
+    def apply(self, df, fn, *args, meta=None, **kwargs):
+        if meta is not None:
+            return df.map_partitions(fn, *args, meta=meta(), **kwargs)
+        return df.map_partitions(fn, *args, **kwargs)
+
+    def index_names(self, df):
+        return df.index._meta.names
+
+    def agg_kwargs(self, df):
+        # one output partition per input partition
+        return {"split_out": df.npartitions}
+
+    def finalize(self, df):
+        return df.persist()
+
+    def mutable(self, df):
+        # assignment on a Dask frame builds a new graph; nothing is aliased
+        return df
+
+    def meta_source(self, df):
+        return df
+
+
+class PandasOps(DataFrameOps):
+    """Operations against a pandas DataFrame, emulating Dask's meta semantics."""
+
+    is_dask = False
+
+    def apply(self, df, fn, *args, meta=None, **kwargs):
+        out = fn(df, *args, **kwargs)
+        if not getattr(out, "empty", False):
+            return out
+        schema = (
+            meta() if meta is not None else self._infer_schema(df, fn, args, kwargs)
+        )
+        if schema is None:
+            return out
+        out = out.reindex(columns=schema.columns)
+        return out.astype(
+            {c: dt for c, dt in schema.dtypes.items() if c in out.columns}
+        )
+
+    @staticmethod
+    def _infer_schema(df, fn, args, kwargs):
+        """What Dask would have inferred: run ``fn`` on a synthetic non-empty frame."""
+        try:
+            from dask.dataframe.utils import meta_nonempty
+        except ImportError:
+            return None
+        try:
+            return fn(meta_nonempty(df.iloc[:0]), *args, **kwargs).iloc[:0]
+        except Exception:  # noqa: BLE001 - any failure means we cannot know the
+            # schema, and leaving the frame alone is safer than guessing at one
+            return None
+
+    def index_names(self, df):
+        return df.index.names
+
+    def agg_kwargs(self, df):
+        # `split_out` is a Dask partitioning concern with no pandas equivalent
+        return {}
+
+    def finalize(self, df):
+        return df
+
+    def mutable(self, df):
+        # column assignment must not reach back into the caller's frame
+        return df.copy(deep=False)
+
+    def meta_source(self, df):
+        return MetaSource(df)
+
+
+class MetaSource:
+    """Adapts a pandas frame to the ``.columns`` / ``._meta`` shape Dask meta
+    builders expect, so one implementation of those builders serves both
+    engines rather than being duplicated per engine."""
+
+    __slots__ = ("_meta", "columns")
+
+    def __init__(self, df: Any) -> None:
+        self.columns = df.columns
+        self._meta = df.iloc[:0]

betterframe-0.1.0.dist-info/METADATA

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: betterframe
+Version: 0.1.0
+Summary: Write a dataframe pipeline once, run it on Dask or pandas
+Project-URL: Homepage, https://github.com/izzet/betterframe
+Project-URL: Repository, https://github.com/izzet/betterframe
+Project-URL: Documentation, https://github.com/izzet/betterframe#readme
+Project-URL: Bug Tracker, https://github.com/izzet/betterframe/issues
+Author-email: Izzet Yildirim <izzetcyildirim@gmail.com>
+Maintainer-email: Izzet Yildirim <izzetcyildirim@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: dask,dataframe,meta,pandas,partitions,python
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: pandas>=1.5
+Provides-Extra: dask
+Requires-Dist: dask[dataframe]>=2023.1.0; extra == 'dask'
+Description-Content-Type: text/markdown
+
+# BetterFrame
+
+Write a dataframe pipeline once, run it on Dask or pandas.
+
+Not a DataFrame implementation. BetterFrame adapts the *execution semantics*
+that differ between the two engines — per-partition application, meta schemas,
+aggregation keywords, persistence — so a pipeline that has to work on both does
+not fill up with `if is_dask:` branches.
+
+## Why
+
+Dask is the right tool when data outgrows memory, and a needless tax when it
+does not. A pipeline that wants both ends up either duplicated or littered with
+engine checks. The differences are few but awkward:
+
+| | Dask | pandas |
+|---|---|---|
+| apply a function per partition | `df.map_partitions(fn, meta=…)` | `fn(df)` |
+| index level names | `df.index._meta.names` | `df.index.names` |
+| `groupby().agg()` extras | `split_out=…` | — |
+| make the result concrete | `df.persist()` | already is |
+| column assignment | builds a graph | mutates in place |
+
+## Install
+
+```bash
+pip install betterframe          # pandas only
+pip install "betterframe[dask]"  # to drive Dask frames as well
+```
+
+## Use
+
+```python
+from betterframe import BetterFrame
+
+
+def compute(records):
+    frame = BetterFrame(records).mutable()  # records may be Dask or pandas
+    frame = frame.apply(normalise, meta=lambda: build_meta(frame.meta_source()))
+    result = frame.pipe(
+        lambda df, kw=frame.agg_kwargs(): df.groupby("key").agg({"scaled": "sum"}, **kw)
+    )
+    return result.finalize()
+```
+
+`BetterFrame` binds a frame to the operations its engine needs, so the frame
+stops being an argument to every call. Methods that produce a frame return a
+`BetterFrame`, so a pipeline chains; `finalize()` ends the chain and hands back
+a native frame, and `native` reaches the underlying one at any point.
+
+It is deliberately thin — it does **not** proxy the dataframe API. Real work
+still happens on the frame itself, through `pipe()` or `native`. Wrapping exists
+to answer engine questions, not to replace pandas.
+
+The same function now runs on either engine, and there is exactly one place —
+this package — that knows the difference.
+
+## The part worth knowing about
+
+Most of the surface is mechanical. The subtle part is Dask's `meta` handling,
+because getting it wrong produces frames that differ **only in dtype, only for
+empty inputs, and only sometimes**:
+
+* Dask coerces *every* partition to a meta schema. A helper that short-circuits
+  on an empty frame and returns it unchanged still comes out with the populated
+  schema, because Dask repairs it afterwards.
+* Where no explicit meta is given, Dask *infers* one by running the function
+  against a synthetic non-empty frame. So even without a meta, an empty
+  partition ends up with the schema the function would have produced had there
+  been rows.
+
+pandas does neither. `PandasOps.apply` reproduces both, so an empty frame is
+not silently a different dtype depending on which engine produced it. This is
+the failure mode BetterFrame exists to prevent: it is invisible in every test
+whose fixtures happen to be fully populated.
+
+## Extending
+
+Subclass `DaskOps` / `PandasOps` for engine-specific behaviour of your own —
+custom aggregations, say — and hand the subclasses to `BetterFrame`:
+
+```python
+from betterframe import BetterFrame, DaskOps, PandasOps
+
+
+class MyDaskOps(DaskOps):
+    def set_union(self):
+        return some_dask_aggregation()
+
+
+class MyPandasOps(PandasOps):
+    def set_union(self):
+        return some_pandas_callable
+
+
+frame = BetterFrame(records, dask_ops=MyDaskOps, pandas_ops=MyPandasOps)
+frame.ops.set_union()  # `ops` reaches engine questions that take no frame
+```
+
+Domain-specific aggregations are deliberately left out of the core so the
+package stays about engine semantics.
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest
+uv run ruff check .
+uv run ruff format --check .
+```
+
+## License
+
+MIT

betterframe-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+betterframe/__init__.py,sha256=jiqFSY3gtPNThOx3RuP25yT6Dr3-a0jHlkR7soUrDTk,1224
+betterframe/frame.py,sha256=02YnP0yjS5I6t_P0kZQzcJtqOL-dU8Pd-J0iPUnYhiE,4027
+betterframe/ops.py,sha256=cKb16ZSA17WHlj7qJ1P8fVszwFbJcpjFJWxVKxZEQmo,6554
+betterframe-0.1.0.dist-info/METADATA,sha256=LskvQ_YAPP22ZzC_oRCgnbx6BvzxFGiFOxlXJOy89x8,5131
+betterframe-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+betterframe-0.1.0.dist-info/licenses/LICENSE,sha256=JZMi5bggUJ2443hX6ES9BPrwuduIsBPvTM6yoN5BAm4,1071
+betterframe-0.1.0.dist-info/RECORD,,

betterframe-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

betterframe-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Izzet Yildirim
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.