lazyscribe-arrow 0.2.1__tar.gz

lazyscribe_arrow-0.2.1/PKG-INFO

@@ -0,0 +1,85 @@
+Metadata-Version: 2.3
+Name: lazyscribe-arrow
+Version: 0.2.1
+Summary: Arrow-based artifact handlers for Lazyscribe
+Author: Akshay Gupta
+Author-email: Akshay Gupta <akgcodes@gmail.com>
+License: MIT license
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: attrs>=21.2,<=25.3
+Requires-Dist: lazyscribe>=1,<=1.2
+Requires-Dist: pyarrow>=14.0.1,<=21
+Requires-Dist: python-slugify>=5,<=8.0.4
+Requires-Dist: commitizen ; extra == 'build'
+Requires-Dist: uv ; extra == 'build'
+Requires-Dist: lazyscribe-arrow[build] ; extra == 'dev'
+Requires-Dist: lazyscribe-arrow[qa] ; extra == 'dev'
+Requires-Dist: lazyscribe-arrow[tests] ; extra == 'dev'
+Requires-Dist: edgetest ; extra == 'qa'
+Requires-Dist: mypy ; extra == 'qa'
+Requires-Dist: pre-commit ; extra == 'qa'
+Requires-Dist: pyproject-fmt ; extra == 'qa'
+Requires-Dist: ruff ; extra == 'qa'
+Requires-Dist: types-python-slugify ; extra == 'qa'
+Requires-Dist: uv ; extra == 'qa'
+Requires-Dist: pandas ; extra == 'tests'
+Requires-Dist: pytest ; extra == 'tests'
+Requires-Dist: pytest-cov ; extra == 'tests'
+Requires-Dist: time-machine ; extra == 'tests'
+Requires-Python: >=3.10.0
+Project-URL: Documentation, https://github.com/lazyscribe/lazyscribe-arrow
+Project-URL: Repository, https://github.com/lazyscribe/lazyscribe-arrow
+Provides-Extra: build
+Provides-Extra: dev
+Provides-Extra: qa
+Provides-Extra: tests
+Description-Content-Type: text/markdown
+
+[![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![PyPI](https://img.shields.io/pypi/v/lazyscribe-arrow)](https://pypi.org/project/lazyscribe-arrow/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/lazyscribe-arrow)](https://pypi.org/project/lazyscrib-arrow/) [![codecov](https://codecov.io/gh/lazyscribe/lazyscribe-arrow/graph/badge.svg?token=W5TPK7GX7G)](https://codecov.io/gh/lazyscribe/lazyscribe-arrow)
+
+# Arrow-based artifact handling for lazyscribe
+
+`lazyscribe-arrow` is a lightweight package that adds the following artifact handlers for `lazyscribe`:
+
+* `csv`
+
+Any data structure that implements the [Arrow PyCapsule Interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html)
+will be compatible with the handlers in this library. Popular compatible open source data structures include
+
+* `pandas.DataFrame`
+* `polars.DataFrame`
+* `polars.LazyFrame`
+
+# Installation
+
+Python 3.10 and above is required. use `pip` to install:
+
+```console
+$ python -m pip install lazyscribe-arrow
+```
+
+# Usage
+
+To use this library, simply log an artifact to a `lazyscribe` experiment or repository with
+
+* `handler="csv"` for a CSV output
+
+
+```python
+import pyarrow as pa
+from lazyscribe import Project
+
+project = Project("project.json", mode="w")
+with project.log("My experiment") as exp:
+    data = pa.Table.from_arrays([[0, 1, 2]], names=["a"])
+    exp.log_artifact(name="data", value=data, handler="csv")
+
+project.save()
+```

lazyscribe_arrow-0.2.1/README.md

@@ -0,0 +1,41 @@
+[![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![PyPI](https://img.shields.io/pypi/v/lazyscribe-arrow)](https://pypi.org/project/lazyscribe-arrow/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/lazyscribe-arrow)](https://pypi.org/project/lazyscrib-arrow/) [![codecov](https://codecov.io/gh/lazyscribe/lazyscribe-arrow/graph/badge.svg?token=W5TPK7GX7G)](https://codecov.io/gh/lazyscribe/lazyscribe-arrow)
+
+# Arrow-based artifact handling for lazyscribe
+
+`lazyscribe-arrow` is a lightweight package that adds the following artifact handlers for `lazyscribe`:
+
+* `csv`
+
+Any data structure that implements the [Arrow PyCapsule Interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html)
+will be compatible with the handlers in this library. Popular compatible open source data structures include
+
+* `pandas.DataFrame`
+* `polars.DataFrame`
+* `polars.LazyFrame`
+
+# Installation
+
+Python 3.10 and above is required. use `pip` to install:
+
+```console
+$ python -m pip install lazyscribe-arrow
+```
+
+# Usage
+
+To use this library, simply log an artifact to a `lazyscribe` experiment or repository with
+
+* `handler="csv"` for a CSV output
+
+
+```python
+import pyarrow as pa
+from lazyscribe import Project
+
+project = Project("project.json", mode="w")
+with project.log("My experiment") as exp:
+    data = pa.Table.from_arrays([[0, 1, 2]], names=["a"])
+    exp.log_artifact(name="data", value=data, handler="csv")
+
+project.save()
+```

lazyscribe_arrow-0.2.1/lazyscribe_arrow/__init__.py

@@ -0,0 +1,6 @@
+"""Import the custom artifact handlers."""
+
+from lazyscribe_arrow.csv import CSVArtifact
+from lazyscribe_arrow.parquet import ParquetArtifact
+
+__all__: list[str] = ["CSVArtifact", "ParquetArtifact"]

lazyscribe_arrow-0.2.1/lazyscribe_arrow/_meta.py

@@ -0,0 +1,3 @@
+"""Version."""
+
+__version__ = "0.2.1"

lazyscribe_arrow-0.2.1/lazyscribe_arrow/csv.py

@@ -0,0 +1,111 @@
+"""Custom artifact handlers for CSVs."""
+
+import logging
+from datetime import datetime
+from typing import Any, ClassVar
+
+import pyarrow as pa
+from attrs import define
+from lazyscribe._utils import utcnow
+from lazyscribe.artifacts.base import Artifact
+from pyarrow import csv
+from pyarrow.interchange import from_dataframe
+from slugify import slugify
+
+from lazyscribe_arrow.protocols import (
+    ArrowArrayExportable,
+    ArrowStreamExportable,
+    SupportsInterchange,
+)
+
+LOG = logging.getLogger(__name__)
+
+
+@define(auto_attribs=True)
+class CSVArtifact(Artifact):
+    """Arrow-powered CSV handler."""
+
+    alias: ClassVar[str] = "csv"
+    suffix: ClassVar[str] = "csv"
+    binary: ClassVar[bool] = True
+    output_only: ClassVar[bool] = False
+
+    @classmethod
+    def construct(
+        cls,
+        name: str,
+        value: Any | None = None,
+        fname: str | None = None,
+        created_at: datetime | None = None,
+        writer_kwargs: dict | None = None,
+        version: int = 0,
+        dirty: bool = True,
+        **kwargs,
+    ):
+        """Construct the handler class."""
+        created_at = created_at or utcnow()
+
+        return cls(  # type: ignore[call-arg]
+            name=name,
+            value=value,
+            fname=fname
+            or f"{slugify(name)}-{slugify(created_at.strftime('%Y%m%d%H%M%S'))}.{cls.suffix}",
+            writer_kwargs=writer_kwargs or {},
+            version=version,
+            created_at=created_at,
+            dirty=dirty,
+        )
+
+    @classmethod
+    def read(cls, buf, **kwargs) -> pa.Table:
+        """Read in the CSV file.
+
+        Parameters
+        ----------
+        buf : file-like object
+            The buffer from a ``fsspec`` filesystem.
+        **kwargs
+            Keyword arguments for the read method.
+
+        Returns
+        -------
+        pyarrow.lib.Table
+            A ``pyarrow`` table with the data.
+        """
+        return csv.read_csv(buf, **kwargs)
+
+    @classmethod
+    def write(cls, obj, buf, **kwargs):
+        """Write the CSV file using pyarrow.
+
+        Parameters
+        ----------
+        obj : object
+            The object to write.
+        buf : file-like object
+            The buffer from a ``fsspec`` filesystem.
+        **kwargs
+            Keyword arguments for :py:meth:`pyarrow.csv.write_csv`.
+
+        Raises
+        ------
+        ValueError
+            Raised if the supplied object does not have ``__arrow_c_array__``
+            or ``__arrow_c_stream__`` attributes. These attributes allow us to
+            perform a zero-copy transformation from the native obejct to a PyArrow
+            Table.
+        """
+        if isinstance(obj, pa.Table):
+            LOG.debug("Provided object is already a PyArrow table.")
+        elif isinstance(obj, (ArrowArrayExportable, ArrowStreamExportable)):
+            obj = pa.table(obj)
+        elif isinstance(obj, SupportsInterchange):
+            obj = from_dataframe(obj)
+        else:
+            raise ValueError(
+                f"Object of type `{type(obj)}` cannot be easily coerced into a PyArrow Table. "
+                "Please provide an object that implements the Arrow PyCapsule Interface or the "
+                "Dataframe Interchange Protocol."
+            )
+
+        csv.write_csv(obj, buf, **kwargs)

lazyscribe_arrow-0.2.1/lazyscribe_arrow/parquet.py

@@ -0,0 +1,113 @@
+"""Custom artifact handlers for parquets."""
+
+import logging
+from datetime import datetime
+from typing import Any, ClassVar
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from attrs import define
+from lazyscribe._utils import utcnow
+from lazyscribe.artifacts.base import Artifact
+from pyarrow.interchange import from_dataframe
+from slugify import slugify
+
+from lazyscribe_arrow.protocols import (
+    ArrowArrayExportable,
+    ArrowStreamExportable,
+    SupportsInterchange,
+)
+
+LOG = logging.getLogger(__name__)
+
+
+@define(auto_attribs=True)
+class ParquetArtifact(Artifact):
+    """Arrow-powered Parquet handler."""
+
+    alias: ClassVar[str] = "parquet"
+    suffix: ClassVar[str] = "parquet"
+    binary: ClassVar[bool] = True
+    output_only: ClassVar[bool] = False
+
+    @classmethod
+    def construct(
+        cls,
+        name: str,
+        value: Any | None = None,
+        fname: str | None = None,
+        created_at: datetime | None = None,
+        writer_kwargs: dict | None = None,
+        version: int = 0,
+        dirty: bool = True,
+        **kwargs,
+    ):
+        """Construct the handler class."""
+        created_at = created_at or utcnow()
+
+        return cls(  # type: ignore[call-arg]
+            name=name,
+            value=value,
+            fname=fname
+            or f"{slugify(name)}-{slugify(created_at.strftime('%Y%m%d%H%M%S'))}.{cls.suffix}",
+            writer_kwargs=writer_kwargs or {},
+            version=version,
+            created_at=created_at,
+            dirty=dirty,
+        )
+
+    @classmethod
+    def read(cls, buf, **kwargs) -> pa.Table:
+        """Read in the parquet file.
+
+        Parameters
+        ----------
+        buf : file-like object
+            The buffer from a ``fsspec`` filesystem.
+        **kwargs
+            Keyword arguments for the read method.
+
+        Returns
+        -------
+        pyarrow.lib.Table
+            A ``pyarrow`` table with the data.
+        """
+        return pq.read_table(buf, **kwargs)
+
+    @classmethod
+    def write(cls, obj, buf, **kwargs):
+        """Write the parquet file using pyarrow.
+
+        Parameters
+        ----------
+        obj : object
+            The object to write.
+        buf : file-like object
+            The buffer from a ``fsspec`` filesystem.
+        **kwargs
+            Keyword arguments for :py:meth:`pyarrow.parquet.write_table`.
+
+        Raises
+        ------
+        ValueError
+            Raised if the supplied object does not have ``__arrow_c_array__``
+            or ``__arrow_c_stream__`` attribute
+            or if the object does not
+            implement the dataframe interchange protocol. These attributes allow us to
+            perform a zero-copy transformation from the native obejct to a PyArrow
+            Table.
+        """
+        if isinstance(obj, pa.Table):
+            LOG.debug("Provided object is already a PyArrow table.")
+        elif isinstance(obj, (ArrowArrayExportable, ArrowStreamExportable)):
+            obj = pa.table(obj)
+        elif isinstance(obj, SupportsInterchange):
+            obj = from_dataframe(obj)
+        else:
+            raise ValueError(
+                f"Object of type `{type(obj)}` cannot be easily coerced into a PyArrow Table. "
+                "Please provide an object that implements the Arrow PyCapsule Interface or the "
+                "Dataframe Interchange Protocol."
+            )
+
+        pq.write_table(obj, buf, **kwargs)

lazyscribe_arrow-0.2.1/lazyscribe_arrow/protocols.py

@@ -0,0 +1,38 @@
+"""Arrow exportable protocols."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class ArrowArrayExportable(Protocol):
+    """Type protocol for Arrow C Data Interface via Arrow PyCapsule Interface."""
+
+    def __arrow_c_array__(
+        self, requested_schema: object | None = None
+    ) -> tuple[object, object]:
+        """Export the object as a pair of ArrowSchema and ArrowArray structures."""
+        ...
+
+
+@runtime_checkable
+class ArrowStreamExportable(Protocol):
+    """Type protocol for Arrow C Stream Interface via Arrow PyCapsule Interface."""
+
+    def __arrow_c_stream__(self, requested_schema: object | None = None) -> object:
+        """Export the object as an ArrowArrayStream."""
+        ...
+
+
+@runtime_checkable
+class SupportsInterchange(Protocol):
+    """Dataframe that supports conversion into an interchange dataframe object."""
+
+    def __dataframe__(
+        self,
+        nan_as_null: bool = False,
+        allow_copy: bool = True,
+    ) -> SupportsInterchange:
+        """Convert to a dataframe object implementing the dataframe interchange protocol."""
+        ...

lazyscribe_arrow-0.2.1/pyproject.toml

@@ -0,0 +1,90 @@
+[build-system]
+build-backend = "uv_build"
+
+requires = [ "uv-build>=0.8,<0.9" ]
+
+[project]
+name = "lazyscribe-arrow"
+version = "0.2.1"
+
+description = "Arrow-based artifact handlers for Lazyscribe"
+readme = { file = "README.md", content-type = "text/markdown" }
+license = { text = "MIT license" }
+authors = [
+  { name = "Akshay Gupta", email = "akgcodes@gmail.com" },
+]
+requires-python = ">=3.10.0"
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "License :: OSI Approved :: MIT License",
+  "Natural Language :: English",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+]
+dependencies = [ "attrs>=21.2,<=25.3", "lazyscribe>=1,<=1.2", "pyarrow>=14.0.1,<=21", "python-slugify>=5,<=8.0.4" ]
+
+optional-dependencies.build = [ "commitizen", "uv" ]
+optional-dependencies.dev = [ "lazyscribe-arrow[build]", "lazyscribe-arrow[qa]", "lazyscribe-arrow[tests]" ]
+optional-dependencies.qa = [ "edgetest", "mypy", "pre-commit", "pyproject-fmt", "ruff", "types-python-slugify", "uv" ]
+optional-dependencies.tests = [ "pandas", "pytest", "pytest-cov", "time-machine" ]
+urls."Documentation" = "https://github.com/lazyscribe/lazyscribe-arrow"
+urls."Repository" = "https://github.com/lazyscribe/lazyscribe-arrow"
+entry-points."lazyscribe.artifact_type".csv = "lazyscribe_arrow:CSVArtifact"
+entry-points."lazyscribe.artifact_type".parquet = "lazyscribe_arrow:ParquetArtifact"
+
+[tool.ruff]
+target-version = "py310"
+
+lint.select = [
+  "B",      # flake8-bugbear
+  "C",      # flake8-comprehensions
+  "D",      # pydocstyle
+  "E",      # pycodestyle errors
+  "F",      # pyflakes
+  "I",      # isort
+  "LOG",    # flake8-logging
+  "RUF",    # Ruff errors
+  "SIM",    # flake8-simplify
+  "T20",    # flake8-print
+  "TID252", # flake8-tidy-imports ban relative imports
+  "UP",     # pyupgrade
+  "W",      # pycodestyle warnings
+]
+lint.ignore = [
+  "C901", # Function/method is too complex. (Add back in later.)
+  "E501", # Line too long. Using formatter instead.
+]
+lint.per-file-ignores."**/{tests,docs}/*" = [ "ARG", "D", "E402", "F841" ]
+lint.per-file-ignores."__init__.py" = [ "E402" ]
+lint.per-file-ignores."tutorials/*" = [ "D", "T201" ]
+lint.flake8-tidy-imports.ban-relative-imports = "all"
+lint.pydocstyle.convention = "numpy"
+lint.preview = true
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true
+ignore_missing_imports = true
+allow_redefinition = true
+check_untyped_defs = true
+
+[tool.uv.build-backend]
+module-name = "lazyscribe_arrow"
+module-root = ""
+
+[tool.commitizen]
+version_provider = "pep621"
+tag_format = "v$version"
+update_changelog_on_bump = true
+version_files = [
+  "lazyscribe_arrow/_meta.py:__version__",
+]
+
+[edgetest.envs.core]
+extras = [ "tests" ]
+upgrade = [ "attrs", "lazyscribe", "pyarrow", "python-slugify" ]
+command = "pytest tests"