pipeloom-pandas 0.2.0__tar.gz

pipeloom_pandas-0.2.0/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+.venv/
+build/
+dist/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+coverage.xml
+htmlcov/
+site/
+target/
+.pipeloom/
+.env

pipeloom_pandas-0.2.0/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pipeloom_pandas-0.2.0/NOTICE

@@ -0,0 +1,2 @@
+pipeloom
+Copyright 2026 Emanuel Luis

pipeloom_pandas-0.2.0/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.5
+Name: pipeloom-pandas
+Version: 0.2.0
+Summary: pandas engine for pipeloom: eager DataFrames and file tables.
+Project-URL: Repository, https://github.com/emanuel-luis/pipeloom
+Project-URL: Documentation, https://emanuel-luis.github.io/pipeloom/
+Project-URL: Changelog, https://github.com/emanuel-luis/pipeloom/blob/main/CHANGELOG.md
+Author-email: Emanuel Luis <emanuel.filho08@gmail.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Keywords: data-engineering,pandas,pipelines,pipeloom
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: pandas<4,>=2.3.3
+Requires-Dist: pipeloom~=0.2.0
+Requires-Dist: pyarrow>=22
+Description-Content-Type: text/markdown
+
+# pipeloom-pandas
+
+The [pandas](https://pandas.pydata.org) engine for [pipeloom](https://github.com/emanuel-luis/pipeloom).
+
+```console
+$ pip install "pipeloom[pandas]"
+```
+
+Select it in a target:
+
+```yaml
+targets:
+  dev:
+    engine: {kind: pandas}
+    catalog: {kind: filesystem, root: warehouse}
+```
+
+- Relations are eager `DataFrame`s. Python models receive `DataFrame` inputs, or Narwhals frames
+  with `frame_api: narwhals`.
+- No SQL models and no views in 0.2: validation rejects them (PLC502, PLC507).
+- Reads `parquet`, `csv`, `json` and `ndjson` sources. Writes tables through the `filesystem`
+  catalog with the `append` and `overwrite` strategies, and an idempotency token makes a repeated
+  write a no-op.
+
+Documentation: https://emanuel-luis.github.io/pipeloom/latest/plugins/pandas/
+
+License: Apache-2.0.

pipeloom_pandas-0.2.0/README.md

@@ -0,0 +1,27 @@
+# pipeloom-pandas
+
+The [pandas](https://pandas.pydata.org) engine for [pipeloom](https://github.com/emanuel-luis/pipeloom).
+
+```console
+$ pip install "pipeloom[pandas]"
+```
+
+Select it in a target:
+
+```yaml
+targets:
+  dev:
+    engine: {kind: pandas}
+    catalog: {kind: filesystem, root: warehouse}
+```
+
+- Relations are eager `DataFrame`s. Python models receive `DataFrame` inputs, or Narwhals frames
+  with `frame_api: narwhals`.
+- No SQL models and no views in 0.2: validation rejects them (PLC502, PLC507).
+- Reads `parquet`, `csv`, `json` and `ndjson` sources. Writes tables through the `filesystem`
+  catalog with the `append` and `overwrite` strategies, and an idempotency token makes a repeated
+  write a no-op.
+
+Documentation: https://emanuel-luis.github.io/pipeloom/latest/plugins/pandas/
+
+License: Apache-2.0.

pipeloom_pandas-0.2.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "pipeloom-pandas"
+version = "0.2.0"
+description = "pandas engine for pipeloom: eager DataFrames and file tables."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+license-files = ["LICENSE", "NOTICE"]
+authors = [{ name = "Emanuel Luis", email = "emanuel.filho08@gmail.com" }]
+keywords = ["pipeloom", "pandas", "data-engineering", "pipelines"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Typing :: Typed",
+]
+dependencies = ["pipeloom~=0.2.0", "pandas>=2.3.3,<4", "pyarrow>=22"]
+
+[project.urls]
+Repository = "https://github.com/emanuel-luis/pipeloom"
+Documentation = "https://emanuel-luis.github.io/pipeloom/"
+Changelog = "https://github.com/emanuel-luis/pipeloom/blob/main/CHANGELOG.md"
+
+[project.entry-points."pipeloom.engines"]
+pandas = "pipeloom_pandas.spec:SPEC"
+
+[tool.uv.sources]
+pipeloom = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pipeloom_pandas"]
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src", "tests", "LICENSE", "NOTICE", "README.md"]

pipeloom_pandas-0.2.0/src/pipeloom_pandas/__init__.py

@@ -0,0 +1 @@
+"""pandas engine plugin for pipeloom. Import `pipeloom_pandas.spec` to read its capabilities."""

pipeloom_pandas-0.2.0/src/pipeloom_pandas/engine.py

@@ -0,0 +1,154 @@
+"""The pandas engine: eager DataFrames, tables on the filesystem catalog. No SQL in 0.2."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any
+
+import pandas as pd
+
+from pipeloom.builtins.filesystem import FilesystemCatalog
+from pipeloom.builtins.fs_table import FsTable
+from pipeloom.engine import portable
+from pipeloom.engine.protocols import Catalog, EngineSession, Materializer, Reader
+from pipeloom.engine.types import ReadSpec, Relation, Schema, TargetRef, WriteConfig, WriteResult
+from pipeloom_pandas import io
+from pipeloom_pandas.spec import CAPABILITIES
+
+if TYPE_CHECKING:
+    from pipeloom.runtime.context import RuntimeContext
+
+NAME = "pandas"
+
+
+def _rel(frame: pd.DataFrame) -> Relation:
+    return Relation(frame, NAME, lazy=False)
+
+
+class _Reader:
+    def __init__(self, session: PandasSession, fmt: str) -> None:
+        self._session = session
+        self._fmt = fmt
+
+    def read(self, spec: ReadSpec) -> Relation:
+        if self._fmt == "table":
+            if spec.table is None:
+                raise ValueError("a table read needs a table")
+            return self._session.read_table(spec.table)
+        if spec.location is None:
+            raise ValueError(f"reading {self._fmt} needs a location")
+        return _rel(io.read(self._fmt, spec.location, spec.options))
+
+
+class _Materializer:
+    def __init__(self, session: PandasSession, strategy: str) -> None:
+        self.strategy = strategy
+        self._session = session
+
+    def materialize(
+        self, rel: Relation, target: TargetRef, cfg: WriteConfig, token: str
+    ) -> WriteResult:
+        catalog = self._session.catalog
+        table = catalog.table(target)
+        replay = table.applied(token)
+        if replay is not None:
+            return replay
+        path = table.new_file(io.EXTENSIONS[catalog.format])
+        try:
+            df: pd.DataFrame = rel.native
+            io.write(df, catalog.format, path)
+            commit = table.commit(
+                mode="append" if self.strategy == "append" else "overwrite",
+                files=[path],
+                columns=portable.schema(df),
+                rows=len(df),
+                token=token,
+                result=WriteResult(rows_inserted=len(df), rows_written=len(df)),
+            )
+        except BaseException:
+            table.discard([path])
+            raise
+        return commit.result
+
+
+class PandasSession:
+    """Thread-safe: it holds no mutable state. A model must not mutate its inputs in place."""
+
+    native: Any
+
+    def __init__(self, catalog: Catalog) -> None:
+        if not isinstance(catalog, FilesystemCatalog):
+            raise TypeError(
+                f"the pandas engine needs the filesystem catalog, not {type(catalog).__name__}"
+            )
+        self.catalog = catalog
+        self.native = pd
+
+    # -- reading ------------------------------------------------------------------------------
+
+    def reader(self, fmt: str) -> Reader:
+        if fmt != "table" and fmt not in CAPABILITIES.formats:
+            raise ValueError(f"pandas engine cannot read format '{fmt}'")
+        return _Reader(self, fmt)
+
+    def read_table(self, target: TargetRef) -> Relation:
+        table: FsTable = self.catalog.table(target)
+        commit = table.current()
+        if commit is None:
+            raise FileNotFoundError(f"table '{target.relation_name}' does not exist")
+        if commit.kind == "view" or commit.format is None:
+            raise ValueError(f"'{target.relation_name}' is a view; the pandas engine has none")
+        return _rel(io.read_parts(commit.format, table.files(), commit.columns))
+
+    # -- writing ------------------------------------------------------------------------------
+
+    def materializer(self, strategy: str) -> Materializer:
+        if strategy not in CAPABILITIES.strategies:
+            raise ValueError(f"pandas engine does not support strategy '{strategy}'")
+        return _Materializer(self, strategy)
+
+    def create_view(
+        self, target: TargetRef, query: str, inputs: Mapping[str, ReadSpec]
+    ) -> WriteResult:
+        raise NotImplementedError("the pandas engine has no views")
+
+    # -- relations ----------------------------------------------------------------------------
+
+    def sql(self, query: str, inputs: Mapping[str, Relation]) -> Relation:
+        raise NotImplementedError("the pandas engine has no SQL in 0.2")
+
+    def relation(self, native: Any) -> Relation:
+        if not isinstance(native, pd.DataFrame):
+            raise TypeError(f"the pandas engine takes a DataFrame, not {type(native).__name__}")
+        return _rel(native)
+
+    def persist(self, rel: Relation) -> Relation:
+        return rel  # already in memory
+
+    def schema(self, rel: Relation) -> Schema:
+        return portable.schema(rel.native)
+
+    def count(self, rel: Relation) -> int:
+        return portable.count(rel.native)
+
+    def filter_since(self, rel: Relation, column: str, lower: Any) -> Relation:
+        return _rel(portable.filter_since(rel.native, column, lower))
+
+    def max_value(self, rel: Relation, column: str) -> Any:
+        return portable.max_value(rel.native, column)
+
+    def cancel(self, unit_id: str) -> None:
+        return None
+
+
+class PandasEngine:
+    name = NAME
+    capabilities = CAPABILITIES
+
+    def __init__(self) -> None:
+        self.version = pd.__version__
+
+    @contextmanager
+    def session(self, ctx: RuntimeContext, *, catalog: Catalog) -> Iterator[EngineSession]:
+        yield PandasSession(catalog)

pipeloom_pandas-0.2.0/src/pipeloom_pandas/io.py

@@ -0,0 +1,120 @@
+"""Reading and writing files, and the logical-to-Arrow type mapping."""
+
+from __future__ import annotations
+
+import glob
+import re
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any, Final
+
+import pandas as pd
+import pyarrow as pa
+
+from pipeloom.engine.types import Schema
+
+EXTENSIONS: Final = {"parquet": "parquet", "csv": "csv", "ndjson": "ndjson"}
+"""Part file extension by catalog format."""
+
+_SIMPLE: Final[Mapping[str, Any]] = {
+    "int8": pa.int8(),
+    "int16": pa.int16(),
+    "int32": pa.int32(),
+    "int64": pa.int64(),
+    "uint8": pa.uint8(),
+    "uint16": pa.uint16(),
+    "uint32": pa.uint32(),
+    "uint64": pa.uint64(),
+    "float32": pa.float32(),
+    "float64": pa.float64(),
+    "string": pa.string(),
+    "boolean": pa.bool_(),
+    "date": pa.date32(),
+    "time": pa.time64("us"),
+    "binary": pa.binary(),
+}
+_TIMESTAMP = re.compile(r"timestamp\[(ms|us|ns)(?:, ([^\]]+))?\]")
+_DECIMAL = re.compile(r"decimal\((\d+),(\d+)\)")
+
+
+def arrow_type(logical: str) -> Any | None:
+    """The Arrow type of a logical type, or None for one this engine cannot name."""
+    simple = _SIMPLE.get(logical)
+    if simple is not None:
+        return simple
+    ts = _TIMESTAMP.fullmatch(logical)
+    if ts:
+        return pa.timestamp(ts.group(1), tz=ts.group(2))
+    dec = _DECIMAL.fullmatch(logical)
+    if dec:
+        return pa.decimal128(int(dec.group(1)), int(dec.group(2)))
+    return None
+
+
+def _concat(frames: list[pd.DataFrame]) -> pd.DataFrame:
+    return frames[0] if len(frames) == 1 else pd.concat(frames, ignore_index=True)
+
+
+def _files(location: str) -> list[str]:
+    if not any(ch in location for ch in "*?["):
+        return [location]
+    found = sorted(glob.glob(location))
+    if not found:
+        raise FileNotFoundError(f"no file matches '{location}'")
+    return found
+
+
+def _read_one(
+    fmt: str, path: str, options: Mapping[str, Any], text_columns: Sequence[str] = ()
+) -> pd.DataFrame:
+    # Arrow-backed columns keep nulls in integer columns and keep the logical types.
+    kw: dict[str, Any] = {"dtype_backend": "pyarrow", **options}
+    if fmt == "parquet":
+        return pd.read_parquet(path, **kw)
+    if fmt == "csv":
+        # Strings are read as strings: inferring first would turn "007" into 7.
+        return pd.read_csv(path, **{"dtype": dict.fromkeys(text_columns, "string[pyarrow]"), **kw})
+    if fmt == "ndjson":
+        return pd.read_json(path, lines=True, **{"dtype": False, **kw})
+    if fmt == "json":
+        return pd.read_json(path, **{"dtype": False, **kw})
+    raise ValueError(f"pandas engine cannot read format '{fmt}'")
+
+
+def read(fmt: str, location: str, options: Mapping[str, Any]) -> pd.DataFrame:
+    """A DataFrame over a source file or a glob."""
+    return _concat([_read_one(fmt, p, options) for p in _files(location)])
+
+
+def read_parts(fmt: str, files: Sequence[Path], schema: Schema) -> pd.DataFrame:
+    """The committed part files of a table, concatenated, with the stored logical types."""
+    frames = []
+    for path in files:
+        text = [c.name for c in schema if c.dtype == "string"]
+        df = _read_one(fmt, str(path), {}, text)
+        if fmt != "parquet":  # csv and ndjson lose the types; parquet keeps them
+            for col in schema:
+                dtype = arrow_type(col.dtype)
+                if dtype is None or col.name not in df.columns:
+                    continue  # an unnamed type keeps what the reader inferred
+                series = df[col.name]
+                if pa.types.is_date(dtype) and pd.api.types.is_string_dtype(series):
+                    series = series.str.slice(0, 10)  # ndjson writes a date as an ISO datetime
+                if pa.types.is_timestamp(dtype) and pd.api.types.is_string_dtype(series):
+                    # ndjson writes nanosecond ISO text, which a direct Arrow cast rejects
+                    stamps = pd.to_datetime(series, utc=dtype.tz is not None)
+                    series = stamps.dt.tz_convert(dtype.tz) if dtype.tz else stamps
+                df[col.name] = series.astype(pd.ArrowDtype(dtype))
+        frames.append(df)
+    return _concat(frames)
+
+
+def write(df: pd.DataFrame, fmt: str, path: Path) -> None:
+    if fmt == "parquet":
+        df.to_parquet(path, engine="pyarrow", index=False)
+    elif fmt == "csv":
+        df.to_csv(path, index=False, lineterminator="\n")
+    elif fmt == "ndjson":
+        df.to_json(path, orient="records", lines=True, date_format="iso", date_unit="ns")
+    else:
+        raise ValueError(f"cannot write format '{fmt}'")

pipeloom_pandas-0.2.0/src/pipeloom_pandas/spec.py

@@ -0,0 +1,43 @@
+"""The plugin spec. It imports no dataframe library: the engine module loads on first use."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pipeloom.engine.protocols import Engine
+from pipeloom.engine.types import EngineCapabilities
+from pipeloom.plugins import API_VERSION, OptionsModel, PluginSpec
+
+CAPABILITIES = EngineCapabilities(
+    formats=frozenset({"parquet", "csv", "json", "ndjson"}),
+    strategies=frozenset({"append", "overwrite"}),
+    materializations=frozenset({"table", "ephemeral"}),
+    sql_models=False,
+    views=False,
+    idempotent_writes=True,
+    cancel=False,
+    bytes_metrics=True,
+    portable=True,
+)
+
+
+class PandasOptions(OptionsModel):
+    """The pandas engine has no options."""
+
+
+def _build(opts: Any) -> Engine:
+    from pipeloom_pandas.engine import PandasEngine
+
+    return PandasEngine()
+
+
+SPEC: PluginSpec[Engine] = PluginSpec(
+    "pandas",
+    "pandas",
+    "engines",
+    API_VERSION,
+    _build,
+    PandasOptions,
+    capabilities=CAPABILITIES,
+    description="Eager pandas DataFrames; tables on the filesystem catalog; no SQL in 0.2.",
+)

pipeloom_pandas-0.2.0/tests/test_contract.py

@@ -0,0 +1,61 @@
+"""The pandas engine passes the engine contract."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import pyarrow as pa
+import pytest
+
+from pipeloom.builtins.filesystem import FilesystemCatalog
+from pipeloom.engine.protocols import EngineSession
+from pipeloom.engine.types import Relation, Schema
+from pipeloom.testing.contract.engine import EngineContract, EngineHarness
+from pipeloom_pandas import io
+from pipeloom_pandas.engine import PandasEngine
+
+Rows = Sequence[Mapping[str, Any]]
+
+
+def _frame(rows: Rows, schema: Schema) -> pd.DataFrame:
+    arrow = pa.schema([(c.name, io.arrow_type(c.dtype)) for c in schema])
+    table = pa.Table.from_pylist([dict(r) for r in rows], schema=arrow)
+    return table.to_pandas(types_mapper=pd.ArrowDtype)
+
+
+def harness(tmp_path: Path) -> EngineHarness:
+    def write_source(fmt: str, path: Path, rows: Rows, schema: Schema) -> None:
+        df = _frame(rows, schema)
+        if fmt == "json":
+            df.to_json(path, orient="records")
+        else:
+            io.write(df, fmt, path)
+
+    def to_rows(native: Any) -> list[dict[str, Any]]:
+        table = pa.Table.from_pandas(native, preserve_index=False)
+        return list(table.to_pylist())
+
+    def failing(session: EngineSession) -> Relation:
+        return session.relation(pd.DataFrame({"a": [object()]}))  # parquet cannot store it
+
+    return EngineHarness(
+        engine=PandasEngine(),
+        catalog=FilesystemCatalog(tmp_path / "wh"),
+        make_native=_frame,
+        to_rows=to_rows,
+        write_source=write_source,
+        failing_relation=failing,
+        settings_snapshot=lambda: {
+            "string_storage": pd.get_option("mode.string_storage"),
+            "max_rows": pd.get_option("display.max_rows"),
+        },
+    )
+
+
+class TestPandasEngine(EngineContract):
+    @pytest.fixture
+    def engine_factory(self) -> Callable[[Path], EngineHarness]:
+        return harness

pipeloom_pandas-0.2.0/tests/test_engine.py

@@ -0,0 +1,154 @@
+"""pandas engine details beyond the contract."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from pipeloom.api import Project, ProjectOptions
+from pipeloom.builtins.filesystem import FilesystemCatalog
+from pipeloom.engine import portable
+from pipeloom.engine.types import ReadSpec, TargetRef, WriteConfig
+from pipeloom.platform import Identity
+from pipeloom.runtime.context import RuntimeContext
+from pipeloom.testing.clock import FakeClock
+from pipeloom.testing.memory import MemoryCatalog
+from pipeloom_pandas.engine import PandasEngine
+
+T = TargetRef(None, "main", "t")
+
+
+def ctx(tmp_path: Path) -> RuntimeContext:
+    return RuntimeContext("r", "i", "p", "t", tmp_path, FakeClock(), Identity("u", "h"), {})
+
+
+@pytest.mark.parametrize("fmt", ["parquet", "csv", "ndjson"])
+def test_tables_round_trip_in_every_catalog_format(tmp_path: Path, fmt: str) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh", format=fmt)
+    with PandasEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        catalog.ensure_namespace(T.namespace)
+        rel = s.relation(pd.DataFrame({"id": [1, 2], "label": ["a", None]}))
+        s.materializer("overwrite").materialize(rel, T, WriteConfig("overwrite"), "r:1")
+        out = s.reader("table").read(ReadSpec("table", table=T))
+        frame = out.native.sort_values("id")
+        assert [(r.id, None if pd.isna(r.label) else r.label) for r in frame.itertuples()] == [
+            (1, "a"),
+            (2, None),
+        ]
+        assert [c.dtype for c in s.schema(out)] == ["int64", "string"]
+    assert len(list((tmp_path / "wh" / "main" / "t").glob(f"part-*.{fmt}"))) == 1
+
+
+def test_a_replayed_token_keeps_the_commit(tmp_path: Path) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh")
+    with PandasEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        catalog.ensure_namespace(T.namespace)
+        rel = s.relation(pd.DataFrame({"id": [1]}))
+        mat = s.materializer("append")
+        first = mat.materialize(rel, T, WriteConfig("append"), "r:1")
+        before = sorted(p.name for p in (tmp_path / "wh" / "main" / "t").iterdir())
+        assert mat.materialize(rel, T, WriteConfig("append"), "r:1") == first
+        assert sorted(p.name for p in (tmp_path / "wh" / "main" / "t").iterdir()) == before
+        assert first.rows_written == 1 and first.bytes_written and first.target_version == 1
+
+
+def test_the_session_needs_the_filesystem_catalog(tmp_path: Path) -> None:
+    with (
+        pytest.raises(TypeError, match="filesystem catalog"),
+        PandasEngine().session(ctx(tmp_path), catalog=MemoryCatalog()),
+    ):
+        pass
+
+
+def test_unsupported_format_strategy_sql_and_view(tmp_path: Path) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh")
+    with PandasEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        with pytest.raises(ValueError, match="format"):
+            s.reader("orc")
+        with pytest.raises(ValueError, match="strategy"):
+            s.materializer("merge")
+        with pytest.raises(NotImplementedError):
+            s.sql("select 1", {})
+        with pytest.raises(NotImplementedError):
+            s.create_view(T, "select 1", {})
+
+
+def test_a_missing_table_raises(tmp_path: Path) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh")
+    with (
+        PandasEngine().session(ctx(tmp_path), catalog=catalog) as s,
+        pytest.raises(FileNotFoundError),
+    ):
+        s.reader("table").read(ReadSpec("table", table=T))
+
+
+def test_string_columns_map_to_string_in_either_pandas_generation() -> None:
+    df = pd.DataFrame({"a": ["x", "y"], "b": pd.array(["x", None], dtype="string")})
+    assert [(c.name, c.dtype) for c in portable.schema(df)] == [("a", "string"), ("b", "string")]
+
+
+PANDAS_PROJECT = {
+    "pipeloom.yml": (
+        "config_version: 1\nname: p\npython_paths: [src]\n"
+        "targets:\n  dev:\n    engine: {kind: pandas}\n"
+        "    catalog: {kind: filesystem, root: wh}\n"
+    ),
+}
+
+
+def codes(tmp_path: Path, model: str) -> set[str]:
+    files = {**PANDAS_PROJECT, "models/m.yml": f"models:\n  - {model}\n"}
+    for rel, content in files.items():
+        (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True)
+        (tmp_path / rel).write_bytes(content.encode("utf-8"))
+    root = tmp_path
+    outcome = Project(ProjectOptions(project_dir=root, target="dev", env={})).validate()
+    return {d.code for d in outcome.diagnostics}
+
+
+def test_a_sql_model_on_pandas_is_plc502(tmp_path: Path) -> None:
+    assert "PLC502" in codes(tmp_path, "{name: a, sql: select 1}")
+
+
+def test_a_view_on_pandas_is_plc507(tmp_path: Path) -> None:
+    assert "PLC507" in codes(tmp_path, "{name: a, sql: select 1, materialized: view}")
+
+
+@pytest.mark.parametrize("fmt", ["csv", "ndjson"])
+def test_text_formats_restore_each_stored_type_even_beside_an_unknown_one(
+    tmp_path: Path, fmt: str
+) -> None:
+    from datetime import UTC, date, datetime
+    from decimal import Decimal
+
+    import pyarrow as pa
+
+    arrow = pa.table(
+        {
+            "d": pa.array([date(2026, 1, 2), None], pa.date32()),
+            "t": pa.array(
+                [datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC), None], pa.timestamp("us", "UTC")
+            ),
+            "n": pa.array([Decimal("12.34"), None], pa.decimal128(18, 2)),
+            "i": pa.array([1, None], pa.int64()),
+            "z": pa.array(["007", None], pa.string()),
+            "us": pa.array([datetime(2026, 1, 2, 3, 4, 5, 123456), None], pa.timestamp("us")),
+            "u": pa.array([[1], None], pa.list_(pa.int64())),  # a type the engine cannot name
+        }
+    )
+    catalog = FilesystemCatalog(tmp_path / "wh", format=fmt)
+    with PandasEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        catalog.ensure_namespace(T.namespace)
+        rel = s.relation(arrow.to_pandas(types_mapper=pd.ArrowDtype))
+        s.materializer("overwrite").materialize(rel, T, WriteConfig("overwrite"), "r:1")
+        out = s.reader("table").read(ReadSpec("table", table=T))
+        got = {c.name: c.dtype for c in s.schema(out)}
+        want = {c.name: c.dtype for c in s.schema(rel)}
+        keys = ["d", "t", "n", "i", "z", "us"]
+        assert {k: got[k] for k in keys} == {k: want[k] for k in keys}
+        assert out.native["z"].iloc[0] == "007"
+        assert out.native["us"].iloc[0] == pd.Timestamp("2026-01-02 03:04:05.123456")
+        assert out.native["t"].iloc[0] == pd.Timestamp("2026-01-02 03:04:05", tz="UTC")
+        assert out.native["n"].iloc[0] == Decimal("12.34")

pipeloom_pandas-0.2.0/tests/test_shop_e2e.py

@@ -0,0 +1,97 @@
+"""The pandas smoke project end to end, with both executors.
+
+The project is `tests/fixtures/smoke-pandas`: csv sources, Python models (one on Narwhals frames),
+an ephemeral model and an append table, on the filesystem catalog and json state.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import pytest
+
+from pipeloom.api import Project, ProjectOptions
+from pipeloom.builtins.fs_table import FsTable
+from pipeloom.events.bus import EventBus
+from pipeloom.runtime.run import RunOptions, RunOutcome
+from pipeloom.testing.clock import FakeClock
+from pipeloom.testing.recording import RecordingSink
+
+SMOKE = Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "smoke-pandas"
+
+
+def project(tmp_path: Path, boundary: str) -> Path:
+    shutil.copytree(
+        SMOKE, tmp_path, ignore=shutil.ignore_patterns("target", ".pipeloom"), dirs_exist_ok=True
+    )
+    yml = tmp_path / "pipeloom.yml"
+    text = yml.read_text(encoding="utf-8").replace(
+        "python_paths:", f"boundary: {boundary}\npython_paths:"
+    )
+    yml.write_text(text, encoding="utf-8", newline="\n")
+    return tmp_path
+
+
+def run(root: Path, run_id: str) -> tuple[RunOutcome, RecordingSink]:
+    clock = FakeClock(datetime(2026, 1, 1, tzinfo=UTC), step=timedelta(0))
+    sink = RecordingSink()
+    proj = Project(
+        ProjectOptions(project_dir=root, target="pandas", env={"PIPELOOM_USER": "tester"}),
+        clock=clock,
+        invocation_id="inv",
+    )
+    with EventBus([(sink, 5)], clock=clock, invocation_id="inv") as bus:
+        return proj.run(RunOptions(run_id=run_id, workers=2), bus), sink
+
+
+def table(root: Path, name: str) -> list[dict[str, Any]]:
+    t = FsTable(root / "target" / "warehouse" / "main" / name, clock=FakeClock())
+    df = pd.concat([pd.read_parquet(f) for f in t.files()], ignore_index=True)
+    df = df.sort_values(list(df.columns)).reset_index(drop=True)
+    return [
+        {k: v.item() if hasattr(v, "item") else v for k, v in r.items()}
+        for r in df.to_dict("records")
+    ]
+
+
+def snapshot(root: Path) -> dict[str, bytes]:
+    wh = root / "target" / "warehouse"
+    return {
+        p.relative_to(wh).as_posix(): p.read_bytes() for p in sorted(wh.rglob("*")) if p.is_file()
+    }
+
+
+@pytest.mark.parametrize(("boundary", "executor"), [("process", "thread"), ("task", "process")])
+def test_smoke_runs_and_a_rerun_with_the_same_run_id_writes_nothing(
+    tmp_path: Path, boundary: str, executor: str
+) -> None:
+    root = project(tmp_path, boundary)
+    outcome, sink = run(root, "r1")
+    assert (outcome.status, outcome.exit_code) == ("succeeded", 0), outcome.diagnostics
+    doc = json.loads((root / "target" / "runs" / "r1" / "run_results.json").read_bytes())
+    assert {r["executor"] for r in doc["results"]} == {executor}
+    assert {r["engine"] for r in doc["results"]} == {"pandas"}
+    assert table(root, "stg_customers") == [
+        {"id": 1, "email": "a@x.com"},
+        {"id": 2, "email": "b@x.com"},
+    ]
+    assert table(root, "fct_orders") == [
+        {"id": 10, "customer_id": 1, "email": "a@x.com", "amount": 5.0},
+        {"id": 11, "customer_id": 2, "email": "b@x.com", "amount": 7.5},
+        {"id": 12, "customer_id": 1, "email": "a@x.com", "amount": 2.5},
+    ]
+    assert table(root, "rpt_revenue") == [
+        {"customer_id": 1, "revenue": 7.5},
+        {"customer_id": 2, "revenue": 7.5},
+    ]
+    assert not [e for e in sink.events if e.type == "unit.failed"]
+
+    before = snapshot(root)
+    again, _ = run(root, "r1")
+    assert again.exit_code == 0
+    assert snapshot(root) == before  # token replay: no new part file, commits untouched

pipeloom_pandas-0.2.0/tests/test_spec_import.py

@@ -0,0 +1,25 @@
+"""The spec module is readable without importing pandas or pyarrow."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+CODE = """
+import sys
+import pipeloom_pandas.spec as spec
+assert spec.SPEC.capabilities.portable
+assert not spec.SPEC.capabilities.sql_models
+assert "pandas" not in sys.modules, "pandas was imported"
+assert "pyarrow" not in sys.modules, "pyarrow was imported"
+assert "pipeloom_pandas.engine" not in sys.modules
+print("ok")
+"""
+
+
+def test_spec_import_does_not_import_pandas() -> None:
+    proc = subprocess.run(
+        [sys.executable, "-I", "-c", CODE], capture_output=True, text=True, check=False
+    )
+    assert proc.returncode == 0, proc.stderr
+    assert proc.stdout.strip() == "ok"