pipeloom-polars 0.2.0__tar.gz

pipeloom_polars-0.2.0/.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+.venv/
+build/
+dist/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+coverage.xml
+htmlcov/
+site/
+target/
+.pipeloom/
+.env

pipeloom_polars-0.2.0/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pipeloom_polars-0.2.0/NOTICE

@@ -0,0 +1,2 @@
+pipeloom
+Copyright 2026 Emanuel Luis

pipeloom_polars-0.2.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.5
+Name: pipeloom-polars
+Version: 0.2.0
+Summary: Polars engine for pipeloom: lazy frames, SQL and file tables.
+Project-URL: Repository, https://github.com/emanuel-luis/pipeloom
+Project-URL: Documentation, https://emanuel-luis.github.io/pipeloom/
+Project-URL: Changelog, https://github.com/emanuel-luis/pipeloom/blob/main/CHANGELOG.md
+Author-email: Emanuel Luis <emanuel.filho08@gmail.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+License-File: NOTICE
+Keywords: data-engineering,pipelines,pipeloom,polars
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: pipeloom~=0.2.0
+Requires-Dist: polars<3,>=1.30
+Description-Content-Type: text/markdown
+
+# pipeloom-polars
+
+The [Polars](https://pola.rs) engine for [pipeloom](https://github.com/emanuel-luis/pipeloom).
+
+```console
+$ pip install "pipeloom[polars]"
+```
+
+Select it in a target:
+
+```yaml
+targets:
+  dev:
+    engine: {kind: polars}
+    catalog: {kind: filesystem, root: warehouse}
+```
+
+- Relations are `LazyFrame`s. Python models receive `LazyFrame` inputs, or Narwhals frames with
+  `frame_api: narwhals`.
+- SQL models and views run on `pl.SQLContext`.
+- Reads `parquet`, `csv`, `json` and `ndjson` sources. Writes tables through the `filesystem`
+  catalog with the `append` and `overwrite` strategies, and an idempotency token makes a repeated
+  write a no-op.
+
+Documentation: https://emanuel-luis.github.io/pipeloom/latest/plugins/polars/
+
+License: Apache-2.0.

pipeloom_polars-0.2.0/README.md

@@ -0,0 +1,27 @@
+# pipeloom-polars
+
+The [Polars](https://pola.rs) engine for [pipeloom](https://github.com/emanuel-luis/pipeloom).
+
+```console
+$ pip install "pipeloom[polars]"
+```
+
+Select it in a target:
+
+```yaml
+targets:
+  dev:
+    engine: {kind: polars}
+    catalog: {kind: filesystem, root: warehouse}
+```
+
+- Relations are `LazyFrame`s. Python models receive `LazyFrame` inputs, or Narwhals frames with
+  `frame_api: narwhals`.
+- SQL models and views run on `pl.SQLContext`.
+- Reads `parquet`, `csv`, `json` and `ndjson` sources. Writes tables through the `filesystem`
+  catalog with the `append` and `overwrite` strategies, and an idempotency token makes a repeated
+  write a no-op.
+
+Documentation: https://emanuel-luis.github.io/pipeloom/latest/plugins/polars/
+
+License: Apache-2.0.

pipeloom_polars-0.2.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "pipeloom-polars"
+version = "0.2.0"
+description = "Polars engine for pipeloom: lazy frames, SQL and file tables."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+license-files = ["LICENSE", "NOTICE"]
+authors = [{ name = "Emanuel Luis", email = "emanuel.filho08@gmail.com" }]
+keywords = ["pipeloom", "polars", "data-engineering", "pipelines"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Typing :: Typed",
+]
+dependencies = ["pipeloom~=0.2.0", "polars>=1.30,<3"]
+
+[project.urls]
+Repository = "https://github.com/emanuel-luis/pipeloom"
+Documentation = "https://emanuel-luis.github.io/pipeloom/"
+Changelog = "https://github.com/emanuel-luis/pipeloom/blob/main/CHANGELOG.md"
+
+[project.entry-points."pipeloom.engines"]
+polars = "pipeloom_polars.spec:SPEC"
+
+[tool.uv.sources]
+pipeloom = { workspace = true }
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pipeloom_polars"]
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src", "tests", "LICENSE", "NOTICE", "README.md"]

pipeloom_polars-0.2.0/src/pipeloom_polars/__init__.py

@@ -0,0 +1 @@
+"""Polars engine plugin for pipeloom. Import `pipeloom_polars.spec` to read its capabilities."""

pipeloom_polars-0.2.0/src/pipeloom_polars/engine.py

@@ -0,0 +1,181 @@
+"""The polars engine: lazy frames, SQL on `pl.SQLContext`, tables on the filesystem catalog."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any
+
+import polars as pl
+
+from pipeloom.builtins.filesystem import FilesystemCatalog
+from pipeloom.builtins.fs_table import FsTable, ViewDefinition
+from pipeloom.engine import portable
+from pipeloom.engine.protocols import Catalog, EngineSession, Materializer, Reader
+from pipeloom.engine.types import ReadSpec, Relation, Schema, TargetRef, WriteConfig, WriteResult
+from pipeloom_polars import io
+from pipeloom_polars.spec import CAPABILITIES
+
+if TYPE_CHECKING:
+    from pipeloom.runtime.context import RuntimeContext
+
+NAME = "polars"
+
+
+def _rel(frame: pl.LazyFrame) -> Relation:
+    return Relation(frame, NAME, lazy=True)
+
+
+class _Reader:
+    def __init__(self, session: PolarsSession, fmt: str) -> None:
+        self._session = session
+        self._fmt = fmt
+
+    def read(self, spec: ReadSpec) -> Relation:
+        if self._fmt == "table":
+            if spec.table is None:
+                raise ValueError("a table read needs a table")
+            return self._session.read_table(spec.table)
+        if spec.location is None:
+            raise ValueError(f"reading {self._fmt} needs a location")
+        return _rel(io.scan(self._fmt, spec.location, spec.options))
+
+
+class _Materializer:
+    def __init__(self, session: PolarsSession, strategy: str) -> None:
+        self.strategy = strategy
+        self._session = session
+
+    def materialize(
+        self, rel: Relation, target: TargetRef, cfg: WriteConfig, token: str
+    ) -> WriteResult:
+        catalog = self._session.catalog
+        table = catalog.table(target)
+        replay = table.applied(token)
+        if replay is not None:
+            return replay
+        path = table.new_file(io.EXTENSIONS[catalog.format])
+        try:
+            df: pl.DataFrame = rel.native.collect()
+            io.write(df, catalog.format, path)
+            commit = table.commit(
+                mode="append" if self.strategy == "append" else "overwrite",
+                files=[path],
+                columns=portable.schema(df),
+                rows=df.height,
+                token=token,
+                result=WriteResult(rows_inserted=df.height, rows_written=df.height),
+            )
+        except BaseException:
+            table.discard([path])
+            raise
+        return commit.result
+
+
+class PolarsSession:
+    """Thread-safe: it holds no mutable state, and a LazyFrame is immutable."""
+
+    native: Any
+
+    def __init__(self, catalog: Catalog) -> None:
+        if not isinstance(catalog, FilesystemCatalog):
+            raise TypeError(
+                f"the polars engine needs the filesystem catalog, not {type(catalog).__name__}"
+            )
+        self.catalog = catalog
+        self.native = pl
+
+    # -- reading ------------------------------------------------------------------------------
+
+    def reader(self, fmt: str) -> Reader:
+        if fmt != "table" and fmt not in CAPABILITIES.formats:
+            raise ValueError(f"polars engine cannot read format '{fmt}'")
+        return _Reader(self, fmt)
+
+    def read_table(self, target: TargetRef) -> Relation:
+        table: FsTable = self.catalog.table(target)
+        commit = table.current()
+        if commit is None:
+            raise FileNotFoundError(f"table '{target.relation_name}' does not exist")
+        if commit.kind == "view":
+            assert commit.view is not None
+            return self._run_view(commit.view)
+        assert commit.format is not None
+        return _rel(io.scan_parts(commit.format, table.files(), commit.columns))
+
+    def _run_view(self, view: ViewDefinition) -> Relation:
+        inputs = {a: self.reader(s.format).read(s) for a, s in view.inputs.items()}
+        return self.sql(view.query, inputs)
+
+    # -- writing ------------------------------------------------------------------------------
+
+    def materializer(self, strategy: str) -> Materializer:
+        if strategy not in CAPABILITIES.strategies:
+            raise ValueError(f"polars engine does not support strategy '{strategy}'")
+        return _Materializer(self, strategy)
+
+    def create_view(
+        self, target: TargetRef, query: str, inputs: Mapping[str, ReadSpec]
+    ) -> WriteResult:
+        view = ViewDefinition(query, dict(inputs))
+        rel = self._run_view(view)  # bad SQL fails here
+        table = self.catalog.table(target)
+        current = table.current()
+        if current is not None and current.kind == "view" and current.view == view:
+            return current.result  # same definition: a rerun commits nothing
+        commit = table.commit(
+            mode="view",
+            files=[],
+            columns=portable.schema(rel.native),
+            rows=None,
+            token="view",
+            result=WriteResult(),
+            view=view,
+        )
+        return commit.result
+
+    # -- relations ----------------------------------------------------------------------------
+
+    def sql(self, query: str, inputs: Mapping[str, Relation]) -> Relation:
+        # A fresh context per query: aliases never leak into another one.
+        ctx = pl.SQLContext(frames={a: r.native for a, r in inputs.items()}, eager=False)
+        frame = ctx.execute(query)
+        frame.collect_schema()  # resolve the plan now: an unknown table or column fails here
+        return _rel(frame)
+
+    def relation(self, native: Any) -> Relation:
+        if isinstance(native, pl.DataFrame):
+            native = native.lazy()
+        if not isinstance(native, pl.LazyFrame):
+            raise TypeError(f"the polars engine takes a LazyFrame, not {type(native).__name__}")
+        return _rel(native)
+
+    def persist(self, rel: Relation) -> Relation:
+        return _rel(rel.native.collect().lazy())
+
+    def schema(self, rel: Relation) -> Schema:
+        return portable.schema(rel.native)
+
+    def count(self, rel: Relation) -> int:
+        return portable.count(rel.native)
+
+    def filter_since(self, rel: Relation, column: str, lower: Any) -> Relation:
+        return _rel(portable.filter_since(rel.native, column, lower))
+
+    def max_value(self, rel: Relation, column: str) -> Any:
+        return portable.max_value(rel.native, column)
+
+    def cancel(self, unit_id: str) -> None:
+        return None
+
+
+class PolarsEngine:
+    name = NAME
+    capabilities = CAPABILITIES
+
+    def __init__(self) -> None:
+        self.version = pl.__version__
+
+    @contextmanager
+    def session(self, ctx: RuntimeContext, *, catalog: Catalog) -> Iterator[EngineSession]:
+        yield PolarsSession(catalog)

pipeloom_polars-0.2.0/src/pipeloom_polars/io.py

@@ -0,0 +1,107 @@
+"""Reading and writing files, and the logical-to-polars type mapping."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any, Final
+
+import polars as pl
+
+from pipeloom.engine.types import Schema
+
+EXTENSIONS: Final = {"parquet": "parquet", "csv": "csv", "ndjson": "ndjson"}
+"""Part file extension by catalog format."""
+
+_SIMPLE: dict[str, Any] = {
+    "int8": pl.Int8,
+    "int16": pl.Int16,
+    "int32": pl.Int32,
+    "int64": pl.Int64,
+    "int128": pl.Int128,
+    "uint8": pl.UInt8,
+    "uint16": pl.UInt16,
+    "uint32": pl.UInt32,
+    "uint64": pl.UInt64,
+    "float32": pl.Float32,
+    "float64": pl.Float64,
+    "string": pl.String,
+    "boolean": pl.Boolean,
+    "date": pl.Date,
+    "time": pl.Time,
+    "binary": pl.Binary,
+}
+if hasattr(pl, "UInt128"):  # polars 1.34 and later; the floor is 1.30
+    _SIMPLE["uint128"] = pl.UInt128
+_TIMESTAMP = re.compile(r"timestamp\[(ms|us|ns)(?:, ([^\]]+))?\]")
+_DECIMAL = re.compile(r"decimal\((\d+),(\d+)\)")
+
+
+def polars_dtype(logical: str) -> Any | None:
+    """The polars dtype of a logical type, or None for one this engine cannot name."""
+    simple = _SIMPLE.get(logical)
+    if simple is not None:
+        return simple
+    ts = _TIMESTAMP.fullmatch(logical)
+    if ts:
+        return pl.Datetime(ts.group(1), ts.group(2))  # type: ignore[arg-type]
+    dec = _DECIMAL.fullmatch(logical)
+    if dec:
+        return pl.Decimal(int(dec.group(1)), int(dec.group(2)))
+    return None
+
+
+def typed_columns(columns: Schema) -> dict[str, Any]:
+    """The polars dtype of every column this engine can name; the others are left out."""
+    out: dict[str, Any] = {}
+    for col in columns:
+        dtype = polars_dtype(col.dtype)
+        if dtype is not None:
+            out[col.name] = dtype
+    return out
+
+
+def scan(fmt: str, location: str, options: Mapping[str, Any]) -> pl.LazyFrame:
+    """A lazy frame over a source file, a directory or a glob."""
+    if fmt == "parquet":
+        return pl.scan_parquet(location, **options)
+    if fmt == "csv":
+        return pl.scan_csv(location, **options)
+    if fmt == "ndjson":
+        return pl.scan_ndjson(location, **options)
+    if fmt == "json":
+        return pl.read_json(location, **options).lazy()
+    raise ValueError(f"polars engine cannot read format '{fmt}'")
+
+
+def scan_parts(fmt: str, files: Sequence[Path], schema: Schema) -> pl.LazyFrame:
+    """A lazy frame over the committed part files of a table.
+
+    csv and ndjson lose type information, so the stored logical types are applied column by
+    column; a column this engine cannot name keeps what polars infers.
+    """
+    paths = [str(f) for f in files]
+    if fmt == "parquet":
+        return pl.scan_parquet(paths)
+    known = typed_columns(schema)
+    if fmt == "csv":
+        return pl.concat([pl.scan_csv(p, schema_overrides=known) for p in paths])
+    if fmt == "ndjson":
+        # A decimal is a JSON number there: read it as text and cast, or the schema fails.
+        decimals = {n: t for n, t in known.items() if isinstance(t, pl.Decimal)}
+        read = {n: pl.String if n in decimals else t for n, t in known.items()}
+        frames = [pl.scan_ndjson(p, schema_overrides=read) for p in paths]
+        return pl.concat(frames).with_columns(*(pl.col(n).cast(t) for n, t in decimals.items()))
+    raise ValueError(f"cannot read a table stored as '{fmt}'")
+
+
+def write(df: pl.DataFrame, fmt: str, path: Path) -> None:
+    if fmt == "parquet":
+        df.write_parquet(path)
+    elif fmt == "csv":
+        df.write_csv(path)
+    elif fmt == "ndjson":
+        df.write_ndjson(path)
+    else:
+        raise ValueError(f"cannot write format '{fmt}'")

pipeloom_polars-0.2.0/src/pipeloom_polars/spec.py

@@ -0,0 +1,43 @@
+"""The plugin spec. It imports no dataframe library: the engine module loads on first use."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pipeloom.engine.protocols import Engine
+from pipeloom.engine.types import EngineCapabilities
+from pipeloom.plugins import API_VERSION, OptionsModel, PluginSpec
+
+CAPABILITIES = EngineCapabilities(
+    formats=frozenset({"parquet", "csv", "json", "ndjson"}),
+    strategies=frozenset({"append", "overwrite"}),
+    materializations=frozenset({"table", "view", "ephemeral"}),
+    sql_models=True,
+    views=True,
+    idempotent_writes=True,
+    cancel=False,
+    bytes_metrics=True,
+    portable=True,
+)
+
+
+class PolarsOptions(OptionsModel):
+    """The polars engine has no options."""
+
+
+def _build(opts: Any) -> Engine:
+    from pipeloom_polars.engine import PolarsEngine
+
+    return PolarsEngine()
+
+
+SPEC: PluginSpec[Engine] = PluginSpec(
+    "polars",
+    "polars",
+    "engines",
+    API_VERSION,
+    _build,
+    PolarsOptions,
+    capabilities=CAPABILITIES,
+    description="Polars lazy frames; SQL through pl.SQLContext; tables on the filesystem catalog.",
+)

pipeloom_polars-0.2.0/tests/conftest.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+
+
+def write_files(root: Path, files: Mapping[str, str]) -> Path:
+    """Write text files under `root` with LF endings."""
+    for rel, content in files.items():
+        target = root / rel
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(content.encode("utf-8"))
+    return root

pipeloom_polars-0.2.0/tests/test_contract.py

@@ -0,0 +1,64 @@
+"""The polars engine passes the engine contract."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+import pytest
+
+from pipeloom.builtins.filesystem import FilesystemCatalog
+from pipeloom.engine.protocols import EngineSession
+from pipeloom.engine.types import Relation, Schema
+from pipeloom.testing.contract.engine import EngineContract, EngineHarness
+from pipeloom_polars import io
+from pipeloom_polars.engine import PolarsEngine
+
+Rows = Sequence[Mapping[str, Any]]
+
+
+def _frame(rows: Rows, schema: Schema) -> pl.DataFrame:
+    dtypes = {c.name: io.polars_dtype(c.dtype) for c in schema}
+    return pl.DataFrame([dict(r) for r in rows], schema=dtypes)
+
+
+def _boom(df: pl.DataFrame) -> pl.DataFrame:
+    raise RuntimeError("cannot produce rows")
+
+
+def harness(tmp_path: Path) -> EngineHarness:
+    def write_source(fmt: str, path: Path, rows: Rows, schema: Schema) -> None:
+        df = _frame(rows, schema)
+        if fmt == "parquet":
+            df.write_parquet(path)
+        elif fmt == "csv":
+            df.write_csv(path)
+        elif fmt == "ndjson":
+            df.write_ndjson(path)
+        else:
+            path.write_text(df.write_json(), encoding="utf-8")
+
+    def to_rows(native: Any) -> list[dict[str, Any]]:
+        df = native.collect() if isinstance(native, pl.LazyFrame) else native
+        return list(df.to_dicts())
+
+    def failing(session: EngineSession) -> Relation:
+        return session.relation(pl.LazyFrame({"a": [1]}).map_batches(_boom))
+
+    return EngineHarness(
+        engine=PolarsEngine(),
+        catalog=FilesystemCatalog(tmp_path / "wh"),
+        make_native=lambda rows, schema: _frame(rows, schema).lazy(),
+        to_rows=to_rows,
+        write_source=write_source,
+        failing_relation=failing,
+        settings_snapshot=lambda: pl.Config.state(),
+    )
+
+
+class TestPolarsEngine(EngineContract):
+    @pytest.fixture
+    def engine_factory(self) -> Callable[[Path], EngineHarness]:
+        return harness

pipeloom_polars-0.2.0/tests/test_engine.py

@@ -0,0 +1,102 @@
+"""Polars engine details beyond the contract."""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+from decimal import Decimal
+from pathlib import Path
+
+import polars as pl
+import pytest
+
+from pipeloom.builtins.filesystem import FilesystemCatalog
+from pipeloom.engine.types import ReadSpec, TargetRef, WriteConfig
+from pipeloom.platform import Identity
+from pipeloom.runtime.context import RuntimeContext
+from pipeloom.testing.clock import FakeClock
+from pipeloom.testing.memory import MemoryCatalog
+from pipeloom_polars.engine import PolarsEngine
+
+T = TargetRef(None, "main", "t")
+
+
+def ctx(tmp_path: Path) -> RuntimeContext:
+    return RuntimeContext("r", "i", "p", "t", tmp_path, FakeClock(), Identity("u", "h"), {})
+
+
+@pytest.mark.parametrize("fmt", ["parquet", "csv", "ndjson"])
+def test_tables_round_trip_in_every_catalog_format(tmp_path: Path, fmt: str) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh", format=fmt)
+    with PolarsEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        catalog.ensure_namespace(T.namespace)
+        rel = s.relation(pl.DataFrame({"id": [1, 2], "label": ["a", None]}))
+        s.materializer("overwrite").materialize(rel, T, WriteConfig("overwrite"), "r:1")
+        out = s.reader("table").read(ReadSpec("table", table=T))
+        assert out.native.collect().sort("id").to_dicts() == [
+            {"id": 1, "label": "a"},
+            {"id": 2, "label": None},
+        ]
+        assert [c.dtype for c in s.schema(out)] == ["int64", "string"]
+    assert len(list((tmp_path / "wh" / "main" / "t").glob(f"part-*.{fmt}"))) == 1
+
+
+def test_a_replayed_token_keeps_the_commit(tmp_path: Path) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh")
+    with PolarsEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        catalog.ensure_namespace(T.namespace)
+        rel = s.relation(pl.DataFrame({"id": [1]}))
+        mat = s.materializer("append")
+        first = mat.materialize(rel, T, WriteConfig("append"), "r:1")
+        before = sorted(p.name for p in (tmp_path / "wh" / "main" / "t").iterdir())
+        assert mat.materialize(rel, T, WriteConfig("append"), "r:1") == first
+        assert sorted(p.name for p in (tmp_path / "wh" / "main" / "t").iterdir()) == before
+        assert first.rows_written == 1 and first.bytes_written and first.target_version == 1
+
+
+def test_the_session_needs_the_filesystem_catalog(tmp_path: Path) -> None:
+    with (
+        pytest.raises(TypeError, match="filesystem catalog"),
+        PolarsEngine().session(ctx(tmp_path), catalog=MemoryCatalog()),
+    ):
+        pass
+
+
+def test_unsupported_format_and_strategy(tmp_path: Path) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh")
+    with PolarsEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        with pytest.raises(ValueError, match="format"):
+            s.reader("orc")
+        with pytest.raises(ValueError, match="strategy"):
+            s.materializer("merge")
+
+
+def test_a_missing_table_raises(tmp_path: Path) -> None:
+    catalog = FilesystemCatalog(tmp_path / "wh")
+    with (
+        PolarsEngine().session(ctx(tmp_path), catalog=catalog) as s,
+        pytest.raises(FileNotFoundError),
+    ):
+        s.reader("table").read(ReadSpec("table", table=T))
+
+
+@pytest.mark.parametrize("fmt", ["csv", "ndjson"])
+def test_text_formats_keep_types_per_column(tmp_path: Path, fmt: str) -> None:
+    df = pl.DataFrame(
+        {
+            "d": [date(2024, 1, 2), None],
+            "ts": pl.Series([datetime(2024, 1, 2, 3, 4, 5), None]).cast(pl.Datetime("us", "UTC")),
+            "n": pl.Series([Decimal("12.34"), None]).cast(pl.Decimal(18, 2)),
+            "s": ["007", None],
+            "i": [1, None],
+            "tags": [[1, 2], [3]],  # a type the csv format cannot hold and the engine cannot name
+        }
+    )
+    if fmt == "csv":
+        df = df.drop("tags")
+    catalog = FilesystemCatalog(tmp_path / "wh", format=fmt)
+    with PolarsEngine().session(ctx(tmp_path), catalog=catalog) as s:
+        catalog.ensure_namespace(T.namespace)
+        s.materializer("overwrite").materialize(s.relation(df), T, WriteConfig("overwrite"), "r:1")
+        back = s.reader("table").read(ReadSpec("table", table=T)).native.collect()
+    assert back.schema == df.schema
+    assert back.to_dicts() == df.to_dicts()

pipeloom_polars-0.2.0/tests/test_shop_e2e.py

@@ -0,0 +1,99 @@
+"""A shop-style project end to end on polars, with both executors.
+
+The project is `tests/fixtures/smoke`: csv sources, SQL and Python models (one on Narwhals
+frames), an ephemeral model, an append table and a view, on the filesystem catalog and json state.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+import pytest
+
+from pipeloom.api import Project, ProjectOptions
+from pipeloom.builtins.fs_table import FsTable
+from pipeloom.events.bus import EventBus
+from pipeloom.runtime.run import RunOptions, RunOutcome
+from pipeloom.testing.clock import FakeClock
+from pipeloom.testing.recording import RecordingSink
+
+SMOKE = Path(__file__).resolve().parents[3] / "tests" / "fixtures" / "smoke"
+
+
+def project(tmp_path: Path, boundary: str) -> Path:
+    shutil.copytree(
+        SMOKE, tmp_path, ignore=shutil.ignore_patterns("target", ".pipeloom"), dirs_exist_ok=True
+    )
+    yml = tmp_path / "pipeloom.yml"
+    text = yml.read_text(encoding="utf-8").replace(
+        "python_paths:", f"boundary: {boundary}\npython_paths:"
+    )
+    yml.write_text(text, encoding="utf-8", newline="\n")
+    return tmp_path
+
+
+def run(root: Path, run_id: str) -> tuple[RunOutcome, RecordingSink]:
+    clock = FakeClock(datetime(2026, 1, 1, tzinfo=UTC), step=timedelta(0))
+    sink = RecordingSink()
+    proj = Project(
+        ProjectOptions(project_dir=root, target="polars", env={"PIPELOOM_USER": "tester"}),
+        clock=clock,
+        invocation_id="inv",
+    )
+    with EventBus([(sink, 5)], clock=clock, invocation_id="inv") as bus:
+        return proj.run(RunOptions(run_id=run_id, workers=2), bus), sink
+
+
+def table(root: Path, name: str) -> list[dict[str, Any]]:
+    t = FsTable(root / "target" / "warehouse" / "main" / name, clock=FakeClock())
+    return pl.scan_parquet([str(f) for f in t.files()]).collect().sort(pl.all()).to_dicts()
+
+
+def snapshot(root: Path) -> dict[str, bytes]:
+    wh = root / "target" / "warehouse"
+    return {
+        p.relative_to(wh).as_posix(): p.read_bytes() for p in sorted(wh.rglob("*")) if p.is_file()
+    }
+
+
+@pytest.mark.parametrize(("boundary", "executor"), [("process", "thread"), ("task", "process")])
+def test_shop_runs_and_a_rerun_with_the_same_run_id_writes_nothing(
+    tmp_path: Path, boundary: str, executor: str
+) -> None:
+    root = project(tmp_path, boundary)
+    outcome, sink = run(root, "r1")
+    assert (outcome.status, outcome.exit_code) == ("succeeded", 0), outcome.diagnostics
+    doc = json.loads((root / "target" / "runs" / "r1" / "run_results.json").read_bytes())
+    assert {r["executor"] for r in doc["results"]} == {executor}
+    assert {r["engine"] for r in doc["results"]} == {"polars"}
+    assert table(root, "stg_customers") == [
+        {"id": 1, "email": "a@x.com"},
+        {"id": 2, "email": "b@x.com"},
+    ]
+    assert table(root, "fct_orders") == [
+        {"id": 10, "customer_id": 1, "email": "a@x.com", "amount": 5.0},
+        {"id": 11, "customer_id": 2, "email": "b@x.com", "amount": 7.5},
+        {"id": 12, "customer_id": 1, "email": "a@x.com", "amount": 2.5},
+    ]
+    assert table(root, "dim_customers") == [
+        {"customer_id": 1, "email": "a@x.com"},
+        {"customer_id": 2, "email": "b@x.com"},
+    ]
+    assert table(root, "rpt_revenue") == [
+        {"customer_id": 1, "revenue": 7.5},
+        {"customer_id": 2, "revenue": 7.5},
+    ]
+    view = FsTable(root / "target" / "warehouse" / "main" / "v_big_customers", clock=FakeClock())
+    commit = view.current()
+    assert commit is not None and commit.kind == "view" and commit.files == ()
+    assert not [e for e in sink.events if e.type == "unit.failed"]
+
+    before = snapshot(root)
+    again, _ = run(root, "r1")
+    assert again.exit_code == 0
+    assert snapshot(root) == before  # token replay: no new part file, commits untouched

pipeloom_polars-0.2.0/tests/test_spec_import.py

@@ -0,0 +1,23 @@
+"""The spec module is readable without importing polars."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+
+CODE = """
+import sys
+import pipeloom_polars.spec as spec
+assert spec.SPEC.capabilities.portable
+assert "polars" not in sys.modules, "polars was imported"
+assert "pipeloom_polars.engine" not in sys.modules
+print("ok")
+"""
+
+
+def test_spec_import_does_not_import_polars() -> None:
+    proc = subprocess.run(
+        [sys.executable, "-I", "-c", CODE], capture_output=True, text=True, check=False
+    )
+    assert proc.returncode == 0, proc.stderr
+    assert proc.stdout.strip() == "ok"