df-etl-cli 0.1.0__py3-none-any.whl

df_etl_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,249 @@
+Metadata-Version: 2.4
+Name: df-etl-cli
+Version: 0.1.0
+Summary: URI-based, engine-agnostic ETL (Python port of dataframe-io, built on Ibis)
+Project-URL: Homepage, https://github.com/nightscape/df-etl-cli
+Project-URL: Repository, https://github.com/nightscape/df-etl-cli
+Project-URL: Issues, https://github.com/nightscape/df-etl-cli/issues
+Author-email: Martin Mauch <martin.mauch@gmail.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: data-engineering,dataframe,duckdb,etl,ibis,polars
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: ibis-framework[duckdb]>=9.0
+Requires-Dist: pyyaml>=6
+Provides-Extra: dev
+Requires-Dist: hypothesis>=6.100; extra == 'dev'
+Requires-Dist: ibis-framework[polars]>=9.0; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: types-pyyaml; extra == 'dev'
+Provides-Extra: polars
+Requires-Dist: ibis-framework[polars]>=9.0; extra == 'polars'
+Provides-Extra: pyspark
+Requires-Dist: ibis-framework[pyspark]>=9.0; extra == 'pyspark'
+Description-Content-Type: text/markdown
+
+# DataFrame IO (Python)
+
+A Python port of the Scala [`dataframe-io`](../README.md) library. DataFrame IO
+allows you to
+
+* read data from various **sources**
+* transform it using various **transformers**
+* write data to various **sinks**
+
+by specifying source, transform and sink URIs.
+
+Unlike the original, which is built on Apache Spark, this port is built on
+[Ibis](https://ibis-project.org/) — a portable dataframe API. Ibis is the
+dataframe abstraction: the same pipeline runs on any Ibis backend (duckdb,
+polars, pyspark, postgres, …) by changing a single `--engine` flag, with no
+change to the pipeline itself. The default backend is duckdb.
+
+The URI schema is
+
+```
+protocol://host/path?queryParam1=value1&queryParam2=value2
+```
+
+The protocol decides which source or sink is actually used. Currently, the
+following options for sources and sinks are available:
+
+* [Console](#console) (`console://`)
+* [Values](#values) (`values://`)
+* [Text](#text) (CSV, TSV) (`text://`)
+* [Parquet](#parquet) (`parquet://`)
+
+For transformations, the following options are available:
+
+* [Identity](#identity) (`identity://`)
+* [SQL](#sql) (`sql://`)
+* [SQL-file](#sql-file) (`sql-file://`)
+* [Flatten](#flatten) (`flatten://`)
+* [Flatten and explode](#flatten-and-explode) (`flatten-explode://`)
+
+> Delta, Excel, Hive, Kafka, Solr, Avro and streaming sources from the Scala
+> version are **not yet ported**.
+
+## Installation
+
+```sh
+uv venv --python 3.12
+uv pip install -e '.[dev]'          # add ,polars or ,pyspark for those engines
+```
+
+## Running
+
+### CLI
+
+The package installs a `dfio` console script that wires `--source`,
+`--transform`, and `--sink` URIs into a pipeline. Each option may be repeated.
+
+```sh
+dfio --source 'data+text:///path/to/in.csv' \
+     --transform 'data+out+sql:///SELECT%20*%20FROM%20data%20WHERE%20n%3E1' \
+     --sink 'out+parquet:///path/to/out.parquet'
+```
+
+Select a different engine — the rest of the pipeline is unchanged:
+
+```sh
+dfio --engine polars --source ... --sink ...
+```
+
+`dfio --help` lists the available schemes.
+
+### Python API
+
+```python
+from dfio import ETL, Source, Transformation, Sink
+
+ETL(
+    sources=[Source.parse("data+text:///path/to/in.csv")],
+    transforms=[Transformation.parse("data+out+sql:///SELECT * FROM data WHERE n > 1")],
+    sinks=[Sink.parse("out+parquet:///path/to/out.parquet")],
+    backend="duckdb",
+).run()
+```
+
+## Source and Sink URI schemas
+
+The URI schema for sources and sinks generally looks like this
+
+```
+dfToReadInto+sourceType://some-host/some-path?additional=parameters
+sourceType://some-host/some-path?additional=parameters
+
+dfToPersist+sinkType://some-host/some-path?additional=parameters
+sinkType://some-host/some-path?additional=parameters
+```
+
+The possible options for `sourceType` and `sinkType` are listed below.
+The `dfToReadInto` is used to save the result of reading the source. If not
+specified, it defaults to `"source"`.
+The `dfToPersist` is the name of the DataFrame that should be persisted to the
+sink. If not specified, it defaults to `"sink"`.
+
+Both `dfToReadInto` and `dfToPersist` are optional. Because they live in the URI
+*scheme*, they must be valid scheme characters: use hyphens, not underscores
+(hyphens are normalized to underscores internally so names are valid SQL
+identifiers, e.g. `my-data` becomes the table `my_data`).
+
+### Console
+```
+console://anything
+```
+The source returns an empty DataFrame.
+The sink prints an excerpt of the DataFrame to the console.
+
+### Values
+```
+values:///?header=foo:int,bar:string&values=1,a;2,b
+```
+The source returns a DataFrame with column names and types specified in `header`
+and values specified in `values` (rows separated by `;`, cells by `,`).
+Supported types are `int`, `long`, `double`; anything else is `string`.
+The sink prints an excerpt of the DataFrame to the console.
+
+### Text
+```
+text:///path/to/some.csv
+```
+Reads/writes CSV or TSV files, with the delimiter determined by file extension
+(`.csv` → `,`, `.tsv` → tab). `?header=` defaults to `true`. CSV IO is routed
+through Apache Arrow so behaviour is identical across every engine.
+
+### Parquet
+```
+parquet:///path/to/file.parquet
+```
+Reads/writes Parquet at the given path.
+
+## Transformation URI Schemas
+
+The URI schema for transformations generally looks like this
+
+```
+sourceName+sinkName+transformationType://some-host/some-path?additional=parameters
+```
+
+The `sourceName` is the previously named intermediate DataFrame used as input.
+By default it is `"source"`. The `sinkName` registers the result under a name;
+by default `"sink"`. Both can be specified or omitted:
+
+```
+transformationType://                       # both default to "source"/"sink"
+sourceName+transformationType://             # only sourceName given
+sourceName+sinkName+transformationType://    # both given
+```
+
+### Identity
+```
+sourceName+sinkName+identity:///
+```
+Renames a DataFrame from `sourceName` to `sinkName` (passthrough).
+
+### SQL
+```
+sql:///SELECT%20foo%20AS%20bar%20FROM%20sourceName
+```
+Applies inline SQL to its input. The SQL must be URL encoded (a space is `%20`)
+and follow the triple slash `sql:///`. The query runs against the engine's
+catalog, so it may reference any previously registered named DataFrame by name.
+This is only convenient for short queries.
+
+The SQL is parsed in a fixed dialect (`duckdb` by default) and transpiled by Ibis
+to whichever engine runs it, so the *same* query means the same thing on every
+backend rather than being reinterpreted per engine. Override the input dialect
+with `?dialect=` (e.g. `sql:///<encoded>?dialect=postgres`).
+
+### SQL-File
+```
+sql-file:///path/to/query.sql
+```
+Applies SQL read from a file to its input. The referenced tables must have been
+registered in a previous step.
+
+### Flatten
+```
+sourceName+sinkName+flatten:///
+```
+Recursively unpacks nested struct columns into `parent_child` columns.
+
+### Flatten and explode
+```
+sourceName+sinkName+flatten-explode:///
+```
+Recursively unpacks struct columns **and** explodes array columns into rows.
+
+## Extending with plugins
+
+External packages register new schemes via entry points, the Python analogue of
+the Scala `ServiceLoader`:
+
+```toml
+[project.entry-points."dfio.sources"]
+my-scheme = "my_package:MyUriParser"
+
+[project.entry-points."dfio.transforms"]
+my-transform = "my_package:MyTransformerParser"
+```
+
+## Tests
+
+Tests are property-based, using [Hypothesis](https://hypothesis.readthedocs.io/):
+
+```sh
+.venv/bin/pytest | tee test-output.txt
+```

df_etl_cli-0.1.0.dist-info/RECORD

@@ -0,0 +1,25 @@
+dfio/__init__.py,sha256=089oHViIDLmIzFFbw71pNGn4ddUaxz9d7VdbZxwrvnc,232
+dfio/base.py,sha256=OTojCNLzyP4IoUHOQOBdax3YUdoy8aZkzzEk8ygzYyQ,1597
+dfio/cli.py,sha256=GvrcBe_aep0UH9lf7nJmt_ePKlaGsbQEQedr8zDUdkU,2312
+dfio/engine.py,sha256=cc-07GSF4K7QAdozJJXzQb33YaXh4xiXIReH4yJUhqY,1268
+dfio/etl.py,sha256=wzJ5mN3_FegB0B4AWDwwK9-zfqEEMDML0VTMDywgmSc,3649
+dfio/graph.py,sha256=UwlTo3ye80_IP7_S4fVwwJn7KkmFE0_koPxVIbf7_MQ,6200
+dfio/node.py,sha256=XpSAUJhpzQZTkSmZCwYYtARWvWVirXVsfrgKLlwLYNk,2824
+dfio/registry.py,sha256=aUYZ0HsD3IKfTZ6CE3ViLHp0Eodu66eSPwGSPjA_6H4,2666
+dfio/runner.py,sha256=dZguFSJYxdruWp5Zmk2Q0nh14XuWNit6rRSgG9gEGqM,1712
+dfio/types.py,sha256=a9H4wopATeQLyBgNvQfKr7OqUVzPE6DIIKgYQpadHLk,1072
+dfio/uri.py,sha256=SRwGyH5aUDy1W9AgImT7DADGvBKaT7FOrqScw-PwjT8,3718
+dfio/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dfio/sources/console.py,sha256=VLkAgqKSO8jvSb7OkPHF2sbxAwHnDyet2gwf_KU9yA4,777
+dfio/sources/parquet.py,sha256=XFZIcXvBR9TQgM9pjAnwxyIcfj2aN83Y7E0qc6OPN88,869
+dfio/sources/text.py,sha256=vmUMlanLk6dwwZoDCCHFGsH9M98CawbIvba3qIeBw88,4036
+dfio/sources/values.py,sha256=7JgeceUXDF9E6UMeTltrHobTv7V1zm6YVzGc54NJbQc,1579
+dfio/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dfio/transforms/flatten.py,sha256=OhEchjrwKMOljf-MwLfBDaDt0L0mMaXDJQZOXUcPKAE,2482
+dfio/transforms/identity.py,sha256=slIiN3IigeJaTYMqt2czYQh-VGj6Au9L2X1W8vXv24I,421
+dfio/transforms/sql.py,sha256=orzjzLGuQxPL89sxPaHuplfdqmUd1xaKTdvUdgA86Dc,1986
+df_etl_cli-0.1.0.dist-info/METADATA,sha256=MLEMVmxPFXNS59JNjyaoqCqhwnK3oRuViMJ4qsvJWic,8002
+df_etl_cli-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+df_etl_cli-0.1.0.dist-info/entry_points.txt,sha256=lGnAZMCOzN-zu_IzyvS8dfNk2eEdaIvpHEkNLs0toRY,39
+df_etl_cli-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+df_etl_cli-0.1.0.dist-info/RECORD,,

df_etl_cli-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

df_etl_cli-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dfio = dfio.cli:main

df_etl_cli-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

dfio/__init__.py

@@ -0,0 +1,6 @@
+"""dfio: URI-based, engine-agnostic ETL built on Ibis (Python port of dataframe-io)."""
+
+from .engine import Engine
+from .etl import ETL, Sink, Source, Transformation
+
+__all__ = ["Engine", "ETL", "Source", "Sink", "Transformation"]

dfio/base.py

@@ -0,0 +1,58 @@
+"""Core abstractions, ported from the Scala traits.
+
+- ``DataFrameSource`` / ``DataFrameSink``  (DataFrameSource.scala, DataFrameSink.scala)
+- ``UriParser``                            (DataFrameUriParser.scala)
+- ``TransformerParser``                    (TransformerParser.scala)
+
+A ``UriParser`` builds a source/sink from a parsed URI and the engine. A
+``TransformerParser`` builds a ``Table -> Table`` function.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Callable, Protocol, runtime_checkable
+
+import ibis
+
+from .engine import Engine
+from .uri import ParsedUri
+
+Transformer = Callable[[ibis.Table], ibis.Table]
+
+
+@runtime_checkable
+class DataFrameSource(Protocol):
+    def read(self) -> ibis.Table: ...
+
+
+@runtime_checkable
+class DataFrameSink(Protocol):
+    def write(self, table: ibis.Table) -> bool: ...
+
+
+class UriParser(ABC):
+    """Builds a source and/or sink for a set of URI schemes."""
+
+    @property
+    @abstractmethod
+    def schemes(self) -> list[str]: ...
+
+    def is_defined_at(self, uri: ParsedUri) -> bool:
+        return uri.scheme_and_name()[0] in self.schemes
+
+    @abstractmethod
+    def build(self, uri: ParsedUri, engine: Engine) -> object:
+        """Return an object implementing DataFrameSource and/or DataFrameSink."""
+
+
+class TransformerParser(ABC):
+    @property
+    @abstractmethod
+    def schemes(self) -> list[str]: ...
+
+    def is_defined_at(self, uri: ParsedUri) -> bool:
+        return uri.scheme_source_sink()[0] in self.schemes
+
+    @abstractmethod
+    def build(self, uri: ParsedUri) -> Transformer: ...

dfio/cli.py

@@ -0,0 +1,70 @@
+"""``dfio`` command line. Ports the ETL.scala mainargs interface.
+
+    dfio --source name+scheme://...  --transform src+sink+scheme://...  --sink ...
+
+``--source``/``--sink``/``--transform`` may be repeated. ``--engine`` selects the
+Ibis backend (duckdb default), the lever that makes the same pipeline run on a
+different dataframe implementation.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from .engine import Engine
+from .etl import ETL, Sink, Source, Transformation
+from .graph import Graph
+from .registry import source_sink_schemes, transform_schemes
+from .runner import run_nodes
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="dfio",
+        description="URI-based, engine-agnostic ETL (built on Ibis).",
+    )
+    parser.add_argument(
+        "--source", action="append", default=[], metavar="URI",
+        help=f"Source URI (repeatable). Schemes: {sorted(source_sink_schemes())}",
+    )
+    parser.add_argument(
+        "--transform", action="append", default=[], metavar="URI",
+        help=f"Transform URI (repeatable). Schemes: {sorted(transform_schemes())}",
+    )
+    parser.add_argument(
+        "--sink", action="append", default=[], metavar="URI",
+        help="Sink URI (repeatable).",
+    )
+    parser.add_argument(
+        "--graph", metavar="PATH",
+        help="Declarative graph file (.json/.yaml). Mutually exclusive with "
+        "--source/--transform/--sink.",
+    )
+    parser.add_argument(
+        "--engine", default="duckdb", help="Ibis backend (default: duckdb).",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = build_parser().parse_args(argv)
+    if args.graph:
+        assert not (args.source or args.transform or args.sink), (
+            "--graph cannot be combined with --source/--transform/--sink"
+        )
+        engine = Engine.from_config(args.engine)
+        nodes = Graph.load(args.graph).compile()
+        run_nodes(nodes, engine)
+        print("Write successful")
+        return
+    etl = ETL(
+        sources=[Source.parse(s) for s in args.source],
+        sinks=[Sink.parse(s) for s in args.sink],
+        transforms=[Transformation.parse(t) for t in args.transform],
+        backend=args.engine,
+    )
+    etl.run()
+
+
+if __name__ == "__main__":
+    main()

dfio/engine.py

@@ -0,0 +1,36 @@
+"""The execution engine: an Ibis backend connection plus a named-table catalog.
+
+Replaces Spark's ``SparkSession``. ``register``/``table`` mirror
+``createOrReplaceTempView``/``spark.table`` — named tables are also registered as
+views in the backend so the SQL transformer can reference them by name.
+"""
+
+from __future__ import annotations
+
+import ibis
+from ibis.backends import BaseBackend
+
+
+class Engine:
+    def __init__(self, backend: BaseBackend, name: str = "duckdb"):
+        self.con = backend
+        self.backend_name = name
+        self._catalog: dict[str, ibis.Table] = {}
+
+    @classmethod
+    def from_config(cls, backend: str = "duckdb") -> "Engine":
+        con = ibis.connect(f"{backend}://")
+        return cls(con, name=backend)
+
+    def register(self, name: str, table: ibis.Table) -> ibis.Table:
+        """Register ``table`` under ``name`` and expose it as a backend view."""
+        view = self.con.create_view(name, table, overwrite=True)
+        self._catalog[name] = view
+        return view
+
+    def table(self, name: str) -> ibis.Table:
+        if name not in self._catalog:
+            raise KeyError(
+                f"No table named {name!r} in catalog; known: {sorted(self._catalog)}"
+            )
+        return self._catalog[name]