polymo 1.0.1__tar.gz → 1.2.0__tar.gz

{polymo-1.0.1 → polymo-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: polymo
-Version: 1.0.1
+Version: 1.2.0
 Summary: Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder
 Keywords: spark,pyspark,rest,api,ingestion,data-engineering,etl,http
 Author: Daniel Tom
@@ -21,18 +21,16 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Operating System :: OS Independent
 Classifier: Typing :: Typed
 Requires-Dist: jinja2>=3.1.6
-Requires-Dist: fastapi>=0.110 ; extra == 'builder'
-Requires-Dist: uvicorn>=0.24 ; extra == 'builder'
-Requires-Dist: jinja2>=3.1.6 ; extra == 'builder'
-Requires-Dist: pyspark>=4 ; extra == 'builder'
-Requires-Dist: pyarrow>=13 ; extra == 'builder'
-Requires-Dist: requests>=2.31 ; extra == 'builder'
+Requires-Dist: fastapi>=0.110
+Requires-Dist: uvicorn>=0.24
+Requires-Dist: pyspark>=4
+Requires-Dist: pyarrow>=13
+Requires-Dist: requests>=2.31
 Requires-Python: >=3.10
 Project-URL: Documentation, https://dan1elt0m.github.io/polymo/
 Project-URL: Homepage, https://github.com/dan1elt0m/polymo
 Project-URL: Issues, https://github.com/dan1elt0m/polymo/issues
 Project-URL: Repository, https://github.com/dan1elt0m/polymo.git
-Provides-Extra: builder
 Description-Content-Type: text/markdown
 
 <p align="center">
@@ -71,10 +69,16 @@ PARAMS: dict = {"_limit": 20}
 
 # … fetch/retry helpers elided …
 
+# … inline Data Source (RestSource/_Reader) elided; the batch @dp.table
+# below ingests through it, same as Lakeflow requires for the streaming
+# variant …
+
+spark.dataSource.register(RestSource)
+
+
 @dp.table(name="posts")
 def posts():
-    rows = list(fetch_records())
-    return spark.createDataFrame(rows)
+    return spark.read.format("posts_source").load()
 ```
 
 See [docs/superpowers/specs/2026-08-28-codegen-pivot-design.md](docs/superpowers/specs/2026-08-28-codegen-pivot-design.md) for the full design behind this pivot.
@@ -108,30 +112,28 @@ The YAML runtime (`spark.read.format("polymo")`, `PolymoConfig`,
 ready to move yet.
 
 ## How to start?
-Locally you probably want to install polymo along with the Builder UI: 
 
 ```bash
-pip install "polymo[builder]"
+pip install polymo
 ```
 
-This comes with all UI deps such as pyspark.
+This installs everything you need, Builder UI included — FastAPI, Uvicorn,
+PySpark, PyArrow, and requests all come along, so there's nothing extra to
+opt into.
+
+Note that `polymo` is never a dependency of the scripts it generates —
+nothing you export needs polymo installed to run.
+
+## Launch the builder UI
 
-Scripting connector generation directly in Python (calling `generate()` /
-`parse_config()` without the UI — see the [API reference](docs/api.md))
-doesn't need pyspark at all. In that case, just install the bare minimum
-deps with:
 ```bash
-pip install polymo
+polymo
 ```
 
-Note that `pip install polymo` alone does not install the Builder UI, and
-`polymo` is never a dependency of the scripts it generates — nothing you
-export needs polymo installed to run.
+Or without installing it first:
 
-## Launch the builder UI 
-
-```bash 
-polymo builder
+```bash
+uvx polymo
 ```
 
 #### (Optional) Run the Builder in Docker

{polymo-1.0.1 → polymo-1.2.0}/README.md

@@ -34,10 +34,16 @@ PARAMS: dict = {"_limit": 20}
 
 # … fetch/retry helpers elided …
 
+# … inline Data Source (RestSource/_Reader) elided; the batch @dp.table
+# below ingests through it, same as Lakeflow requires for the streaming
+# variant …
+
+spark.dataSource.register(RestSource)
+
+
 @dp.table(name="posts")
 def posts():
-    rows = list(fetch_records())
-    return spark.createDataFrame(rows)
+    return spark.read.format("posts_source").load()
 ```
 
 See [docs/superpowers/specs/2026-08-28-codegen-pivot-design.md](docs/superpowers/specs/2026-08-28-codegen-pivot-design.md) for the full design behind this pivot.
@@ -71,30 +77,28 @@ The YAML runtime (`spark.read.format("polymo")`, `PolymoConfig`,
 ready to move yet.
 
 ## How to start?
-Locally you probably want to install polymo along with the Builder UI: 
 
 ```bash
-pip install "polymo[builder]"
+pip install polymo
 ```
 
-This comes with all UI deps such as pyspark.
+This installs everything you need, Builder UI included — FastAPI, Uvicorn,
+PySpark, PyArrow, and requests all come along, so there's nothing extra to
+opt into.
+
+Note that `polymo` is never a dependency of the scripts it generates —
+nothing you export needs polymo installed to run.
+
+## Launch the builder UI
 
-Scripting connector generation directly in Python (calling `generate()` /
-`parse_config()` without the UI — see the [API reference](docs/api.md))
-doesn't need pyspark at all. In that case, just install the bare minimum
-deps with:
 ```bash
-pip install polymo
+polymo
 ```
 
-Note that `pip install polymo` alone does not install the Builder UI, and
-`polymo` is never a dependency of the scripts it generates — nothing you
-export needs polymo installed to run.
+Or without installing it first:
 
-## Launch the builder UI 
-
-```bash 
-polymo builder
+```bash
+uvx polymo
 ```
 
 #### (Optional) Run the Builder in Docker

{polymo-1.0.1 → polymo-1.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "polymo"
-version = "1.0.1"
+version = "1.2.0"
 description = "Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder"
 readme = "README.md"
 authors = [
@@ -27,13 +27,8 @@ classifiers = [
 ]
 dependencies = [
     "jinja2>=3.1.6",
-]
-
-[project.optional-dependencies]
-builder = [
     "fastapi>=0.110",
     "uvicorn>=0.24",
-    "jinja2>=3.1.6",
     "pyspark>=4",
     "pyarrow>=13",
     "requests>=2.31",

polymo-1.2.0/src/polymo/cli.py

@@ -0,0 +1,70 @@
+"""Command line entry points for polymo."""
+
+from __future__ import annotations
+
+import argparse
+from typing import Sequence
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="polymo", description="Launch the local polymo Builder UI"
+    )
+    parser.add_argument(
+        "--host", default="127.0.0.1", help="Host to bind (default: %(default)s)"
+    )
+    parser.add_argument(
+        "--port", type=int, default=8000, help="Port to bind (default: %(default)s)"
+    )
+    parser.add_argument(
+        "--reload",
+        action="store_true",
+        help="Enable auto-reload (useful during development)",
+    )
+
+    args = parser.parse_args(argv)
+
+    if not _require_builder_deps():
+        return 1
+
+    import uvicorn
+
+    uvicorn.run(
+        "polymo.builder.app:create_app",
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+        factory=True,
+    )
+    return 0
+
+
+def _require_builder_deps() -> bool:
+    """Verify the install is intact (pyspark present and compatible).
+
+    polymo now depends on pyspark unconditionally, so a missing or
+    incompatible pyspark means the install itself is broken. Returns True
+    when the environment is ready to launch the builder. On a missing
+    pyspark install, prints a friendly hint and returns False so the caller
+    can exit cleanly instead of tracebacking.
+    """
+    try:
+        import pyspark
+    except ModuleNotFoundError:
+        print(
+            "polymo's dependencies are incomplete — reinstall with: "
+            "pip install --force-reinstall polymo"
+        )
+        return False
+
+    if not pyspark.__version__.startswith("4."):
+        raise ImportError(
+            "pyspark>=4.0.0 is required: reinstall with "
+            "pip install --force-reinstall polymo"
+        )
+
+    return True
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

{polymo-1.0.1 → polymo-1.2.0}/src/polymo/codegen/templates/core.py.jinja

@@ -45,6 +45,7 @@ WINDOWS = {{ windows_repr }}
 # durable path (e.g. a Databricks Volume: /Volumes/cat/schema/vol/{{ stream_name_comment }}.json).
 STATE_PATH = {{ state_path_repr }}
 {% if not has_windows %}
+# used by the builder preview; the dp table below tracks its own cursor
 LAST_CURSOR = {"value": None}
 {% endif %}
 
@@ -448,6 +449,53 @@ def fetch_records(extra_params=None, path=None):
             LAST_CURSOR["value"] = value
 {% endif %}
 {% endif %}
+{% if not schema_ddl %}
+
+
+def _infer_schema():
+    """Sample records to derive a DDL schema (explicit SCHEMA recommended)."""
+    from itertools import islice
+
+{% if has_windows %}
+    sample = []
+    for window in WINDOWS:
+        sample.extend(islice(fetch_records(**window), max(1, 50 // len(WINDOWS))))
+{% else %}
+    sample = list(islice(fetch_records(), 50))
+{% endif %}
+    if not sample:
+        raise RuntimeError(
+            "cannot infer a schema from an empty response; set an explicit schema"
+        )
+    fields = {}
+    for record in sample:
+        for key, value in record.items():
+            if value is None:
+                continue
+            if isinstance(value, bool):
+                candidate = "BOOLEAN"
+            elif isinstance(value, int):
+                candidate = "BIGINT"
+            elif isinstance(value, float):
+                candidate = "DOUBLE"
+            else:
+                candidate = "STRING"
+            current = fields.get(key)
+            if current is None:
+                fields[key] = candidate
+            elif current == candidate:
+                pass
+            elif {current, candidate} == {"BIGINT", "DOUBLE"}:
+                fields[key] = "DOUBLE"
+            else:
+                fields[key] = "STRING"
+    if not fields:
+        raise RuntimeError(
+            "cannot infer a schema: sampled records have no fields; set an explicit"
+            " schema"
+        )
+    return ", ".join(f"`{name}` {type_}" for name, type_ in fields.items())
+{% endif %}
 {% if streaming %}
 
 

polymo-1.2.0/src/polymo/codegen/templates/dp.py.jinja

@@ -0,0 +1,131 @@
+from pyspark import pipelines as dp  # noqa: E402
+from pyspark.sql import SparkSession  # noqa: E402
+
+spark = SparkSession.getActiveSession()
+
+{% if schema_ddl %}
+SCHEMA = {{ schema_ddl | tojson }}
+
+{% endif %}
+
+{% if streaming %}
+from pyspark.sql.datasource import DataSource, SimpleDataSourceStreamReader  # noqa: E402
+
+# Assumes SCHEMA is a comma-separated "name TYPE" DDL string (as emitted
+# above); column names are taken as the first whitespace-separated token.
+COLUMNS = [f.split()[0] for f in SCHEMA.split(",")]
+
+
+class _Reader(SimpleDataSourceStreamReader):
+    def initialOffset(self):
+        return {"page": 0}
+
+    def read(self, start):
+        page = start["page"]
+        records = fetch_page(page)
+        rows = [tuple(r.get(c) for c in COLUMNS) for r in records]
+        next_page = page + 1 if records else page
+        return iter(rows), {"page": next_page}
+
+    def readBetweenOffsets(self, start, end):
+        rows = []
+        for page in range(start["page"], end["page"]):
+            for r in fetch_page(page):
+                rows.append(tuple(r.get(c) for c in COLUMNS))
+        return iter(rows)
+
+
+class RestStreamSource(DataSource):
+    @classmethod
+    def name(cls):
+        return "{{ func_name }}_stream"
+
+    def schema(self):
+        return SCHEMA
+
+    def simpleStreamReader(self, schema):
+        return _Reader()
+
+
+spark.dataSource.register(RestStreamSource)
+
+
+@dp.table(name={{ table_name_repr }})
+def {{ func_name }}():
+    return spark.readStream.format("{{ func_name }}_stream").load()
+{% else %}
+{% if has_windows %}
+from pyspark.sql.datasource import DataSource, DataSourceReader, InputPartition  # noqa: E402
+{% else %}
+from pyspark.sql.datasource import DataSource, DataSourceReader  # noqa: E402
+{% endif %}
+
+
+class RestSource(DataSource):
+    @classmethod
+    def name(cls):
+        return "{{ func_name }}_source"
+
+    def schema(self):
+        return {% if schema_ddl %}SCHEMA{% else %}_infer_schema(){% endif %}
+
+
+    def reader(self, schema):
+        return _Reader(schema)
+
+
+class _Reader(DataSourceReader):
+    def __init__(self, schema):
+        self._columns = [field.name for field in schema.fields]
+
+{% if has_windows %}
+    def partitions(self):
+        return [InputPartition(index) for index in range(len(WINDOWS))]
+
+    def read(self, partition):
+        records = fetch_records(**WINDOWS[partition.value])
+{% else %}
+    def read(self, partition):
+        records = fetch_records()
+{% endif %}
+{% if incremental_mode %}
+        cursor = None
+{% endif %}
+        for record in records:
+{% if incremental_mode %}
+            value = record.get({{ cursor_field_repr }})
+            if value is not None and (cursor is None or value > cursor):
+                cursor = value
+{% endif %}
+            yield tuple(_cell(record.get(column)) for column in self._columns)
+{% if incremental_mode %}
+        # read() runs per partition on the executor that owns it, so each
+        # partition writes its own max cursor independently; whichever
+        # partition finishes last wins. Batch tables re-fetch fully each
+        # run, so a stale cursor only costs redundant fetching, not missed
+        # data. STATE_PATH must point somewhere every executor can reach
+        # (e.g. a Databricks Volume, per the comment on it above) for this
+        # to work at all.
+        _write_state(cursor)
+{% endif %}
+
+
+def _cell(value):
+    """Nested structures become JSON strings; scalars pass through."""
+    if isinstance(value, (dict, list)):
+        import json
+
+        return json.dumps(value)
+    return value
+
+
+spark.dataSource.register(RestSource)
+
+
+{% if partition_strategy == "pagination" %}
+# partition strategy "pagination" runs sequentially in generated scripts
+{% endif %}
+@dp.table(name={{ table_name_repr }})
+def {{ func_name }}():
+    return spark.read.format("{{ func_name }}_source").load()
+{% endif %}

polymo-1.0.1/src/polymo/cli.py

@@ -1,70 +0,0 @@
-"""Command line entry points for polymo."""
-
-from __future__ import annotations
-
-import argparse
-from typing import Sequence
-
-
-def main(argv: Sequence[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(
-        prog="polymo", description="Utilities for the polymo toolkit"
-    )
-    subparsers = parser.add_subparsers(dest="command")
-    bld_parser = subparsers.add_parser("builder", help="Launch the local builder UI")
-    bld_parser.add_argument(
-        "--host", default="127.0.0.1", help="Host to bind (default: %(default)s)"
-    )
-    bld_parser.add_argument(
-        "--port", type=int, default=8000, help="Port to bind (default: %(default)s)"
-    )
-    bld_parser.add_argument(
-        "--reload",
-        action="store_true",
-        help="Enable auto-reload (useful during development)",
-    )
-
-    args = parser.parse_args(argv)
-
-    if args.command == "builder":
-        if not _require_builder_deps():
-            return 1
-
-        import uvicorn
-
-        uvicorn.run(
-            "polymo.builder.app:create_app",
-            host=args.host,
-            port=args.port,
-            reload=args.reload,
-            factory=True,
-        )
-        return 0
-
-    parser.print_help()
-    return 1
-
-
-def _require_builder_deps() -> bool:
-    """Verify the ``builder`` extra (pyspark) is installed and compatible.
-
-    Returns True when the environment is ready to launch the builder.
-    On a missing pyspark install, prints a friendly hint and returns False
-    so the caller can exit cleanly instead of tracebacking.
-    """
-    try:
-        import pyspark
-    except ModuleNotFoundError:
-        print("The builder needs extras: pip install 'polymo[builder]'")
-        return False
-
-    if not pyspark.__version__.startswith("4."):
-        raise ImportError(
-            "pyspark>=4.0.0 is required: run pip install 'polymo[builder]'"
-        )
-
-    return True
-
-
-if __name__ == "__main__":  # pragma: no cover
-    raise SystemExit(main())

polymo-1.0.1/src/polymo/codegen/templates/dp.py.jinja

@@ -1,97 +0,0 @@
-from pyspark import pipelines as dp  # noqa: E402
-from pyspark.sql import SparkSession  # noqa: E402
-
-spark = SparkSession.getActiveSession()
-
-{% if schema_ddl %}
-SCHEMA = {{ schema_ddl | tojson }}
-
-{% endif %}
-
-{% if streaming %}
-from pyspark.sql.datasource import DataSource, SimpleDataSourceStreamReader  # noqa: E402
-
-# Assumes SCHEMA is a comma-separated "name TYPE" DDL string (as emitted
-# above); column names are taken as the first whitespace-separated token.
-COLUMNS = [f.split()[0] for f in SCHEMA.split(",")]
-
-
-class _Reader(SimpleDataSourceStreamReader):
-    def initialOffset(self):
-        return {"page": 0}
-
-    def read(self, start):
-        page = start["page"]
-        records = fetch_page(page)
-        rows = [tuple(r.get(c) for c in COLUMNS) for r in records]
-        next_page = page + 1 if records else page
-        return iter(rows), {"page": next_page}
-
-    def readBetweenOffsets(self, start, end):
-        rows = []
-        for page in range(start["page"], end["page"]):
-            for r in fetch_page(page):
-                rows.append(tuple(r.get(c) for c in COLUMNS))
-        return iter(rows)
-
-
-class RestStreamSource(DataSource):
-    @classmethod
-    def name(cls):
-        return "{{ func_name }}_stream"
-
-    def schema(self):
-        return SCHEMA
-
-    def simpleStreamReader(self, schema):
-        return _Reader()
-
-
-spark.dataSource.register(RestStreamSource)
-
-
-@dp.table(name={{ table_name_repr }})
-def {{ func_name }}():
-    return spark.readStream.format("{{ func_name }}_stream").load()
-{% else %}
-{% if windows_repr %}
-@dp.table(name={{ table_name_repr }})
-def {{ func_name }}():
-    sc = spark.sparkContext
-    rows = sc.parallelize(WINDOWS, len(WINDOWS)).flatMap(
-        lambda window: list(fetch_records(**window))
-    ).collect()
-{% if incremental_mode %}
-    # fetch_records runs on executors under parallelize/flatMap, which don't
-    # share memory with the driver, so the cursor can't be tracked as
-    # in-process state; compute it here instead, from the rows collected
-    # back on the driver.
-    cursor_values = [
-        r.get({{ cursor_field_repr }})
-        for r in rows
-        if isinstance(r, dict) and r.get({{ cursor_field_repr }}) is not None
-    ]
-    _write_state(max(cursor_values) if cursor_values else None)
-{% endif %}
-{% if schema_ddl %}
-    return spark.createDataFrame(rows, schema=SCHEMA)
-{% else %}
-    return spark.createDataFrame(rows)
-{% endif %}
-{% else %}
-{% if partition_strategy == "pagination" %}
-# partition strategy "pagination" runs sequentially in generated scripts
-{% endif %}
-@dp.table(name={{ table_name_repr }})
-def {{ func_name }}():
-    rows = list(fetch_records())
-{% if incremental_mode %}
-    _write_state(LAST_CURSOR["value"])
-{% endif %}
-{% if schema_ddl %}
-    return spark.createDataFrame(rows, schema=SCHEMA)
-{% else %}
-    return spark.createDataFrame(rows)
-{% endif %}
-{% endif %}
-{% endif %}