polymo 1.4.1__tar.gz → 1.4.2__tar.gz

{polymo-1.4.1 → polymo-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: polymo
-Version: 1.4.1
+Version: 1.4.2
 Summary: Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder
 Keywords: spark,pyspark,rest,api,ingestion,data-engineering,etl,http
 Author: Daniel Tom

{polymo-1.4.1 → polymo-1.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "polymo"
-version = "1.4.1"
+version = "1.4.2"
 description = "Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder"
 readme = "README.md"
 requires-python = ">=3.10"

{polymo-1.4.1 → polymo-1.4.2}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "polymo"
-version = "1.4.1"
+version = "1.4.2"
 description = "Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder"
 readme = "README.md"
 authors = [

{polymo-1.4.1 → polymo-1.4.2}/src/polymo/builder/app.py

@@ -20,7 +20,7 @@ from ..codegen import CodegenError, generate, generate_bundle
 from ..codegen.generator import _identifier
 from ..config import ConfigError, RestSourceConfig, config_to_dict, parse_config
 from . import databricks
-from .preview import run_preview
+from .preview import _infer_field_types, infer_ddl_from_records, run_preview
 
 if TYPE_CHECKING:  # pragma: no cover - typing only
     from pyspark.sql import SparkSession
@@ -215,11 +215,25 @@ def create_app() -> FastAPI:
             records, dtypes = await run_in_threadpool(
                 partial(_collect_records, config, payload.token, payload.limit)
             )
-        except Exception as exc:  # pragma: no cover - surfaced to UI
+        except Exception as exc:
+            # The fetch itself already succeeded (raw_pages was collected
+            # above, before this records/dtypes step ever ran) — only the
+            # local dtypes-inference step failed. Surface that as a normal
+            # 200 with `rest_error` set rather than a 502, so the already
+            # -collected `raw_pages` aren't discarded: the UI's Raw API tab
+            # renders `raw_pages`/`rest_error` the same way it does for an
+            # actual fetch failure, and the prefix below makes clear to the
+            # user that the request itself worked.
             detail = str(exc)
             if needles:
                 detail = _redact_secret(detail, needles)
-            raise HTTPException(status_code=502, detail=detail) from exc
+            return SampleResponse(
+                stream=stream_config.name,
+                records=[],
+                dtypes=[],
+                raw_pages=raw_pages,
+                rest_error=f"records/schema step failed: {detail}",
+            )
 
         if needles:
             records = _redact_secret(records, needles)
@@ -634,7 +648,50 @@ def _get_preview_df(
     """
     if schema_ddl:
         return spark.createDataFrame(records, schema=schema_ddl)
-    return spark.createDataFrame(records)
+
+    # No explicit schema: don't hand `records` to Spark's own type
+    # inference. It can't cope with a column that is `None` in every
+    # sampled record (`[CANNOT_DETERMINE_TYPE]`) — common for XML APIs,
+    # where an always-empty element decodes to `None`. Instead derive a DDL
+    # ourselves with the same voting rules the generated script's
+    # `_infer_schema` uses (see `infer_ddl_from_records`), which always
+    # produces a type for every column seen (defaulting to STRING).
+    #
+    # A DDL-typed `createDataFrame` also requires every value to match its
+    # declared column type, so a dict/list value under an inferred STRING
+    # column would crash Spark's row conversion. Mirror the generated batch
+    # reader's `_cell` helper (`codegen/templates/dp.py.jinja`) and
+    # JSON-encode nested values first — only in this inferred-schema path;
+    # `records` itself (returned to the API response) is left untouched.
+    field_types = _infer_field_types(records)
+    if not field_types:
+        # Records exist but carry no fields at all (e.g. all `{}`) — no
+        # columns to declare a DDL for; fall back to letting Spark infer.
+        return spark.createDataFrame(records)
+
+    # Unlike Spark's own type inference, `createDataFrame(data, schema=...)`
+    # verifies each value strictly against its declared column type instead
+    # of upcasting — a plain Python `int` under a merged DOUBLE column (int
+    # in one sampled record, float in another) raises
+    # `[FIELD_DATA_TYPE_UNACCEPTABLE_WITH_NAME]` unless it's coerced to
+    # `float` first.
+    double_columns = {name for name, type_ in field_types.items() if type_ == "DOUBLE"}
+    safe_records = [
+        {
+            key: (
+                json.dumps(value)
+                if isinstance(value, (dict, list))
+                else float(value)
+                if key in double_columns
+                and isinstance(value, int)
+                and not isinstance(value, bool)
+                else value
+            )
+            for key, value in record.items()
+        }
+        for record in records
+    ]
+    return spark.createDataFrame(safe_records, schema=infer_ddl_from_records(records))
 
 
 def _get_or_create_spark() -> Any:

{polymo-1.4.1 → polymo-1.4.2}/src/polymo/builder/preview.py

@@ -116,3 +116,61 @@ def run_preview(
         error = str(exc)
 
     return records, raw_pages, error
+
+
+def _infer_field_types(records: List[Dict[str, Any]]) -> Dict[str, str]:
+    """Vote a Spark type per column across `records`; the shared core of
+    `infer_ddl_from_records` (also used by `_get_preview_df` in `app.py` to
+    know which columns need numeric coercion before `createDataFrame`).
+
+    Mirrors `_infer_schema` in `codegen/templates/core.py.jinja` — keep the
+    two in sync. That template function is baked into every *generated*
+    script (so it can re-sample the live API at run time), while this one
+    runs against the preview records the builder UI already fetched.
+
+    One deliberate difference from the template: a column whose value is
+    `None` in every sampled record casts no type vote there, so the
+    template's `_infer_schema` just omits it from the DDL entirely (dropping
+    the column). That's fine for a generated script, which raises loudly if
+    sampling ever comes up completely empty. For the builder preview it
+    means the column silently vanishes from `dtypes` instead of showing up
+    as unknown/empty — worse, unqualified `spark.createDataFrame(records)`
+    (no schema at all) actually *crashes* with `[CANNOT_DETERMINE_TYPE]` in
+    this situation, which is the bug this function exists to fix (typical
+    for XML APIs, where an always-empty element decodes to `None` on every
+    record). So here, a no-vote column defaults to STRING instead of being
+    dropped.
+    """
+    columns: Dict[str, Optional[str]] = {}
+    for record in records:
+        for key, value in record.items():
+            columns.setdefault(key, None)
+            if value is None:
+                continue
+            if isinstance(value, bool):
+                candidate = "BOOLEAN"
+            elif isinstance(value, int):
+                candidate = "BIGINT"
+            elif isinstance(value, float):
+                candidate = "DOUBLE"
+            else:
+                candidate = "STRING"
+            current = columns[key]
+            if current is None:
+                columns[key] = candidate
+            elif current == candidate:
+                pass
+            elif {current, candidate} == {"BIGINT", "DOUBLE"}:
+                columns[key] = "DOUBLE"
+            else:
+                columns[key] = "STRING"
+    return {name: type_ or "STRING" for name, type_ in columns.items()}
+
+
+def infer_ddl_from_records(records: List[Dict[str, Any]]) -> str:
+    """Infer a Spark DDL schema string from already-collected preview records.
+
+    See `_infer_field_types` for the voting rules this applies.
+    """
+    fields = _infer_field_types(records)
+    return ", ".join(f"`{name}` {type_}" for name, type_ in fields.items())