overture-schema-pyspark 0.1.1.dev0__tar.gz

overture_schema_pyspark-0.1.1.dev0/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: overture-schema-pyspark
+Version: 0.1.1.dev0
+Summary: PySpark validation expressions for Overture Maps data
+License-Expression: MIT
+Requires-Dist: click>=8.0
+Requires-Dist: importlib-resources>=6.2 ; python_full_version < '3.13'
+Requires-Dist: overture-schema-system>=0.1.1
+Requires-Dist: packaging>=22
+Requires-Dist: pyspark>=3.4 ; extra == 'spark'
+Requires-Python: >=3.10
+Provides-Extra: spark

overture_schema_pyspark-0.1.1.dev0/pyproject.toml

@@ -0,0 +1,28 @@
+[build-system]
+requires = ["uv_build>=0.11.32,<0.13"]
+build-backend = "uv_build"
+
+[project]
+dependencies = [
+    "click>=8.0",
+    "importlib-resources>=6.2; python_version < '3.13'",
+    "overture-schema-system>=0.1.1",
+    "packaging>=22",
+]
+description = "PySpark validation expressions for Overture Maps data"
+license = "MIT"
+name = "overture-schema-pyspark"
+requires-python = ">=3.10"
+version = "0.1.1.dev0"
+
+[project.optional-dependencies]
+spark = ["pyspark>=3.4"]
+
+[project.scripts]
+overture-validate = "overture.schema.pyspark.cli:validate_cli"
+
+[tool.uv.build-backend]
+module-name = "overture.schema.pyspark"
+
+[tool.uv.sources.overture-schema-system]
+workspace = true

overture_schema_pyspark-0.1.1.dev0/pyproject.toml.orig

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["uv_build>=0.11.32,<0.13"]
+build-backend = "uv_build"
+
+[project]
+dependencies = [
+    "click>=8.0",
+    # 6.2 is the first release whose `files()` resolves a namespace portion
+    # inside a zip; the stdlib gained the same fix in 3.13.
+    "importlib-resources>=6.2; python_version < '3.13'",
+    "overture-schema-system>=0.1.1",
+    "packaging>=22",
+]
+description = "PySpark validation expressions for Overture Maps data"
+license = "MIT"
+name = "overture-schema-pyspark"
+requires-python = ">=3.10"
+version = "0.1.1.dev0"
+
+[project.optional-dependencies]
+spark = ["pyspark>=3.4"]
+
+[project.scripts]
+overture-validate = "overture.schema.pyspark.cli:validate_cli"
+
+[tool.uv.build-backend]
+module-name = "overture.schema.pyspark"
+
+[tool.uv.sources]
+overture-schema-system = { workspace = true }

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/__init__.py

@@ -0,0 +1,52 @@
+"""PySpark validation expressions for Overture Maps data."""
+
+from ._pyspark_version import pyspark_version_problem
+
+# pyspark is an optional extra (the `spark` extra): a bare install lets this
+# package's metadata and console script resolve, but every module below needs
+# pyspark itself to do anything. Probe for it here -- this module runs on any
+# import of any submodule -- so a bare install gets an actionable message. A
+# pyspark that is present but broken still raises its own error, because the
+# imports below run for real.
+try:
+    import pyspark
+except ModuleNotFoundError as exc:
+    if exc.name != "pyspark":
+        raise
+    raise ModuleNotFoundError(
+        "overture-schema-pyspark requires PySpark, which isn't installed. "
+        "Install it with `pip install overture-schema-pyspark[spark]`, or run "
+        "in an environment that already provides PySpark (e.g. a Spark cluster)."
+    ) from exc
+
+# Installing without the extra leaves no resolver to enforce the version floor
+# declared alongside it, so enforce it here, against the PySpark that actually
+# turned up. The floor is read back out of this package's own metadata.
+if _problem := pyspark_version_problem(getattr(pyspark, "__version__", None)):
+    raise ImportError(_problem)
+
+from .check import Check, CheckShape
+from .schema_check import SchemaMismatch, compare_schemas
+from .validate import (
+    ValidationResult,
+    evaluate_checks,
+    explain_errors,
+    filter_errors,
+    model_keys,
+    model_names,
+    validate_model,
+)
+
+__all__ = [
+    "Check",
+    "CheckShape",
+    "SchemaMismatch",
+    "ValidationResult",
+    "compare_schemas",
+    "evaluate_checks",
+    "explain_errors",
+    "model_keys",
+    "model_names",
+    "filter_errors",
+    "validate_model",
+]

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/_pyspark_version.py

@@ -0,0 +1,62 @@
+"""The PySpark version floor this package declares, checked at import time."""
+
+import importlib.metadata
+
+from packaging.requirements import Requirement
+from packaging.specifiers import SpecifierSet
+from packaging.utils import canonicalize_name
+
+# The distribution this module ships in. A name that stopped resolving would
+# disable the check below in silence -- no floor found reads exactly like a
+# floor satisfied -- so a test pins that this one finds real metadata.
+_DISTRIBUTION = "overture-schema-pyspark"
+
+
+def declared_pyspark_specifier() -> SpecifierSet | None:
+    """Return the PySpark version range this package declares, if readable.
+
+    The range lives in pyproject.toml (`pyspark>=N` on the `spark` extra) and
+    reaches the installed distribution as a `Requires-Dist` entry. Reading it
+    back from there rather than restating it here means one declaration, so a
+    check against it cannot drift from what the package actually requires.
+
+    Returns None when the metadata isn't installed -- a source tree run
+    without an install -- because there is then no declaration to enforce.
+    """
+    try:
+        declared = importlib.metadata.requires(_DISTRIBUTION) or ()
+    except importlib.metadata.PackageNotFoundError:
+        return None
+    for raw in declared:
+        requirement = Requirement(raw)
+        if canonicalize_name(requirement.name) == "pyspark":
+            return requirement.specifier
+    return None
+
+
+def pyspark_version_problem(version: str | None) -> str | None:
+    """Describe how `version` falls outside the declared range, or None.
+
+    Installing without the `spark` extra is the case the extra exists for --
+    a runtime that provides its own PySpark -- and it is also the case no
+    resolver sees, so nothing enforces the range at install time. This is
+    where it gets enforced instead.
+
+    `version` is `pyspark.__version__` rather than the version recorded in
+    PySpark's own metadata, because a PySpark supplied by a Spark
+    distribution is on `sys.path` without a `dist-info` directory to read.
+    A PySpark that reports no version passes: unjudgeable is not out of range.
+    """
+    specifier = declared_pyspark_specifier()
+    if version is None or specifier is None:
+        return None
+    # Prereleases count: Spark ships release candidates and dev builds, and a
+    # `>=` floor is a statement about the release they belong to.
+    if specifier.contains(version, prereleases=True):
+        return None
+    return (
+        f"overture-schema-pyspark requires PySpark {specifier}, but PySpark "
+        f"{version} is installed. Upgrade the PySpark in this environment, or "
+        f"install this package with its extra (`pip install "
+        f"overture-schema-pyspark[spark]`) to let the resolver choose one."
+    )

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/_registry.py

@@ -0,0 +1,104 @@
+"""Runtime registry of feature validations.
+
+Built at import time by walking the generated `expressions.generated`
+namespace and collecting every module that exposes the
+codegen-emitted `ENTRY_POINT` and `MODEL_VALIDATION` constants.
+
+The generated tree is the runtime source of truth: the registry
+contains exactly what was generated, regardless of which theme
+packages are installed alongside the pyspark package. A missing
+`expressions/generated/` subtree simply yields an empty registry --
+the package still imports cleanly.
+
+The tree is read through `importlib.resources`, which resolves a
+namespace portion whether it is a directory on disk or a member of an
+archive. Any wheel left unextracted on `sys.path` is zipimported and
+`pathlib` cannot traverse into it -- Spark ships wheels that way with
+`--py-files`, as does AWS Glue with `--extra-py-files`.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+import sys
+from collections.abc import Iterator
+
+if sys.version_info >= (3, 13):
+    from importlib.resources import files
+    from importlib.resources.abc import Traversable
+else:
+    # `importlib.resources.files` raises `NotADirectoryError` for a namespace
+    # package with any non-directory portion through Python 3.12; the backport
+    # carries the 3.13 fix. Runtimes known to sit below it, where this is always
+    # the branch taken: AWS Glue 4.0 (Python 3.10) and Glue 5.0 (3.11). Other
+    # runtimes that ship wheels unextracted belong on that list -- if you hit
+    # this somewhere else, please add what you find.
+    from importlib_resources import files
+    from importlib_resources.abc import Traversable
+
+from .check import ModelValidation
+
+logger = logging.getLogger(__name__)
+
+_GENERATED_ROOT = "overture.schema.pyspark.expressions.generated"
+
+
+def _iter_generated_module_names(root: str = _GENERATED_ROOT) -> list[str]:
+    """Return the dotted names of every generated module under `root`.
+
+    The generated tree is PEP 420 (no `__init__.py`), so its subdirectories
+    are namespace packages, which `pkgutil.walk_packages` skips. It is walked
+    as resources instead: every `.py` below `root`, keyed to a dotted name.
+    `files` multiplexes every portion of the namespace, so a tree assembled
+    from more than one distribution is walked whole.
+    """
+
+    def walk(node: Traversable, prefix: tuple[str, ...]) -> Iterator[str]:
+        for child in node.iterdir():
+            if child.is_dir():
+                yield from walk(child, (*prefix, child.name))
+            elif child.name.endswith(".py") and child.name != "__init__.py":
+                yield ".".join([root, *prefix, child.name[: -len(".py")]])
+
+    try:
+        anchor = files(root)
+    except ModuleNotFoundError:
+        return []
+    return sorted(walk(anchor, ()))
+
+
+def _walk() -> tuple[dict[str, ModelValidation], dict[str, dict[str, str]]]:
+    """Walk the generated tree and collect registry + partition map.
+
+    Returns a `(registry, partition_map)` pair:
+
+    * `registry` keys every feature by its `ENTRY_POINT` value.
+    * `partition_map` keys partitioned features by entry-point, mapping
+      to a Hive partition dict (e.g. `{"theme": "places", "type":
+      "place"}`) for path construction. Features with no `PARTITIONS`
+      data (empty dict) are omitted; the codegen only sets `PARTITIONS`
+      when the data lake organizes the feature by Hive partitions.
+      `type` is appended here from the module file name so consumers
+      get a complete partition path without the codegen having to
+      duplicate the type value.
+    """
+    registry: dict[str, ModelValidation] = {}
+    partition_map: dict[str, dict[str, str]] = {}
+
+    for name in _iter_generated_module_names():
+        module = importlib.import_module(name)
+        entry_point = getattr(module, "ENTRY_POINT", None)
+        validation = getattr(module, "MODEL_VALIDATION", None)
+        if entry_point is None or validation is None:
+            continue
+        registry[entry_point] = validation
+        partitions = getattr(module, "PARTITIONS", None) or {}
+        if partitions:
+            feature_type = name.rsplit(".", 1)[-1]
+            partition_map[entry_point] = {**partitions, "type": feature_type}
+
+    return registry, partition_map
+
+
+REGISTRY, PARTITION_MAP = _walk()

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/check.py

@@ -0,0 +1,59 @@
+"""Check dataclass — interface between expression builders and composition."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import Enum
+
+from overture.schema.system.geometric import GeometryType
+from pyspark.sql import Column
+from pyspark.sql.types import StructType
+
+
+class CheckShape(Enum):
+    """How the composition layer handles a check expression."""
+
+    SCALAR = "scalar"  # expression returns nullable string
+    ARRAY = "array"  # expression returns array<string>
+
+
+@dataclass(frozen=True)
+class Check:
+    """One validation check.
+
+    `field` identifies what the check is about (for error column naming
+    and report grouping), not how to access the data.  The expression in
+    `expr` already encodes the access pattern.
+
+    `expr` and `read_columns` are two views of one computation, and each is
+    a "column" in a different sense.  `read_columns` are real columns of the
+    underlying schema model -- the top-level columns the check must read to
+    evaluate.  There is always at least one; a model-level constraint that
+    spans fields names several, plus any discriminator a variant gate reads.
+    `expr` is a *virtual column*: it is not a column of the schema model but
+    one synthesized by the generated validation machinery to hold the
+    composed expression the Spark engine evaluates.  The two travel together
+    because the builder knows the read-set as it composes `expr`; recording
+    it is surer than recovering it from the finished `Column`.
+
+    `validate_model` drops a check when any column in `read_columns` is
+    skipped or structurally absent, so an unresolvable `F.col()` never
+    reaches Spark; it also treats these as the columns a check can be
+    suppressed by name.
+    """
+
+    field: str
+    name: str
+    expr: Column
+    shape: CheckShape
+    read_columns: frozenset[str]
+
+
+@dataclass(frozen=True)
+class ModelValidation:
+    """Pairs an expected schema with check builders for a feature type."""
+
+    schema: StructType
+    checks: Callable[[], list[Check]]
+    geometry_types: tuple[GeometryType, ...] = ()

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/cli.py

@@ -0,0 +1,309 @@
+"""CLI entry point for validation."""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Collection, Mapping
+from dataclasses import dataclass
+
+import click
+
+from overture.schema.system.discovery import resolve_entry_point_key
+from overture.schema.system.geometric import GeometryType
+from pyspark.errors import AnalysisException
+from pyspark.sql import DataFrame, SparkSession
+
+from ._registry import PARTITION_MAP, REGISTRY
+from .validate import (
+    explain_errors,
+    model_names,
+    validate_model,
+)
+
+
+@dataclass(frozen=True)
+class ReadSpec:
+    """Parquet read plan.
+
+    `data_path` selects the files to read; `base_path`, when set, tells
+    Spark where to start discovering Hive partition columns.
+    """
+
+    data_path: str
+    base_path: str | None = None
+
+
+def absent_column(exc: AnalysisException, columns: Collection[str]) -> str | None:
+    """Return the top-level column named by an unresolved-column error, if absent.
+
+    Returns the column name only when `exc` is an `UNRESOLVED_COLUMN` error
+    whose target is genuinely missing from `columns` -- the case a re-run with
+    `--skip-columns` resolves.  Every other `AnalysisException` (a struct field
+    accessed on a scalar, a type mismatch, an unresolved column that is in fact
+    present) returns None, marking it a generator or expression bug to surface
+    rather than steer toward `--skip-columns`.
+
+    Parameters
+    ----------
+    exc
+        The exception raised while Spark planned the check expressions.
+    columns
+        The data's top-level column names (`df.columns`).
+    """
+    # getCondition() is pyspark 4.0+; the 3.4 floor exposes the same value as
+    # getErrorClass() (renamed, then deprecated, in 4.0).
+    get_condition = getattr(exc, "getCondition", None)
+    condition = get_condition() if get_condition is not None else exc.getErrorClass()
+    if condition is None or not condition.startswith("UNRESOLVED_COLUMN"):
+        return None
+    object_name = (exc.getMessageParameters() or {}).get("objectName")
+    if not object_name:
+        return None
+    # objectName is backtick-quoted, e.g. `phantom` or `bbox`.`xmin`; the
+    # top-level segment is the column df.columns would carry.
+    top_level = object_name.split(".", 1)[0].strip("`")
+    return top_level if top_level not in columns else None
+
+
+def resolve_read(path: str, partitions: Mapping[str, str] | None) -> ReadSpec:
+    """Determine read strategy from path structure.
+
+    The partition map is an ordered Hive hierarchy
+    (`{"theme": "buildings", "type": "building"}`). A path supplies a
+    prefix of it; the leaves below the deepest level already present are
+    appended so the read always lands on a single feature type. Cases:
+
+    1. **Individual file** (`*.parquet`) or no partitions -- read
+       directly; data already contains the partition columns inline.
+    2. **Release root** (no partition directories) -- append the full
+       partition path and set `basePath` to the original path.
+    3. **Partial partition path** (`theme=X/`) -- append the missing
+       leaves (`type=Y`) so a single feature's checks aren't run against
+       every type sharing the theme directory.
+    4. **Leaf partition path** (`theme=X/type=Y/`) -- nothing to append;
+       read it directly with `basePath` derived.
+    """
+    stripped = path.rstrip("/")
+
+    # Individual file or no partition mapping — data has partition columns inline
+    if stripped.endswith(".parquet") or not partitions:
+        return ReadSpec(data_path=path)
+
+    keys = list(partitions)
+    # Partition levels already present in the path, in hierarchy order.
+    present = [i for i, key in enumerate(keys) if f"/{key}=" in stripped]
+    depth = present[-1] + 1 if present else 0  # count of levels already filled
+    leaves = "/".join(f"{key}={partitions[key]}" for key in keys[depth:])
+
+    if not present:
+        # Release root — append the full partition path; it is the base.
+        return ReadSpec(data_path=f"{stripped}/{leaves}", base_path=stripped)
+
+    # Path already contains partition directories: the base is the release
+    # root (before the first one); append any leaves below the deepest
+    # present level (none for a leaf path, which then reads as-is).
+    base_idx = stripped.find(f"/{keys[present[0]]}=")
+    data_path = f"{stripped}/{leaves}" if leaves else path
+    return ReadSpec(data_path=data_path, base_path=stripped[:base_idx])
+
+
+def read_feature(spark: SparkSession, spec: ReadSpec) -> DataFrame:
+    """Read a DataFrame according to a ReadSpec."""
+    reader = spark.read
+    if spec.base_path:
+        reader = reader.option("basePath", spec.base_path)
+    return reader.parquet(spec.data_path)
+
+
+_S3A_DEFAULTS: dict[str, str] = {
+    "spark.jars.packages": "org.apache.hadoop:hadoop-aws:3.4.1",
+    "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem",
+    "spark.hadoop.fs.s3a.aws.credentials.provider": (
+        "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
+    ),
+}
+
+_LARGE_GEOMETRY_TYPES = frozenset(
+    {
+        GeometryType.LINE_STRING,
+        GeometryType.MULTI_LINE_STRING,
+        GeometryType.POLYGON,
+        GeometryType.MULTI_POLYGON,
+        GeometryType.GEOMETRY_COLLECTION,
+    }
+)
+
+
+def _may_have_large_geometry(feature_key: str) -> bool:
+    """Whether a registered feature's geometries may be large.
+
+    Returns True when the registered geometry types include
+    (multi)linestrings, (multi)polygons, or geometry collections,
+    or when geometry types are unspecified (safe default).
+    """
+    validation = REGISTRY[feature_key]
+    if not validation.geometry_types:
+        return True
+    return bool(set(validation.geometry_types) & _LARGE_GEOMETRY_TYPES)
+
+
+def _spark_config(path: str, conf: tuple[str, ...], feature_key: str) -> dict[str, str]:
+    """Build Spark config dict with safe defaults.
+
+    Disables the vectorized Parquet reader for features with large
+    geometries (polygons, linestrings) to avoid OOM on WKB binary
+    columns.  Adds S3A credentials for `s3a://` paths.  User-supplied
+    `--conf` values override any defaults.
+    """
+    config: dict[str, str] = {}
+    if _may_have_large_geometry(feature_key):
+        config["spark.sql.parquet.enableVectorizedReader"] = "false"
+    if path.startswith("s3a://"):
+        config.update(_S3A_DEFAULTS)
+    for pair in conf:
+        key, _, value = pair.partition("=")
+        config[key] = value
+    return config
+
+
+@click.command("overture-validate")
+@click.argument("feature_type")
+@click.argument("path")
+@click.option("-o", "--output", default=None, help="Output path for validated Parquet.")
+@click.option(
+    "--head",
+    "head_n",
+    default=20,
+    type=int,
+    show_default=True,
+    help="Error rows to display.",
+)
+@click.option("--conf", multiple=True, help="Spark config key=value pairs.")
+@click.option(
+    "--count-only",
+    is_flag=True,
+    default=False,
+    help="Report error count only; skip explain/unpivot.",
+)
+@click.option(
+    "--skip-schema-check",
+    is_flag=True,
+    default=False,
+    help="Warn on schema mismatches instead of aborting.",
+)
+@click.option(
+    "--skip-columns",
+    multiple=True,
+    help="Columns declared absent from data; skips their checks.",
+)
+@click.option(
+    "--ignore-extra-columns",
+    multiple=True,
+    help="Extra data columns to ignore in schema comparison.",
+)
+@click.option(
+    "--suppress",
+    "suppress_specs",
+    multiple=True,
+    help="Suppress checks: FIELD (all checks) or FIELD:CHECK (specific).",
+)
+def validate_cli(
+    feature_type: str,
+    path: str,
+    output: str | None,
+    head_n: int,
+    conf: tuple[str, ...],
+    count_only: bool,
+    skip_schema_check: bool,
+    skip_columns: tuple[str, ...],
+    ignore_extra_columns: tuple[str, ...],
+    suppress_specs: tuple[str, ...],
+) -> None:
+    """Validate Overture data at PATH and write annotated Parquet."""
+    try:
+        resolved = resolve_entry_point_key(feature_type, REGISTRY)
+    except ValueError:
+        click.echo(
+            f"Unknown type '{feature_type}'. Known: {', '.join(model_names())}",
+            err=True,
+        )
+        sys.exit(1)
+
+    builder = SparkSession.builder
+    for key, value in _spark_config(path, conf, resolved).items():
+        builder = builder.config(key, value)
+    spark = builder.getOrCreate()
+    spark.sparkContext.setLogLevel("ERROR")
+
+    spec = resolve_read(path, PARTITION_MAP.get(resolved))
+    df = read_feature(spark, spec)
+
+    suppress: list[str | tuple[str, str]] = []
+    for s in suppress_specs:
+        if ":" in s:
+            field, name = s.split(":", 1)
+            suppress.append((field, name))
+        else:
+            suppress.append(s)
+
+    try:
+        result = validate_model(
+            df,
+            resolved,
+            skip_columns=skip_columns,
+            ignore_extra_columns=ignore_extra_columns,
+            suppress=suppress,
+        )
+    except ValueError as e:
+        click.echo(str(e), err=True)
+        sys.exit(1)
+    except AnalysisException as e:
+        # Backstop, narrowed to the one cause `--skip-columns` can address: a
+        # check that names a column missing from the data.  validate_model
+        # already drops checks for skipped and schema-absent columns, so this
+        # fires only on a column outside the expected schema -- offer the
+        # operator the skip lever and name the column.  Every other
+        # AnalysisException (a type mismatch, a struct field read off a scalar)
+        # is a generator bug `--skip-columns` cannot fix; let it propagate as a
+        # traceback rather than mask it behind the skip hint.
+        column = absent_column(e, df.columns)
+        if column is None:
+            raise
+        click.echo(
+            f"A check references column '{column}', absent from the data at {path}.",
+            err=True,
+        )
+        click.echo(
+            f"Re-run with `--skip-columns {column}` to skip its checks, "
+            "or `--skip-schema-check`.",
+            err=True,
+        )
+        sys.exit(1)
+
+    if result.schema_mismatches:
+        click.echo(f"Schema mismatches for {resolved}:", err=True)
+        for m in result.schema_mismatches:
+            click.echo(f"  {m.path}: expected {m.expected}, got {m.actual}", err=True)
+        if result.absent_columns:
+            flags = " ".join(f"--skip-columns {c}" for c in result.absent_columns)
+            click.echo(
+                f"  Re-run with `{flags}` to skip missing columns.",
+                err=True,
+            )
+        if not skip_schema_check:
+            sys.exit(1)
+
+    total_rows, error_count = result.row_counts()
+    click.echo(f"{error_count} / {total_rows} rows with errors", err=True)
+
+    if error_count > 0:
+        if not count_only:
+            explained = explain_errors(result.evaluated, result.checks).drop("geometry")
+            if output and head_n > 0:
+                explained = explained.cache()
+            if output:
+                explained.write.mode("overwrite").parquet(output)
+                click.echo(f"Written to {output}", err=True)
+            if head_n > 0:
+                explained.show(head_n, truncate=False)
+        sys.exit(1)

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/expressions/__init__.py

@@ -0,0 +1 @@
+"""Expression builders and reusable PySpark column patterns."""

overture_schema_pyspark-0.1.1.dev0/src/overture/schema/pyspark/expressions/_schema_structs.py

@@ -0,0 +1,22 @@
+"""Hand-written Spark StructType fragments for types the codegen can't generate.
+
+The codegen builds feature schemas by walking Pydantic `BaseModel`
+subclasses. `BBox` is a plain class, not a `BaseModel`, so extraction
+can't reach it -- `BBOX_STRUCT` is hand-written here to fill the gap.
+Every other nested type is a `BaseModel` and gets generated directly
+into each feature module, which is why this file holds only the one
+struct.
+"""
+
+from __future__ import annotations
+
+from pyspark.sql.types import DoubleType, StructField, StructType
+
+BBOX_STRUCT = StructType(
+    [
+        StructField("xmin", DoubleType(), True),
+        StructField("xmax", DoubleType(), True),
+        StructField("ymin", DoubleType(), True),
+        StructField("ymax", DoubleType(), True),
+    ]
+)