vecorel-cli 0.2.17__tar.gz → 0.2.18__tar.gz

{vecorel_cli-0.2.17/vecorel_cli.egg-info → vecorel_cli-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.2.17
+Version: 0.2.18
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -32,6 +32,7 @@ Requires-Dist: rarfile<5.0,>=4.0
 Requires-Dist: semantic-version<3.0,>=2.10.0
 Requires-Dist: json-stream<3.0,>=2.3.0
 Requires-Dist: loguru==0.7.3
+Requires-Dist: duckdb<2.0,>=1.4
 Provides-Extra: s3
 Requires-Dist: s3fs>=2025.7.0; extra == "s3"
 Provides-Extra: gcs

{vecorel_cli-0.2.17 → vecorel_cli-0.2.18}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vecorel-cli"
-version = "0.2.17"
+version = "0.2.18"
 description = "CLI tools such as validation and file format conversion for vecorel."
 readme = "README.md"
 license = "Apache-2.0"
@@ -38,6 +38,7 @@ dependencies = [
     "semantic-version>=2.10.0,<3.0",
     "json-stream>=2.3.0,<3.0",
     "loguru==0.7.3",
+    "duckdb>=1.4,<2.0",
 ]
 
 [project.optional-dependencies]

vecorel_cli-0.2.18/tests/test_convert_duckdb.py

@@ -0,0 +1,220 @@
+import json
+
+import geopandas as gpd
+import numpy as np
+import pyarrow.parquet as pq
+import pytest
+import shapely
+
+from vecorel_cli.conversion.base import BaseConverter
+from vecorel_cli.conversion.duckdb import DuckDBBaseConverter
+from vecorel_cli.validate import ValidateData
+from vecorel_cli.vecorel.hilbert import hilbert_keys_for_table
+
+# One shared converter configuration, so the two codepaths cannot drift apart.
+# column_filters and column_migrations must stay out of it: they are Python
+# callables in the GeoDataFrame-based codepath and SQL fragments in the
+# DuckDB-based one.
+CONFIG = {
+    "id": "test",
+    "short_name": "Test",
+    "title": "Test dataset",
+    "description": "Test dataset",
+    "license": "CC0-1.0",
+    "columns": {
+        "geometry": "geometry",
+        "id": "id",
+        "name": "name",
+    },
+    "missing_schemas": {
+        "properties": {
+            "name": {"type": "string"},
+        }
+    },
+}
+
+Converter = type("Converter", (DuckDBBaseConverter,), dict(CONFIG))
+PandasConverter = type("PandasConverter", (BaseConverter,), dict(CONFIG))
+
+
+def _source_file(folder):
+    """A source with everything the geometry handling must fix:
+    a multi-part geometry, an invalid bowtie, a Z polygon, and a point."""
+    square = shapely.Polygon([(0, 0), (0, 1), (1, 1), (1, 0)])
+    multi = shapely.MultiPolygon(
+        [
+            shapely.Polygon([(2, 0), (2, 1), (3, 1), (3, 0)]),
+            shapely.Polygon([(4, 0), (4, 1), (5, 1), (5, 0)]),
+        ]
+    )
+    bowtie = shapely.Polygon([(6, 0), (7, 1), (7, 0), (6, 1)])
+    with_z = shapely.Polygon([(8, 0, 5), (8, 1, 5), (9, 1, 5), (9, 0, 5)])
+    point = shapely.Point(10, 0)
+
+    gdf = gpd.GeoDataFrame(
+        {
+            "id": ["square", "multi", "bowtie", "with_z", "point"],
+            "name": ["a", "b", "c", "d", "e"],
+            "geometry": [square, multi, bowtie, with_z, point],
+        },
+        crs="EPSG:4326",
+    )
+    path = folder / "source.parquet"
+    gdf.to_parquet(path)
+    return str(path)
+
+
+def test_duckdb_converter(tmp_folder):
+    src = _source_file(tmp_folder)
+    dest = tmp_folder / "converted.parquet"
+
+    Converter().convert(dest, input_files={src: "source.parquet"})
+
+    result = gpd.read_parquet(dest)
+    # multi is split in two, bowtie is repaired into two valid polygons,
+    # the point is dropped and the Z dimension is removed
+    assert sorted(result["id"]) == ["bowtie", "bowtie", "multi", "multi", "square", "with_z"]
+    assert set(result.geometry.geom_type) == {"Polygon"}
+    assert result.geometry.is_valid.all()
+    assert not result.geometry.has_z.any()
+
+    with pq.ParquetFile(dest) as pf:
+        schema = pf.schema_arrow
+    field = schema.field("geometry")
+    assert str(field.type) == "binary"
+    assert not field.nullable
+    field = schema.field("id")
+    assert str(field.type) == "string"
+    assert not field.nullable
+    assert str(schema.field("name").type) == "string"
+    assert "bbox" in schema.names
+
+    # sorted against the CRS-derived Hilbert grid
+    with pq.ParquetFile(dest) as pf:
+        table = pf.read()
+    keys = hilbert_keys_for_table(table, "geometry", (-180.0, -90.0, 180.0, 90.0))
+    assert bool(np.all(keys[1:] >= keys[:-1]))
+
+    validation = ValidateData().validate(dest, num=100, schema_map={})
+    assert validation.errors == []
+
+
+def test_duckdb_converter_index_as_id(tmp_folder):
+    src = _source_file(tmp_folder)
+    dest = tmp_folder / "converted.parquet"
+
+    IndexConverter = type("IndexConverter", (DuckDBBaseConverter,), {**CONFIG, "index_as_id": True})
+    IndexConverter().convert(dest, input_files={src: "source.parquet"})
+
+    result = gpd.read_parquet(dest)
+    # row numbers are assigned before geometries are split,
+    # so the parts of one source feature share an id (like the default codepath)
+    assert sorted(result["id"]) == ["0", "1", "1", "2", "2", "3"]
+
+
+def test_duckdb_converter_source_crs(tmp_folder):
+    src1 = _source_file(tmp_folder)
+    dest = tmp_folder / "converted.parquet"
+
+    # the same CRS, declared as a differently rendered PROJJSON object
+    table = pq.read_table(src1)
+    metadata = dict(table.schema.metadata)
+    geo = json.loads(metadata[b"geo"])
+    crs = geo["columns"]["geometry"]["crs"]
+    crs.pop("scope", None)
+    crs.pop("area", None)
+    crs["$schema"] = "https://proj.org/schemas/v0.5/projjson.schema.json"
+    metadata[b"geo"] = json.dumps(geo).encode()
+    src2 = str(tmp_folder / "source2.parquet")
+    pq.write_table(table.replace_schema_metadata(metadata), src2)
+
+    Converter().convert(dest, input_files={src1: "a.parquet", src2: "b.parquet"})
+    assert len(gpd.read_parquet(dest)) == 12
+
+    src3 = str(tmp_folder / "source3.parquet")
+    gpd.read_parquet(src1).to_crs("EPSG:3857").to_parquet(src3)
+    with pytest.raises(ValueError, match="different coordinate reference"):
+        Converter().convert(dest, input_files={src1: "a.parquet", src3: "c.parquet"})
+
+
+def test_duckdb_converter_original_geometries(tmp_folder):
+    src = _source_file(tmp_folder)
+    dest = tmp_folder / "converted.parquet"
+
+    Converter().convert(dest, input_files={src: "source.parquet"}, original_geometries=True)
+
+    result = gpd.read_parquet(dest)
+    assert len(result) == 5
+    assert set(result.geometry.geom_type) == {"Polygon", "MultiPolygon", "Point"}
+
+
+def test_codepath_parity(tmp_folder):
+    """The GeoDataFrame-based and the DuckDB-based codepaths must produce
+    comparable files from the same source and converter configuration:
+    same schema, same rows in the same order, same key metadata, same packaging.
+    """
+    src = _source_file(tmp_folder)
+    kwargs = {
+        "input_files": {src: "source.parquet"},
+        "compression": "zstd",
+        "geoparquet_version": "1.1.0",
+    }
+    pandas_dest = tmp_folder / "pandas.parquet"
+    duckdb_dest = tmp_folder / "duckdb.parquet"
+    PandasConverter().convert(pandas_dest, **kwargs)
+    Converter().convert(duckdb_dest, **kwargs)
+
+    with pq.ParquetFile(pandas_dest) as pf:
+        pandas_schema = pf.schema_arrow
+        pandas_table = pf.read()
+        pandas_groups = pf.metadata.num_row_groups
+        pandas_compression = pf.metadata.row_group(0).column(0).compression
+    with pq.ParquetFile(duckdb_dest) as pf:
+        duckdb_schema = pf.schema_arrow
+        duckdb_table = pf.read()
+        duckdb_groups = pf.metadata.num_row_groups
+        duckdb_compression = pf.metadata.row_group(0).column(0).compression
+
+    # Same columns with the same types and nullability (the column order is
+    # allowed to differ)
+    assert sorted(pandas_schema.names) == sorted(duckdb_schema.names)
+    for name in pandas_schema.names:
+        f1, f2 = pandas_schema.field(name), duckdb_schema.field(name)
+        assert f1.type == f2.type, f"{name}: {f1.type} != {f2.type}"
+        assert f1.nullable == f2.nullable, f"{name}: nullability differs"
+
+    # Same rows in the same (Hilbert) order; the geometries must describe the
+    # same shapes, but the WKB may differ in vertex order (different GEOS builds)
+    assert pandas_table.num_rows == duckdb_table.num_rows
+    assert pandas_table["id"].to_pylist() == duckdb_table["id"].to_pylist()
+    assert pandas_table["name"].to_pylist() == duckdb_table["name"].to_pylist()
+    pandas_geoms = shapely.from_wkb(pandas_table["geometry"].to_pylist())
+    duckdb_geoms = shapely.from_wkb(duckdb_table["geometry"].to_pylist())
+    for g1, g2 in zip(pandas_geoms, duckdb_geoms):
+        assert shapely.equals(g1, g2), f"{shapely.to_wkt(g1)} != {shapely.to_wkt(g2)}"
+
+    # Same collection metadata and the same key GeoParquet metadata
+    pandas_collection = json.loads(pandas_schema.metadata[b"collection"])
+    duckdb_collection = json.loads(duckdb_schema.metadata[b"collection"])
+    assert pandas_collection == duckdb_collection
+
+    pandas_geo = json.loads(pandas_schema.metadata[b"geo"])
+    duckdb_geo = json.loads(duckdb_schema.metadata[b"geo"])
+    for key in ("version", "primary_column"):
+        assert pandas_geo[key] == duckdb_geo[key]
+    pandas_column = pandas_geo["columns"]["geometry"]
+    duckdb_column = duckdb_geo["columns"]["geometry"]
+    for key in ("encoding", "covering", "crs", "bbox"):
+        assert pandas_column.get(key) == duckdb_column.get(key), f"geo {key} differs"
+    assert sorted(pandas_column.get("geometry_types", [])) == sorted(
+        duckdb_column.get("geometry_types", [])
+    )
+
+    # Same packaging
+    assert pandas_compression == duckdb_compression
+    assert pandas_groups == duckdb_groups
+
+    # Both validate
+    for dest in (pandas_dest, duckdb_dest):
+        validation = ValidateData().validate(dest, num=100, schema_map={})
+        assert validation.errors == []

vecorel_cli-0.2.18/tests/test_encoding_geoparquet.py

@@ -0,0 +1,125 @@
+import json
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pyarrow.types as pat
+
+from vecorel_cli.encoding.geoparquet import GeoParquet
+from vecorel_cli.vecorel.collection import Collection
+
+
+def test_init_paths(tmp_folder):
+    fpath = str(tmp_folder / "test.parquet")
+    ppath = Path(fpath)
+
+    assert GeoParquet(fpath).uri == ppath
+    assert GeoParquet(ppath).uri == ppath
+
+
+def test_get_format():
+    assert (
+        GeoParquet("tests/data-files/inspire.parquet").get_format() == "GeoParquet, version 1.1.0"
+    )
+
+
+def test_get_collection_exists():
+    geojson = GeoParquet("tests/data-files/inspire.parquet")
+
+    collection = geojson.get_collection()
+
+    assert isinstance(collection, dict)
+    fields = list(collection.keys())
+    fields.sort()
+    assert fields == ["collection", "determination_datetime", "schemas", "schemas:custom"]
+    assert "inspire" in collection["schemas"]
+
+
+def test_get_collection_does_not_exist():
+    collection = GeoParquet("invalid.parquet").get_collection()
+    assert isinstance(collection, Collection)
+    assert collection.is_empty()
+
+
+def test_get_collection_returns_existing(tmp_folder):
+    file_path = tmp_folder / "test.parquet"
+    test_collection = Collection({"test": "data"})
+
+    geojson = GeoParquet(file_path)
+    geojson.collection = test_collection
+
+    result = geojson.get_collection()
+    assert isinstance(result, Collection)
+    assert result == test_collection
+
+
+def test_postprocess(tmp_parquet_file):
+    # Degrade a compliant file to what external tools such as DuckDB or GDAL may produce:
+    # large_string/large_binary columns, a naive microsecond timestamp, nullable columns,
+    # no bbox column, GeoParquet 1.0.0
+    src = pq.read_table("tests/data-files/inspire.parquet")
+    fields = []
+    arrays = []
+    for i, field in enumerate(src.schema):
+        if field.name == "bbox":
+            continue
+        if pat.is_string(field.type):
+            dtype = pa.large_string()
+        elif pat.is_binary(field.type):
+            dtype = pa.large_binary()
+        else:
+            dtype = field.type
+        fields.append(pa.field(field.name, dtype, nullable=True))
+        arrays.append(src.column(i).cast(dtype))
+    fields.append(pa.field("determination_datetime", pa.timestamp("us")))
+    arrays.append(pa.array([1672531200000000] * len(src), type=pa.timestamp("us")))
+
+    metadata = dict(src.schema.metadata)
+    geo = json.loads(metadata[b"geo"])
+    geo["version"] = "1.0.0"
+    geo["columns"]["geometry"].pop("covering", None)
+    metadata[b"geo"] = json.dumps(geo).encode("utf-8")
+    degraded = pa.table(arrays, schema=pa.schema(fields, metadata=metadata))
+    pq.write_table(degraded, tmp_parquet_file, store_schema=True)
+
+    gp = GeoParquet(tmp_parquet_file)
+    assert gp.postprocess(geoparquet_version="1.1.0", compression="zstd") is True
+    # a second run detects the compliant file and doesn't rewrite it
+    assert gp.postprocess(geoparquet_version="1.1.0", compression="zstd") is False
+    # unless a different compression is requested
+    assert gp.postprocess(geoparquet_version="1.1.0", compression="brotli") is True
+
+    with pq.ParquetFile(tmp_parquet_file) as result:
+        schema = result.schema_arrow
+        metadata = result.metadata.metadata
+
+    field = schema.field("geometry")
+    assert field.type == pa.binary()
+    assert not field.nullable
+    field = schema.field("id")
+    assert field.type == pa.string()
+    assert not field.nullable
+    assert schema.field("inspire:id").type == pa.string()
+    assert schema.field("determination_datetime").type == pa.timestamp("ms", tz="UTC")
+
+    bbox = schema.field("bbox")
+    assert pat.is_struct(bbox.type)
+    assert bbox.type.field("xmin").type == pa.float64()
+
+    geo = json.loads(metadata[b"geo"])
+    assert geo["version"] == "1.1.0"
+    assert geo["columns"]["geometry"]["covering"]["bbox"]["xmin"] == ["bbox", "xmin"]
+    assert b"collection" in metadata
+
+    # data is intact
+    data = pq.read_table(tmp_parquet_file)
+    assert data.num_rows == src.num_rows
+    assert data["id"].to_pylist() == src["id"].to_pylist()
+    assert data["geometry"].to_pylist() == src["geometry"].to_pylist()
+
+    # a downgrade removes the covering metadata, which only exists since GeoParquet 1.1
+    assert gp.postprocess(geoparquet_version="1.0.0", compression="brotli") is True
+    with pq.ParquetFile(tmp_parquet_file) as pf:
+        geo = json.loads(pf.metadata.metadata[b"geo"])
+    assert geo["version"] == "1.0.0"
+    assert "covering" not in geo["columns"]["geometry"]

vecorel_cli-0.2.18/tests/test_hilbert.py

@@ -0,0 +1,89 @@
+import json
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+import shapely
+
+from vecorel_cli.vecorel.hilbert import (
+    bounds_array_for_table,
+    crs_total_bounds,
+    ensure_hilbert_sorted,
+    hilbert_keys_for_table,
+    hilbert_reference_bounds,
+)
+
+# A bare projection definition without an area of use,
+# like the unnamed "MGI / Austria Lambert" in Austria's 2018 edition
+CUSTOM_CRS = "+proj=laea +lat_0=52 +lon_0=10 +x_0=4321000 +y_0=3210000 +units=m"
+
+
+def test_hilbert_reference_bounds_from_crs():
+    assert hilbert_reference_bounds("EPSG:4326") == (-180.0, -90.0, 180.0, 90.0)
+
+
+def test_hilbert_reference_bounds_fallback():
+    fallback = [1.0, 2.0, 3.0, 4.0]
+    assert hilbert_reference_bounds(CUSTOM_CRS, fallback) == (1.0, 2.0, 3.0, 4.0)
+    assert hilbert_reference_bounds(CUSTOM_CRS) is None
+
+
+def _make_table(points, with_bbox=True, metadata=None):
+    geometries = [shapely.Point(x, y).buffer(0.1) for x, y in points]
+    wkb = [shapely.to_wkb(g) for g in geometries]
+    bounds = shapely.bounds(np.array(geometries, dtype=object))
+    arrays = {
+        "id": pa.array([str(i) for i in range(len(points))], type=pa.string()),
+        "geometry": pa.array(wkb, type=pa.binary()),
+    }
+    if with_bbox:
+        arrays["bbox"] = pa.StructArray.from_arrays(
+            [bounds[:, 0], bounds[:, 1], bounds[:, 2], bounds[:, 3]],
+            names=["xmin", "ymin", "xmax", "ymax"],
+        )
+    table = pa.table(arrays)
+    if metadata:
+        table = table.replace_schema_metadata(metadata)
+    return table
+
+
+def test_bounds_array_bbox_and_wkb_paths_agree():
+    points = [(0, 0), (10, 10), (5, 5)]
+    with_bbox = _make_table(points, with_bbox=True)
+    without_bbox = _make_table(points, with_bbox=False)
+
+    np.testing.assert_allclose(
+        bounds_array_for_table(with_bbox, "geometry"),
+        bounds_array_for_table(without_bbox, "geometry"),
+    )
+
+
+def test_ensure_hilbert_sorted(tmp_folder):
+    path = str(tmp_folder / "unsorted.parquet")
+    # scattered, deliberately not in Hilbert order
+    points = [(30, 40), (-120, -30), (5, 5), (100, 60), (-60, 10)]
+    metadata = {
+        b"geo": json.dumps({"primary_column": "geometry", "columns": {"geometry": {}}}).encode(),
+        b"collection": b'{"schemas": {}}',
+    }
+    table = _make_table(points, metadata=metadata)
+    pq.write_table(table, path)
+
+    total_bounds = crs_total_bounds("EPSG:4326")
+    keys = hilbert_keys_for_table(table, "geometry", total_bounds)
+    assert not bool(np.all(keys[1:] >= keys[:-1])), "Test data must start unsorted"
+
+    assert ensure_hilbert_sorted(path, "geometry", total_bounds, "zstd") is True
+
+    result = pq.read_table(path)
+    sorted_keys = hilbert_keys_for_table(result, "geometry", total_bounds)
+    assert bool(np.all(sorted_keys[1:] >= sorted_keys[:-1]))
+    # data is intact, schema stays narrow, metadata is preserved
+    assert sorted(result["id"].to_pylist()) == sorted(table["id"].to_pylist())
+    assert result.schema.field("geometry").type == pa.binary()
+    assert result.schema.field("id").type == pa.string()
+    assert result.schema.metadata[b"geo"] == metadata[b"geo"]
+    assert result.schema.metadata[b"collection"] == metadata[b"collection"]
+
+    # a second run is a no-op
+    assert ensure_hilbert_sorted(path, "geometry", total_bounds, "zstd") is False

{vecorel_cli-0.2.17 → vecorel_cli-0.2.18}/vecorel_cli/const.py

@@ -1,5 +1,10 @@
 SUPPORTED_PROTOCOLS = ["http", "https", "s3", "gs"]
 
+# Sent on every outgoing HTTP request. Some servers reject the default agents of
+# the underlying libraries, e.g. geoportal.saarland.de answers 403 to any agent
+# containing "aiohttp", which is what fsspec sends.
+USER_AGENT = "vecorel-cli"
+
 COMPRESSION_METHODS = ["brotli", "gzip", "lz4", "snappy", "zstd", "none"]
 
 GEOPARQUET_VERSIONS = ["1.0.0", "1.1.0"]

{vecorel_cli-0.2.17 → vecorel_cli-0.2.18}/vecorel_cli/conversion/base.py

@@ -62,6 +62,11 @@ class BaseConverter(LoggerMixin):
 
     index_as_id: bool = False
 
+    # Rows with null values in schema-required properties are dropped up to this
+    # share of all rows; above it the conversion fails. Rows with an empty or
+    # missing geometry are always dropped, regardless of this share.
+    max_dropped_share: float = 0.01
+
     def __init__(self, *args, **kwargs):
         super().__init__()
 
@@ -71,6 +76,148 @@ class BaseConverter(LoggerMixin):
             if not key.startswith("_") and isinstance(item, (list, dict, set)):
                 setattr(self, key, copy(item))
 
+    def _require_one_source_of_urls(self):
+        """Fail when both `sources` and `variants` are declared:
+        `sources` wins and every `--variant` would silently convert the same file."""
+        if self.sources and self.variants:
+            raise ValueError(
+                f"{type(self).__name__} declares both sources and variants; sources wins "
+                "and every --variant would convert the same file. Drop sources, or set "
+                "variants = {} when the inherited ones do not apply."
+            )
+
+    def _check_id_mapping(self):
+        """Warn before converting when nothing is mapped to the required `id` property.
+        Unmapped columns are dropped, which also removes the column filled by
+        `index_as_id` unless the converter maps `"id": "id"`."""
+        targets = set()
+        for value in list(self.columns.values()) + list(self.column_additions or {}):
+            targets.update(value if isinstance(value, (list, tuple)) else [value])
+        if "id" not in targets:
+            hint = (
+                ' — `index_as_id = True` is set, so add \'"id": "id"\' to columns'
+                if self.index_as_id
+                else " — map a unique source column to it, or set index_as_id = True"
+                ' and add \'"id": "id"\' to columns'
+            )
+            self.warning(f"{type(self).__name__} maps no column to 'id'{hint}")
+
+    def _check_unique_ids(self, gdf, columns):
+        """Warn when the column that becomes `id` does not identify a feature.
+        Runs before geometries are exploded, so it judges what the converter
+        assigned rather than the split parts of one source feature. Null ids
+        are not counted here; they are dropped under a bounded rule afterwards.
+        """
+        sources = [
+            k for k, v in columns.items() if "id" in (v if isinstance(v, (list, tuple)) else [v])
+        ]
+        column = next(
+            (c for c in sources if c in gdf.columns), "id" if "id" in gdf.columns else None
+        )
+        if column is None:
+            self.warning(
+                f"{type(self).__name__}: none of the columns mapped to 'id' "
+                f"({', '.join(sources) or 'none'}) is in this source; it has "
+                f"{', '.join(sorted(gdf.columns)[:12])}"
+            )
+            return
+        ids = gdf[column].dropna()
+        if ids.is_unique:
+            return
+        counts = ids.value_counts()
+        duplicated = int(len(ids) - len(counts))
+        worst = int(counts.iloc[0])
+        self.warning(
+            f"{type(self).__name__}: '{column}' is not unique — {duplicated:,} of {len(ids):,} "
+            f"rows repeat an id (one appears {worst:,} times), so it cannot be `id`. Map a column "
+            "that identifies a feature, build one from the source's key columns, or use the row "
+            "index (index_as_id) only when the conversion reads a single file."
+        )
+
+    def _drop_incomplete_rows(self, gdf, columns):
+        """Drop rows that can never validate. Rows with null values in a
+        schema-required property are dropped up to ``max_dropped_share``;
+        above it the conversion fails, as quietly dropping large parts of a
+        dataset would hide that the converter needs fixing (and the writer
+        rejects nulls in the non-nullable required fields anyway). Rows with
+        an empty or missing geometry are always dropped: they cannot survive
+        the geometry processing anyway and would break the Hilbert sort."""
+        collection = self.create_collection(self.id.strip())
+        schemas = collection.merge_schemas({})
+        collection_only = set(collection.get_collection_only_properties())
+        required = [
+            r for r in schemas.get("required", []) if r != "geometry" and r not in collection_only
+        ]
+
+        # One combined mask, so max_dropped_share bounds the total share
+        invalid = pd.Series(False, index=gdf.index)
+        reasons = []
+
+        # This runs before columns are renamed, so look up the source column
+        for key in required:
+            for src, dst in columns.items():
+                targets = dst if isinstance(dst, (list, tuple)) else [dst]
+                if key in targets and src in gdf.columns:
+                    nulls = gdf[src].isna()
+                    if nulls.any():
+                        reasons.append(f"{int(nulls.sum())} without a value for {key} ({src})")
+                        invalid |= nulls
+
+        if invalid.any():
+            share = invalid.mean()
+            details = "; ".join(reasons)
+            if share > self.max_dropped_share:
+                raise ValueError(
+                    f"{int(invalid.sum())} of {len(gdf)} rows ({share:.1%}) have no value for "
+                    f"a required property ({details}); fix the converter instead of dropping them"
+                )
+            self.warning(
+                f"Dropping {int(invalid.sum())} of {len(gdf)} rows that can never "
+                f"validate ({details})"
+            )
+            gdf = gdf[~invalid]
+
+        if gdf.active_geometry_name is not None:
+            geom = gdf.geometry
+            blank = geom.isna() | geom.is_empty
+            if blank.any():
+                share = blank.mean()
+                message = (
+                    f"Dropping {int(blank.sum())} of {len(gdf)} rows with an empty "
+                    f"or missing geometry"
+                )
+                if share > self.max_dropped_share:
+                    message += f" ({share:.1%}, exceeds max_dropped_share) — fix the converter"
+                self.warning(message)
+                gdf = gdf[~blank]
+
+        return gdf
+
+    def _prewarm_schemas(self):
+        """Fetch every schema this conversion will need upfront, with retries,
+        so a transient schema-host blip cannot kill the conversion at the very
+        last step. load_file caches per process."""
+        import time
+
+        from ..vecorel.util import load_file
+
+        uris = set(self.extensions)
+        uris.add(Schemas.get_core_uri())
+        attempts = 8
+        for uri in sorted(uris):
+            for attempt in range(attempts):
+                try:
+                    load_file(uri)
+                    break
+                except Exception as e:
+                    if attempt == attempts - 1:
+                        raise RuntimeError(
+                            f"Cannot load schema {uri} after {attempts} attempts: {e}"
+                        ) from e
+                    self.warning(f"Schema fetch failed ({uri}), retrying: {str(e)[:100]}")
+                    # ~4 min of tolerance: schema host outages have outlasted a 30 s budget
+                    time.sleep(min(2**attempt * 2, 60))
+
     def migrate(self, gdf) -> GeoDataFrame:
         return gdf
 
@@ -316,6 +463,10 @@ class BaseConverter(LoggerMixin):
         if self.bbox is not None and len(self.bbox) != 4:
             raise ValueError("If provided, the bounding box must consist of 4 numbers")
 
+        self._check_id_mapping()
+        self._require_one_source_of_urls()
+        self._prewarm_schemas()
+
         # Create output folder if it doesn't exist
         directory = os.path.dirname(output_file)
         if directory:
@@ -360,10 +511,6 @@ class BaseConverter(LoggerMixin):
                 gdf[key] = value
                 columns[key] = key
 
-            # Add collection ID
-            columns["collection"] = "collection"
-            gdf["collection"] = cid
-
         # 4. Run column migrations
         if self.column_migrations:
             self.info("Applying column migrations")
@@ -375,6 +522,9 @@ class BaseConverter(LoggerMixin):
 
         gdf = self.post_migrate(gdf)
 
+        self._check_unique_ids(gdf, columns)
+        gdf = self._drop_incomplete_rows(gdf, columns)
+
         if hash_before != self._hash_df(gdf.head()):
             self.info("GeoDataFrame after migrations and filters:")
             self.info(gdf.head().to_string())
@@ -430,7 +580,9 @@ class BaseConverter(LoggerMixin):
         self.info("Creating GeoParquet file: " + str(output_file))
         columns = list(actual_columns.values())
         pq = GeoParquet(output_file)
-        pq.set_collection(self.create_collection(cid))
+        collection = self.create_collection(cid)
+        collection["collection"] = cid
+        pq.set_collection(collection)
 
         pq.write(
             gdf,