vecorel-cli 0.2.18__tar.gz → 0.2.20__tar.gz

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.2.18
+Version: 0.2.20
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyyaml<7.0,>=6.0
 Requires-Dist: click<9.0,>=8.1
-Requires-Dist: geopandas<2.0,>=1.0.0
+Requires-Dist: geopandas<2.0,>=1.1.0
 Requires-Dist: requests<3.0,>=2.30
 Requires-Dist: shapely<3.0,>=2.1
 Requires-Dist: numpy<3.0,>=2.0
@@ -26,7 +26,7 @@ Requires-Dist: pyarrow<24.0,>=21.0
 Requires-Dist: py7zr<2.0,>=1.0
 Requires-Dist: fsspec>=2025.7.0
 Requires-Dist: jsonschema[format]<5.0,>=4.20
-Requires-Dist: aiohttp<4.0,>=3.9
+Requires-Dist: aiohttp<4.0,>=3.13.5
 Requires-Dist: yarl<2.0,>=1.20
 Requires-Dist: rarfile<5.0,>=4.0
 Requires-Dist: semantic-version<3.0,>=2.10.0

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vecorel-cli"
-version = "0.2.18"
+version = "0.2.20"
 description = "CLI tools such as validation and file format conversion for vecorel."
 readme = "README.md"
 license = "Apache-2.0"
@@ -24,7 +24,7 @@ requires-python = ">=3.11"
 dependencies = [
     "pyyaml>=6.0,<7.0",
     "click>=8.1,<9.0",
-    "geopandas>=1.0.0,<2.0",
+    "geopandas>=1.1.0,<2.0",
     "requests>=2.30,<3.0",
     "shapely>=2.1,<3.0",
     "numpy>=2.0,<3.0",
@@ -32,7 +32,7 @@ dependencies = [
     "py7zr>=1.0,<2.0",
     "fsspec>=2025.7.0",
     "jsonschema[format]>=4.20,<5.0",
-    "aiohttp>=3.9,<4.0",
+    "aiohttp>=3.13.5,<4.0",
     "yarl>=1.20,<2.0",
     "rarfile>=4.0,<5.0",
     "semantic-version>=2.10.0,<3.0",
@@ -65,7 +65,7 @@ platforms = ["win-64", "linux-64", "osx-64", "osx-arm64"]
 python = ">=3.11"
 pyyaml = ">=6.0,<7.0"
 click = ">=8.1,<9.0"
-geopandas = ">=1.0.0,<2.0"
+geopandas = ">=1.1.0,<2.0"
 requests = ">=2.30,<3.0"
 shapely = ">=2.1,<3.0"
 numpy = ">=2.0,<3.0"

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/tests/test_convert.py

@@ -180,3 +180,72 @@ def test_data_access_exception(capsys, tmp_folder):
 
     assert isinstance(converter.converter, BaseConverter)
     assert converter.converter.data_access in out
+
+
+@pytest.mark.parametrize("choice", ["example"])
+def test_converter_can_keep_the_constants_in_columns(tmp_folder, monkeypatch, choice):
+    """A conversion that writes one part of a dataset cannot let constants move
+    into the collection metadata: constant is then judged over the part."""
+    import pyarrow.parquet as pq
+
+    from tests.datasets.example import Converter
+
+    dest = tmp_folder / "hydrated.parquet"
+    monkeypatch.setattr(Converter, "dehydrate", False)
+    ConvertData(choice).convert(dest, cache=(test_path / choice))
+
+    schema = pq.ParquetFile(dest).schema_arrow
+    collection = json.loads(schema.metadata[b"collection"])
+    for key in ("admin:country_code", "admin:subdivision_code", "determination_datetime"):
+        assert key in schema.names, f"{key} should have stayed a column"
+        assert key not in collection, f"{key} should not be in the collection metadata"
+
+
+def test_default_variant_is_chosen_before_get_urls():
+    """A converter that overrides get_urls() must see the default variant, so the
+    default lives in convert() and not in the base get_urls() it replaces."""
+
+    class Converter(BaseConverter):
+        id = "variants"
+        # oldest first: the latest year is still the default
+        variants = {
+            "2024": "https://example.com/2024.gpkg",
+            "2025": "https://example.com/2025.gpkg",
+        }
+
+        def get_urls(self):
+            # like the converters that look their files up by year
+            return {f"https://example.com/{self.variant}/": f"{self.variant}.gpkg"}
+
+    converter = Converter()
+    converter.select_variant(None)
+    assert converter.variant == "2025"
+    assert converter.get_urls() == {"https://example.com/2025/": "2025.gpkg"}
+
+    converter.select_variant("2024")
+    assert converter.variant == "2024"
+    assert converter.get_urls() == {"https://example.com/2024/": "2024.gpkg"}
+
+
+def test_default_variant_is_the_first_declared_unless_the_variants_are_years():
+    class Converter(BaseConverter):
+        id = "variants"
+        variants = {"full": "https://example.com/full.gpkg", "2025": "https://example.com/2025"}
+
+    converter = Converter()
+    converter.select_variant(None)
+    assert converter.variant == "full"
+
+
+def test_get_urls_rejects_an_unknown_variant():
+    class Converter(BaseConverter):
+        id = "variants"
+        variants = {"2025": "https://example.com/2025.gpkg"}
+
+    converter = Converter()
+    converter.select_variant("2025")
+    assert converter.get_urls() == "https://example.com/2025.gpkg"
+
+    converter.select_variant("1999")
+    with pytest.raises(ValueError, match="Unknown variant '1999'"):
+        converter.get_urls()

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/tests/test_convert_duckdb.py

@@ -1,10 +1,12 @@
 import json
+import sys
 
 import geopandas as gpd
 import numpy as np
 import pyarrow.parquet as pq
 import pytest
 import shapely
+from loguru import logger
 
 from vecorel_cli.conversion.base import BaseConverter
 from vecorel_cli.conversion.duckdb import DuckDBBaseConverter
@@ -218,3 +220,195 @@ def test_codepath_parity(tmp_folder):
     for dest in (pandas_dest, duckdb_dest):
         validation = ValidateData().validate(dest, num=100, schema_map={})
         assert validation.errors == []
+
+
+def test_merge_parquet(tmp_folder):
+    """Converted files combine into one, sorted over the whole set and
+    packaged like any other output."""
+    parts = []
+    for index, offset in enumerate((0, 20)):
+        gdf = gpd.GeoDataFrame(
+            {
+                "id": [f"{index}-{n}" for n in range(3)],
+                "name": ["a", "b", "c"],
+                "geometry": [shapely.box(offset + n, 0, offset + n + 1, 1) for n in range(3)],
+            },
+            crs="EPSG:4326",
+        )
+        src = tmp_folder / f"src_{index}.parquet"
+        gdf.to_parquet(src)
+        part = tmp_folder / f"part_{index}.parquet"
+        Converter().convert(part, input_files={str(src): src.name})
+        parts.append(part)
+
+    dest = tmp_folder / "merged.parquet"
+    Converter().merge_parquet(parts, dest)
+
+    result = gpd.read_parquet(dest)
+    assert sorted(result["id"]) == ["0-0", "0-1", "0-2", "1-0", "1-1", "1-2"]
+
+    with pq.ParquetFile(dest) as pf:
+        table = pf.read()
+    # sorted over the merged set, not per part
+    keys = hilbert_keys_for_table(table, "geometry", (-180.0, -90.0, 180.0, 90.0))
+    assert bool(np.all(keys[1:] >= keys[:-1]))
+    assert "bbox" in table.schema.names
+    assert b"collection" in table.schema.metadata
+
+    assert ValidateData().validate(dest, num=100, schema_map={}).errors == []
+
+
+def test_merge_parquet_sees_an_id_that_repeats_across_parts(tmp_folder, capsys):
+    """A per-part check cannot see this; a check over the merge can."""
+    parts = []
+    for index in range(2):
+        gdf = gpd.GeoDataFrame(
+            {
+                "id": ["same", f"other-{index}"],
+                "name": ["a", "b"],
+                "geometry": [
+                    shapely.box(index, 0, index + 1, 1),
+                    shapely.box(index, 2, index + 1, 3),
+                ],
+            },
+            crs="EPSG:4326",
+        )
+        src = tmp_folder / f"dup_src_{index}.parquet"
+        gdf.to_parquet(src)
+        part = tmp_folder / f"dup_part_{index}.parquet"
+        Converter().convert(part, input_files={str(src): src.name})
+        parts.append(part)
+
+    # loguru's default sink is bound to stderr at import, so point it at stdout
+    logger.remove()
+    logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False)
+    Converter().merge_parquet(parts, tmp_folder / "dup_merged.parquet")
+    assert "'id' is not unique" in capsys.readouterr().out
+
+
+def test_merge_parquet_rejects_empty_input(tmp_folder):
+    with pytest.raises(ValueError, match="No paths"):
+        Converter().merge_parquet([], tmp_folder / "nothing.parquet")
+
+
+def test_duckdb_converter_can_keep_the_constants_in_columns(tmp_folder):
+    """A conversion that writes one part of a dataset cannot let constants move
+    into the collection metadata, in this codepath either."""
+    src = _source_file(tmp_folder)
+    dest = tmp_folder / "hydrated.parquet"
+
+    Hydrated = type(
+        "Hydrated",
+        (DuckDBBaseConverter,),
+        {
+            **CONFIG,
+            "dehydrate": False,
+            "column_additions": {"region": "north"},
+            "missing_schemas": {
+                "properties": {"name": {"type": "string"}, "region": {"type": "string"}}
+            },
+        },
+    )
+    Hydrated().convert(dest, input_files={src: "source.parquet"})
+
+    table = pq.read_table(dest)
+    assert "region" in table.schema.names, "the constant should have stayed a column"
+    assert set(table.column("region").to_pylist()) == {"north"}
+    assert "region" not in json.loads(table.schema.metadata[b"collection"])
+
+
+def test_merge_parquet_refuses_parts_in_different_crs(tmp_folder):
+    parts = []
+    for index, crs in enumerate(("EPSG:4326", "EPSG:3857")):
+        gdf = gpd.GeoDataFrame(
+            {"id": [f"{index}"], "name": ["a"], "geometry": [shapely.box(0, 0, 1, 1)]},
+            crs="EPSG:4326",
+        ).to_crs(crs)
+        src = tmp_folder / f"crs_src_{index}.parquet"
+        gdf.to_parquet(src)
+        part = tmp_folder / f"crs_part_{index}.parquet"
+        Converter().convert(part, input_files={str(src): src.name})
+        parts.append(part)
+
+    with pytest.raises(ValueError, match="different coordinate reference systems"):
+        Converter().merge_parquet(parts, tmp_folder / "merged.parquet")
+
+
+def test_merge_parquet_unions_parts_that_differ(tmp_folder):
+    """A converter drops a column a source file does not have, so two parts of one
+    dataset can differ; the merge must not fail on that."""
+    parts = []
+    for index, columns in enumerate(({"id": ["0"], "name": ["a"]}, {"id": ["1"]})):
+        gdf = gpd.GeoDataFrame(
+            {**columns, "geometry": [shapely.box(index, 0, index + 1, 1)]}, crs="EPSG:4326"
+        )
+        src = tmp_folder / f"u_src_{index}.parquet"
+        gdf.to_parquet(src)
+        part = tmp_folder / f"u_part_{index}.parquet"
+        Converter().convert(part, input_files={str(src): src.name})
+        parts.append(part)
+
+    dest = tmp_folder / "unioned.parquet"
+    Converter().merge_parquet(parts, dest)
+    table = pq.read_table(dest)
+    assert table.num_rows == 2
+    assert "name" in table.schema.names
+    assert sorted(x for x in table.column("id").to_pylist()) == ["0", "1"]
+
+
+def test_merge_parquet_checks_ids_that_convert_generated(tmp_folder, capsys):
+    """Each part numbered its own rows from zero, so the merge has to check what
+    convert() is allowed to take for granted."""
+    IndexConverter = type("IndexConverter", (DuckDBBaseConverter,), {**CONFIG, "index_as_id": True})
+    parts = []
+    for index in range(2):
+        gdf = gpd.GeoDataFrame(
+            {
+                "name": ["a", "b"],
+                "geometry": [
+                    shapely.box(index, 0, index + 1, 1),
+                    shapely.box(index, 2, index + 1, 3),
+                ],
+            },
+            crs="EPSG:4326",
+        )
+        src = tmp_folder / f"idx_src_{index}.parquet"
+        gdf.to_parquet(src)
+        part = tmp_folder / f"idx_part_{index}.parquet"
+        IndexConverter().convert(part, input_files={str(src): src.name})
+        parts.append(part)
+
+    # both parts start at 0, so every id occurs twice
+    assert (
+        pq.read_table(parts[0]).column("id").to_pylist()
+        == pq.read_table(parts[1]).column("id").to_pylist()
+    )
+
+    logger.remove()
+    logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False)
+    IndexConverter().merge_parquet(parts, tmp_folder / "idx_merged.parquet")
+    assert "'id' is not unique" in capsys.readouterr().out
+
+
+def test_merge_parquet_keeps_a_crs_duckdb_would_drop(tmp_folder):
+    """DuckDB before 1.5 writes no CRS into the merged file's metadata, so it has to
+    come from the parts. Checked with a CRS that is not the default."""
+    parts = []
+    for index in range(2):
+        gdf = gpd.GeoDataFrame(
+            {"id": [f"{index}"], "name": ["a"], "geometry": [shapely.box(index, 0, index + 1, 1)]},
+            crs="EPSG:4326",
+        ).to_crs("EPSG:3857")
+        src = tmp_folder / f"crs_keep_src_{index}.parquet"
+        gdf.to_parquet(src)
+        part = tmp_folder / f"crs_keep_part_{index}.parquet"
+        Converter().convert(part, input_files={str(src): src.name})
+        parts.append(part)
+
+    dest = tmp_folder / "crs_keep.parquet"
+    Converter().merge_parquet(parts, dest)
+
+    geo = json.loads(pq.read_table(dest).schema.metadata[b"geo"])
+    crs = geo["columns"][geo["primary_column"]]["crs"]
+    assert crs is not None, "the merged file lost the CRS of its parts"
+    assert "3857" in json.dumps(crs)

vecorel_cli-0.2.20/tests/test_download.py

@@ -0,0 +1,59 @@
+import pytest
+
+import vecorel_cli.conversion.base as base
+from vecorel_cli.conversion.base import BaseConverter
+
+URI = "https://example.invalid/data/fields.gml"
+
+
+def test_failed_download_leaves_no_cache_file(tmp_folder, monkeypatch):
+    """A download that raises must not leave a file behind that a later run
+    would treat as cached (#43)."""
+
+    def failing_stream(fs, src_uri, dst_file, chunk_size=None):
+        dst_file.write(b"partial")
+        raise RuntimeError("403 Forbidden")
+
+    monkeypatch.setattr(base, "stream_file", failing_stream)
+
+    with pytest.raises(RuntimeError, match="403"):
+        BaseConverter().download_files(URI, cache_folder=str(tmp_folder))
+
+    assert list(tmp_folder.iterdir()) == [], "cache must be empty after a failed download"
+
+
+def test_failed_download_is_retried_on_next_run(tmp_folder, monkeypatch):
+    calls = []
+
+    def stream(fs, src_uri, dst_file, chunk_size=None):
+        calls.append(src_uri)
+        if len(calls) == 1:
+            raise RuntimeError("timeout")
+        dst_file.write(b"<gml/>")
+
+    monkeypatch.setattr(base, "stream_file", stream)
+
+    with pytest.raises(RuntimeError):
+        BaseConverter().download_files(URI, cache_folder=str(tmp_folder))
+
+    paths = BaseConverter().download_files(URI, cache_folder=str(tmp_folder))
+
+    assert len(calls) == 2, "the second run must download again"
+    assert paths == [(str(tmp_folder / "fields.gml"), URI)]
+    assert (tmp_folder / "fields.gml").read_bytes() == b"<gml/>"
+    assert not (tmp_folder / "fields.gml.part").exists()
+
+
+def test_successful_download_is_cached(tmp_folder, monkeypatch):
+    calls = []
+
+    def stream(fs, src_uri, dst_file, chunk_size=None):
+        calls.append(src_uri)
+        dst_file.write(b"<gml/>")
+
+    monkeypatch.setattr(base, "stream_file", stream)
+
+    BaseConverter().download_files(URI, cache_folder=str(tmp_folder))
+    BaseConverter().download_files(URI, cache_folder=str(tmp_folder))
+
+    assert calls == [URI], "a completed download must be reused"

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/tests/test_encoding_geoparquet.py

@@ -1,6 +1,8 @@
 import json
 from pathlib import Path
 
+import numpy as np
+import pandas as pd
 import pyarrow as pa
 import pyarrow.parquet as pq
 import pyarrow.types as pat
@@ -123,3 +125,26 @@ def test_postprocess(tmp_parquet_file):
         geo = json.loads(pf.metadata.metadata[b"geo"])
     assert geo["version"] == "1.0.0"
     assert "covering" not in geo["columns"]["geometry"]
+
+
+def test_read_keeps_integers_that_have_a_null(tmp_parquet_file):
+    # pandas turns an integer column with a missing value into float64, after which
+    # every value in it reads as a float and validation rejects the column
+    src = pq.read_table("tests/data-files/inspire.parquet")
+    counts = pa.array([7] * (len(src) - 1) + [None], pa.uint32())
+    table = src.append_column(pa.field("count", pa.uint32(), nullable=True), counts)
+
+    metadata = dict(src.schema.metadata)
+    collection = json.loads(metadata[b"collection"])
+    collection["schemas:custom"]["properties"]["count"] = {"type": "uint32"}
+    metadata[b"collection"] = json.dumps(collection).encode("utf-8")
+    pq.write_table(table.replace_schema_metadata(metadata), tmp_parquet_file, store_schema=True)
+
+    data = GeoParquet(tmp_parquet_file).read()
+    assert str(data["count"].dtype) == "UInt32"
+    assert isinstance(data["count"].iloc[0], np.uint32)
+    assert pd.isna(data["count"].iloc[-1])
+
+    from vecorel_cli.validate import ValidateData
+
+    assert ValidateData().validate(tmp_parquet_file).errors == []

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli/conversion/base.py

@@ -4,6 +4,7 @@ import hashlib
 import inspect
 import json
 import os
+import re
 import sys
 import tarfile
 import zipfile
@@ -67,6 +68,12 @@ class BaseConverter(LoggerMixin):
     # missing geometry are always dropped, regardless of this share.
     max_dropped_share: float = 0.01
 
+    # Move properties that hold one value for every row into the collection metadata.
+    # A conversion that writes one part of a dataset must not: "one value for every
+    # row" is then judged over the part rather than over the dataset, so a property
+    # that varies between parts is lost.
+    dehydrate: bool = True
+
     def __init__(self, *args, **kwargs):
         super().__init__()
 
@@ -86,6 +93,24 @@ class BaseConverter(LoggerMixin):
                 "variants = {} when the inherited ones do not apply."
             )
 
+    def select_variant(self, variant: Optional[str]) -> None:
+        """Store the requested variant; without one, the default variant is used.
+        Done at the start of convert() rather than in get_urls(), so a converter that
+        overrides get_urls() does not have to repeat the default, and so the default is
+        also set when the user supplies the input files."""
+        self.variant = variant
+        if self.variants and self.variant is None:
+            self.variant = self.default_variant()
+            opts = ", ".join(self.variants)
+            self.warning(f"No variant given, choosing {self.variant} from {opts}")
+
+    def default_variant(self) -> str:
+        """The latest year when the variants are years, otherwise the first declared."""
+        keys = list(self.variants)
+        if all(re.fullmatch(r"\d{4}", key) for key in keys):
+            return max(keys)
+        return keys[0]
+
     def _check_id_mapping(self):
         """Warn before converting when nothing is mapped to the required `id` property.
         Unmapped columns are dropped, which also removes the column filled by
@@ -279,8 +304,7 @@ class BaseConverter(LoggerMixin):
             must_extract = is_archive and not os.path.exists(zip_folder)
 
             if (not is_archive or must_extract) and not cache_fs.exists(cache_file):
-                with cache_fs.open(cache_file, mode="wb") as file:
-                    stream_file(source_fs, uri, file)
+                self._download_file(source_fs, uri, cache_fs, cache_file)
 
             if must_extract:
                 if zipfile.is_zipfile(cache_file):
@@ -316,16 +340,34 @@ class BaseConverter(LoggerMixin):
 
         return paths
 
+    @staticmethod
+    def _download_file(source_fs, uri, cache_fs, cache_file):
+        """Stream a remote file into the cache.
+
+        The data goes to a `.part` file that is renamed only after a clean close,
+        so an interrupted download leaves nothing that a later run could mistake
+        for a cached file.
+        """
+        part_file = cache_file + ".part"
+        try:
+            with cache_fs.open(part_file, mode="wb") as file:
+                stream_file(source_fs, uri, file)
+        except BaseException:
+            try:
+                cache_fs.rm(part_file)
+            except FileNotFoundError:
+                pass
+            raise
+        cache_fs.mv(part_file, cache_file)
+
     def get_urls(self):
         urls = self.sources
         if not urls and self.variants:
-            opts = ", ".join(list(self.variants.keys()))
-            if self.variant is None:
-                self.variant = next(iter(self.variants))
-                self.warning(f"Choosing first available variant {self.variant} from {opts}")
+            # the default variant is chosen in convert(), see select_variant()
             if self.variant in self.variants:
                 urls = self.variants[self.variant]
             else:
+                opts = ", ".join(self.variants)
                 raise ValueError(f"Unknown variant '{self.variant}', choose from {opts}")
         return urls
 
@@ -458,7 +500,7 @@ class BaseConverter(LoggerMixin):
         original_geometries=False,
         **kwargs,
     ) -> str:
-        self.variant = variant
+        self.select_variant(variant)
         cid = self.id.strip()
         if self.bbox is not None and len(self.bbox) != 4:
             raise ValueError("If provided, the bounding box must consist of 4 numbers")
@@ -587,6 +629,7 @@ class BaseConverter(LoggerMixin):
         pq.write(
             gdf,
             properties=columns,
+            dehydrate=self.dehydrate,
             compression=compression,
             compression_level=compression_level,
             geoparquet_version=geoparquet_version,

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli/conversion/duckdb.py

@@ -22,6 +22,29 @@ def _sql_path(path) -> str:
     return f"'{escaped}'"
 
 
+# A COPY statement binds its own parameters before those of its subquery, so a value
+# placed in the SELECT cannot be a bound parameter: it would be read as the output path.
+def _sql_literal(value) -> str:
+    if value is None:
+        return "NULL"
+    if isinstance(value, bool):
+        return "TRUE" if value else "FALSE"
+    if isinstance(value, int):
+        return repr(value)
+    if isinstance(value, float):
+        if np.isnan(value):
+            return "'NaN'::DOUBLE"
+        if np.isposinf(value):
+            return "'Infinity'::DOUBLE"
+        if np.isneginf(value):
+            return "'-Infinity'::DOUBLE"
+        return repr(value)
+    if not isinstance(value, str):
+        raise ValueError(f"Cannot use {value!r} as a constant column; it is not a scalar")
+    escaped = value.replace("'", "''")
+    return f"'{escaped}'"
+
+
 def _normalize_crs(crs):
     """The comparable pyproj CRS for a GeoParquet crs value,
     which defaults to OGC:CRS84 when missing."""
@@ -51,14 +74,7 @@ class DuckDBBaseConverter(BaseConverter):
         original_geometries=False,
         **kwargs,
     ) -> str:
-        geoparquet_version = geoparquet_version or "1.1.0"
-        # Same packaging as the GeoDataFrame-based codepath
-        compression = compression or "zstd"
-        if compression == "zstd" and compression_level is None:
-            compression_level = 15
-        row_group_size = GeoParquet.row_group_size
-
-        self.variant = variant
+        self.select_variant(variant)
         cid = self.id.strip()
         if self.bbox is not None and len(self.bbox) != 4:
             raise ValueError("If provided, the bounding box must consist of 4 numbers")
@@ -108,28 +124,7 @@ class DuckDBBaseConverter(BaseConverter):
             ).fetchall()
         }
 
-        # DuckDB drops the CRS from the GeoParquet metadata, so take it from the
-        # sources for the Hilbert grid and the output metadata. The sources are
-        # combined without reprojection, so they must all use the same CRS.
-        source_crs = None
-        reference = None
-        for i, source in enumerate([sources] if isinstance(sources, str) else sources):
-            crs = None
-            row = con.execute(
-                "SELECT value FROM parquet_kv_metadata(?) WHERE key = 'geo'", [source]
-            ).fetchone()
-            if row:
-                source_geo = json.loads(bytes(row[0]))
-                primary_column = source_geo.get("primary_column", "")
-                crs = source_geo.get("columns", {}).get(primary_column, {}).get("crs")
-            if i == 0:
-                source_crs = crs
-                reference = _normalize_crs(crs)
-            elif not _equal_crs(_normalize_crs(crs), reference):
-                raise ValueError(
-                    f"The sources use different coordinate reference systems: {source} "
-                    "differs from the first source. Reproject the sources to a common CRS."
-                )
+        source_crs = self._common_crs(con, [sources] if isinstance(sources, str) else sources)
         selections = []
         selected_targets = []
         for k, v in self.columns.items():
@@ -150,10 +145,9 @@ class DuckDBBaseConverter(BaseConverter):
         collection = self.create_collection(cid)
         collection["collection"] = cid
 
-        # Constants pinned to the feature level become literal columns, all others
-        # end up in the collection metadata (like the dehydration step of the
-        # GeoDataFrame-based codepath)
-        addition_params = []
+        # A constant the schema pins to the feature level becomes a literal column and one
+        # it pins to the collection goes there; the rest follow `dehydrate`, like the
+        # dehydration step of the GeoDataFrame-based codepath
         if self.column_additions:
             context = collection.get_collection_context()
             for key, value in self.column_additions.items():
@@ -162,12 +156,12 @@ class DuckDBBaseConverter(BaseConverter):
                     keep = [i for i, t in enumerate(selected_targets) if t != key]
                     selections = [selections[i] for i in keep]
                     selected_targets = [selected_targets[i] for i in keep]
-                if context.get(key) is False:
-                    selections.append(f'? as "{key}"')
-                    addition_params.append(value)
-                    selected_targets.append(key)
-                else:
+                where = context.get(key)
+                if where is True or (where is None and self.dehydrate):
                     collection[key] = value
+                else:
+                    selections.append(f'{_sql_literal(value)} as "{key}"')
+                    selected_targets.append(key)
         selection = ", ".join(selections)
 
         filters = []
@@ -191,11 +185,6 @@ class DuckDBBaseConverter(BaseConverter):
         if len(filters) > 0:
             where = f"WHERE {' AND '.join(filters)}"
 
-        if isinstance(output_file, Path):
-            output_file = str(output_file)
-
-        collection_json = json.dumps(collection, cls=VecorelJSONEncoder).encode("utf-8")
-
         if isinstance(sources, str):
             sources_sql = _sql_path(sources)
         else:
@@ -206,6 +195,89 @@ class DuckDBBaseConverter(BaseConverter):
             {where}
         """
 
+        return self.write_query(
+            con,
+            source_query,
+            output_file,
+            collection,
+            targets=selected_targets,
+            source_crs=source_crs,
+            ids_are_generated=self.index_as_id,
+            compression=compression,
+            compression_level=compression_level,
+            geoparquet_version=geoparquet_version,
+            original_geometries=original_geometries,
+        )
+
+    def _common_crs(self, con, sources: list):
+        """The CRS the sources declare, refusing a set that does not agree on one.
+
+        They are combined without reprojection, and DuckDB before 1.5 drops the CRS
+        from the metadata, so it has to be read from the files themselves.
+        """
+        source_crs = None
+        reference = None
+        for i, source in enumerate(sources):
+            crs = None
+            row = con.execute(
+                "SELECT value FROM parquet_kv_metadata(?) WHERE key = 'geo'", [source]
+            ).fetchone()
+            if row:
+                geo = json.loads(bytes(row[0]))
+                primary = geo.get("primary_column", "")
+                crs = geo.get("columns", {}).get(primary, {}).get("crs")
+            if i == 0:
+                source_crs = crs
+                reference = _normalize_crs(crs)
+            elif not _equal_crs(_normalize_crs(crs), reference):
+                raise ValueError(
+                    f"The sources use different coordinate reference systems: {source} "
+                    "differs from the first source. Reproject the sources to a common CRS."
+                )
+        return source_crs
+
+    def write_query(
+        self,
+        con,
+        source_query: str,
+        output_file,
+        collection,
+        targets: list,
+        source_crs=None,
+        # convert() numbers the rows itself, so they are unique by construction; a merge
+        # combines parts that each started over, so it always has to check
+        ids_are_generated: bool = False,
+        compression: Optional[str] = None,
+        compression_level: Optional[int] = None,
+        geoparquet_version: Optional[str] = None,
+        original_geometries: bool = False,
+    ) -> str:
+        """Write the rows a SELECT returns as a Vecorel GeoParquet file: drop rows
+        that no required property or geometry survives, report an id that is not
+        unique, normalize the geometries, sort into Hilbert order and package.
+
+        `targets` names the properties the query returns, which is what the checks
+        run over.
+        """
+        compression = compression or "zstd"
+        if compression == "zstd" and compression_level is None:
+            compression_level = 15
+        geoparquet_version = geoparquet_version or "1.1.0"
+        row_group_size = GeoParquet.row_group_size
+        if isinstance(output_file, Path):
+            output_file = str(output_file)
+        directory = os.path.dirname(output_file)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        selected_targets = targets
+        collection_json = json.dumps(collection, cls=VecorelJSONEncoder).encode("utf-8")
+
+        # An external sort spills to disk; keep that next to the output, which is
+        # where there is room for it, rather than wherever DuckDB defaults to
+        con.execute(
+            f"SET temp_directory = {_sql_path(os.path.join(os.path.dirname(output_file) or '.', '.duckdb_tmp'))}"
+        )
+
         # Same bounded null-value drop, empty-geometry drop and id uniqueness
         # check as in the GeoDataFrame-based codepath, in one scan
         schemas = collection.merge_schemas({})
@@ -221,7 +293,7 @@ class DuckDBBaseConverter(BaseConverter):
             null_cond = " OR ".join(f'"{target}" IS NULL' for target in required)
             stats.append(f"count(*) FILTER (WHERE {null_cond})")
         # row numbers are unique by construction
-        check_ids = "id" in selected_targets and not self.index_as_id
+        check_ids = "id" in selected_targets and not ids_are_generated
         if check_ids:
             stats.append('count("id")')
             stats.append('count(DISTINCT "id")')
@@ -231,9 +303,7 @@ class DuckDBBaseConverter(BaseConverter):
             stats.append(f"count(*) FILTER (WHERE {blank_cond})")
         if len(stats) > 1:
             values = list(
-                con.execute(
-                    f"SELECT {', '.join(stats)} FROM ({source_query})", addition_params
-                ).fetchone()
+                con.execute(f"SELECT {', '.join(stats)} FROM ({source_query})").fetchone()
             )
             total = values.pop(0)
             invalid = values.pop(0) if null_cond else 0
@@ -301,7 +371,7 @@ class DuckDBBaseConverter(BaseConverter):
                 }}
             )
         """,
-            [*addition_params, output_file, compression, collection_json],
+            [output_file, compression, collection_json],
         )
 
         # Sort against the same CRS-derived Hilbert grid as the
@@ -343,6 +413,47 @@ class DuckDBBaseConverter(BaseConverter):
 
         return output_file
 
+    def merge_parquet(self, paths: list, output_file, collection=None, **kwargs) -> str:
+        """Combine Vecorel GeoParquet files into one, checked and sorted over the
+        whole set rather than per file.
+
+        The parts are written by a converter, so they need no column mapping and
+        their geometries are already valid polygons; pass `original_geometries=False`
+        to run the geometry step anyway.
+        """
+        if not paths:
+            raise ValueError("No paths to merge")
+        paths = [str(path) for path in paths]
+        kwargs.setdefault("original_geometries", True)
+
+        con = duckdb.connect()
+        con.install_extension("spatial")
+        con.load_extension("spatial")
+
+        source_crs = self._common_crs(con, paths)
+
+        # union_by_name, because a converter drops a column a source file does not have,
+        # so two parts of one dataset can legitimately differ; the targets then come from
+        # the union rather than from whichever part happens to be first
+        sources = "[" + ",".join(_sql_path(path) for path in paths) + "]"
+        source_query = f"SELECT * FROM read_parquet({sources}, union_by_name=true)"
+        targets = [row[0] for row in con.execute(f"DESCRIBE {source_query}").fetchall()]
+
+        if collection is None:
+            from ..vecorel.ops import merge_collections
+
+            collection = merge_collections([GeoParquet(path).get_collection() for path in paths])
+
+        return self.write_query(
+            con,
+            source_query,
+            output_file,
+            collection,
+            targets=targets,
+            source_crs=source_crs,
+            **kwargs,
+        )
+
     # Streams the Hilbert keys per row group to a sidecar file and reports
     # whether the file is already sorted, so memory stays bounded
     def _write_hilbert_keys(self, output_file, primary, bounds):

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli/encoding/geoparquet.py

@@ -17,6 +17,7 @@ from ..const import GEOPARQUET_DEFAULT_VERSION, GEOPARQUET_VERSIONS
 from ..encoding.geojson import VecorelJSONEncoder
 from ..parquet.geopandas import to_parquet
 from ..parquet.types import (
+    NULLABLE_INTEGERS,
     get_geopandas_dtype,
     get_pyarrow_field,
     get_pyarrow_type,
@@ -516,7 +517,7 @@ class GeoParquet(BaseEncoding):
             rows = next(pf.iter_batches(batch_size=num, columns=properties))
             table = pa.Table.from_batches([rows])
 
-        gdf = _arrow_to_geopandas(table)
+        gdf = _arrow_to_geopandas(table, to_pandas_kwargs={"types_mapper": NULLABLE_INTEGERS.get})
 
         if hydrate:
             gdf = self.hydrate_from_collection(gdf, schema_map=schema_map)

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli/parquet/types.py

@@ -6,6 +6,20 @@ import pyarrow as pa
 import pyarrow.types as pat
 from shapely.geometry.base import BaseGeometry
 
+# pandas has no integer dtype that holds a null, so an integer column with one
+# missing value comes back as float64 and every value in it then reads as a float.
+# The nullable dtypes do hold one, so integers are read into those.
+NULLABLE_INTEGERS = {
+    pa.int8(): pd.Int8Dtype(),
+    pa.uint8(): pd.UInt8Dtype(),
+    pa.int16(): pd.Int16Dtype(),
+    pa.uint16(): pd.UInt16Dtype(),
+    pa.int32(): pd.Int32Dtype(),
+    pa.uint32(): pd.UInt32Dtype(),
+    pa.int64(): pd.Int64Dtype(),
+    pa.uint64(): pd.UInt64Dtype(),
+}
+
 
 def is_enum(schema):
     return isinstance(schema.get("enum"), list)

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.2.18
+Version: 0.2.20
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pyyaml<7.0,>=6.0
 Requires-Dist: click<9.0,>=8.1
-Requires-Dist: geopandas<2.0,>=1.0.0
+Requires-Dist: geopandas<2.0,>=1.1.0
 Requires-Dist: requests<3.0,>=2.30
 Requires-Dist: shapely<3.0,>=2.1
 Requires-Dist: numpy<3.0,>=2.0
@@ -26,7 +26,7 @@ Requires-Dist: pyarrow<24.0,>=21.0
 Requires-Dist: py7zr<2.0,>=1.0
 Requires-Dist: fsspec>=2025.7.0
 Requires-Dist: jsonschema[format]<5.0,>=4.20
-Requires-Dist: aiohttp<4.0,>=3.9
+Requires-Dist: aiohttp<4.0,>=3.13.5
 Requires-Dist: yarl<2.0,>=1.20
 Requires-Dist: rarfile<5.0,>=4.0
 Requires-Dist: semantic-version<3.0,>=2.10.0

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ tests/test_create_geojson.py
 tests/test_create_geoparquet.py
 tests/test_create_stac.py
 tests/test_describe.py
+tests/test_download.py
 tests/test_encoding_auto.py
 tests/test_encoding_geojson.py
 tests/test_encoding_geoparquet.py

{vecorel_cli-0.2.18 → vecorel_cli-0.2.20}/vecorel_cli.egg-info/requires.txt

@@ -1,6 +1,6 @@
 pyyaml<7.0,>=6.0
 click<9.0,>=8.1
-geopandas<2.0,>=1.0.0
+geopandas<2.0,>=1.1.0
 requests<3.0,>=2.30
 shapely<3.0,>=2.1
 numpy<3.0,>=2.0
@@ -8,7 +8,7 @@ pyarrow<24.0,>=21.0
 py7zr<2.0,>=1.0
 fsspec>=2025.7.0
 jsonschema[format]<5.0,>=4.20
-aiohttp<4.0,>=3.9
+aiohttp<4.0,>=3.13.5
 yarl<2.0,>=1.20
 rarfile<5.0,>=4.0
 semantic-version<3.0,>=2.10.0