vecorel-cli 0.2.15__tar.gz → 0.2.17__tar.gz

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.2.15
+Version: 0.2.17
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -24,7 +24,7 @@ Requires-Dist: shapely<3.0,>=2.1
 Requires-Dist: numpy<3.0,>=2.0
 Requires-Dist: pyarrow<24.0,>=21.0
 Requires-Dist: py7zr<2.0,>=1.0
-Requires-Dist: fsspec==2025.7.0
+Requires-Dist: fsspec>=2025.7.0
 Requires-Dist: jsonschema[format]<5.0,>=4.20
 Requires-Dist: aiohttp<4.0,>=3.9
 Requires-Dist: yarl<2.0,>=1.20
@@ -32,6 +32,13 @@ Requires-Dist: rarfile<5.0,>=4.0
 Requires-Dist: semantic-version<3.0,>=2.10.0
 Requires-Dist: json-stream<3.0,>=2.3.0
 Requires-Dist: loguru==0.7.3
+Provides-Extra: s3
+Requires-Dist: s3fs>=2025.7.0; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: gcsfs>=2025.7.0; extra == "gcs"
+Provides-Extra: cloud
+Requires-Dist: s3fs>=2025.7.0; extra == "cloud"
+Requires-Dist: gcsfs>=2025.7.0; extra == "cloud"
 Dynamic: license-file
 
 # Vecorel CLI
@@ -65,12 +72,14 @@ pixi install
 pixi run vec
 ```
 
-#### Using pip
+#### Using pip or uv
 
 Alternatively, you can install from PyPI with **Python 3.11** or any later version:
 
 ```bash
 pip install vecorel-cli
+# or
+uv pip install vecorel-cli
 ```
 
 ### Execute a command
@@ -115,9 +124,9 @@ Check `vec validate --help` for more details.
 The validator also supports remote files.
 
 - `http://` or `https://`: no further configuration is needed.
-- `s3://`: With Pixi, run `pixi install -e s3` or with pip, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
+- `s3://`: With Pixi, run `pixi install -e s3` or with pip/uv, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
   Refer to the [s3fs credentials documentation](https://s3fs.readthedocs.io/en/latest/#credentials) for how to define credentials.
-- `gs://`: With Pixi, run `pixi install -e gcs` or with pip, run `pip install vecorel-cli[gcs]`.
+- `gs://`: With Pixi, run `pixi install -e gcs` or with pip/uv, run `pip install vecorel-cli[gcs]`.
   By default, `gcsfs` will attempt to use your default gcloud credentials or, attempt to get credentials from the google metadata service, or fall back to anonymous access.
 
 ### Create Vecorel GeoParquet from GeoJSON
@@ -237,14 +246,14 @@ pixi install -e dev
 pixi run install-dev
 
 # Run tests
-pixi run test
+pixi run -e dev test
 
 # Format and lint code
 pixi run format
 pixi run lint
 
 # Run all checks (lint, format, test)
-pixi run check
+pixi run -e dev check
 
 # Install and run pre-commit
 pixi run pre-commit-install
@@ -257,6 +266,6 @@ The following high-level description gives an idea how to implement a converter
 
 1. Create a new file in `vecorel_cli/datasets` based on the `template.py`
 2. Fill in the required variables / test it / run it
-3. Add missing dependencies into the appropriate feature group in `pixi.toml` (or `setup.py` for pip users)
+3. Add missing dependencies into `pyproject.toml` (and the appropriate pixi feature group if needed)
 4. Add the converter to the list above
 5. Create a PR to submit your converter for review

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/README.md

@@ -29,12 +29,14 @@ pixi install
 pixi run vec
 ```
 
-#### Using pip
+#### Using pip or uv
 
 Alternatively, you can install from PyPI with **Python 3.11** or any later version:
 
 ```bash
 pip install vecorel-cli
+# or
+uv pip install vecorel-cli
 ```
 
 ### Execute a command
@@ -79,9 +81,9 @@ Check `vec validate --help` for more details.
 The validator also supports remote files.
 
 - `http://` or `https://`: no further configuration is needed.
-- `s3://`: With Pixi, run `pixi install -e s3` or with pip, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
+- `s3://`: With Pixi, run `pixi install -e s3` or with pip/uv, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
   Refer to the [s3fs credentials documentation](https://s3fs.readthedocs.io/en/latest/#credentials) for how to define credentials.
-- `gs://`: With Pixi, run `pixi install -e gcs` or with pip, run `pip install vecorel-cli[gcs]`.
+- `gs://`: With Pixi, run `pixi install -e gcs` or with pip/uv, run `pip install vecorel-cli[gcs]`.
   By default, `gcsfs` will attempt to use your default gcloud credentials or, attempt to get credentials from the google metadata service, or fall back to anonymous access.
 
 ### Create Vecorel GeoParquet from GeoJSON
@@ -201,14 +203,14 @@ pixi install -e dev
 pixi run install-dev
 
 # Run tests
-pixi run test
+pixi run -e dev test
 
 # Format and lint code
 pixi run format
 pixi run lint
 
 # Run all checks (lint, format, test)
-pixi run check
+pixi run -e dev check
 
 # Install and run pre-commit
 pixi run pre-commit-install
@@ -221,6 +223,6 @@ The following high-level description gives an idea how to implement a converter
 
 1. Create a new file in `vecorel_cli/datasets` based on the `template.py`
 2. Fill in the required variables / test it / run it
-3. Add missing dependencies into the appropriate feature group in `pixi.toml` (or `setup.py` for pip users)
+3. Add missing dependencies into `pyproject.toml` (and the appropriate pixi feature group if needed)
 4. Add the converter to the list above
 5. Create a PR to submit your converter for review

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vecorel-cli"
-version = "0.2.15"
+version = "0.2.17"
 description = "CLI tools such as validation and file format conversion for vecorel."
 readme = "README.md"
 license = "Apache-2.0"
@@ -30,7 +30,7 @@ dependencies = [
     "numpy>=2.0,<3.0",
     "pyarrow>=21.0,<24.0",
     "py7zr>=1.0,<2.0",
-    "fsspec==2025.7.0",
+    "fsspec>=2025.7.0",
     "jsonschema[format]>=4.20,<5.0",
     "aiohttp>=3.9,<4.0",
     "yarl>=1.20,<2.0",
@@ -40,6 +40,11 @@ dependencies = [
     "loguru==0.7.3",
 ]
 
+[project.optional-dependencies]
+s3 = ["s3fs>=2025.7.0"]
+gcs = ["gcsfs>=2025.7.0"]
+cloud = ["s3fs>=2025.7.0", "gcsfs>=2025.7.0"]
+
 [project.scripts]
 vec = "vecorel_cli.cli.setup:run"
 
@@ -81,15 +86,19 @@ GitPython = ">=3.0.0,<4.0.0"
 [tool.pixi.feature.dev.pypi-dependencies]
 build = ">=1.0.0,<2.0.0"
 
-[tool.pixi.feature.cloud.dependencies]
-s3fs = "==2025.7.0"
-gcsfs = "==2025.7.0"
+[tool.pixi.feature.s3.dependencies]
+s3fs = ">=2025.7.0"
+
+[tool.pixi.feature.gcs.dependencies]
+gcsfs = ">=2025.7.0"
 
 [tool.pixi.environments]
 default = {solve-group = "default"}
 dev = {features = ["dev"], solve-group = "default"}
-cloud = {features = ["cloud"], solve-group = "default"}
-all = {features = ["dev", "cloud"], solve-group = "default"}
+s3 = {features = ["s3"], solve-group = "default"}
+gcs = {features = ["gcs"], solve-group = "default"}
+cloud = {features = ["s3", "gcs"], solve-group = "default"}
+all = {features = ["dev", "s3", "gcs"], solve-group = "default"}
 
 [tool.pixi.tasks]
 # Development tasks

vecorel_cli-0.2.17/tests/test_convert.py

@@ -0,0 +1,182 @@
+import json
+import sys
+from pathlib import Path
+
+import geopandas as gpd
+import numpy as np
+import pytest
+from loguru import logger
+
+from vecorel_cli.conversion.base import BaseConverter
+from vecorel_cli.convert import ConvertData
+from vecorel_cli.registry import Registry
+from vecorel_cli.validate import ValidateData
+from vecorel_cli.vecorel.hilbert import crs_total_bounds, hilbert_distances_from_bounds
+
+
+@pytest.fixture(autouse=True)
+def registry_reset():
+    ignored = Registry.ignored_datasets
+    src_package = Registry.src_package
+    Registry.src_package = "tests"
+    yield
+    Registry.ignore_datasets = ignored
+    Registry.src_package = src_package
+
+
+test_path = Path("tests/data-files/convert")
+
+
+@pytest.mark.parametrize("choice", ["example"])
+def test_converter(tmp_folder, choice):
+    dest = tmp_folder / "converted.parquet"
+    converter = ConvertData(choice)
+    converter.convert(dest, cache=(test_path / choice))
+
+    assert dest.exists(), f"Expected file {dest} to be created."
+
+    # todo: Validation works, but fails for the created file
+    validator = ValidateData()
+    validation_result = validator.validate(
+        dest,
+        num=100,
+        schema_map={},
+    )
+    assert validation_result.errors == []
+
+
+@pytest.mark.parametrize("choice", ["example"])
+def test_converter_hilbert_curve_order(tmp_folder, choice):
+    """The converter must emit rows in Hilbert-curve order against the CRS's
+    total bounds. This is what makes per-partition outputs of the same dataset
+    mergeable without re-sorting: they all share a CRS-derived Hilbert grid."""
+    dest = tmp_folder / "converted.parquet"
+    ConvertData(choice).convert(dest, cache=(test_path / choice))
+
+    gdf = gpd.read_parquet(dest)
+    assert len(gdf) > 1, "Need at least 2 rows to verify ordering."
+
+    total_bounds = crs_total_bounds(gdf.crs)
+    bounds = gdf.geometry.bounds.to_numpy(dtype=np.float64, copy=False)
+    keys = hilbert_distances_from_bounds(bounds, total_bounds)
+
+    # NB: keys are uint64 — np.diff would wrap descents into huge positives
+    # and never flag them; compare adjacent keys directly instead.
+    descents = keys[1:] < keys[:-1]
+    if np.any(descents):
+        bad = int(np.argmax(descents))
+        raise AssertionError(
+            "Output rows are not in non-decreasing Hilbert order against the "
+            f"CRS total bounds ({total_bounds}).\n"
+            f"  First out-of-order pair at index {bad}->{bad + 1}: "
+            f"key {int(keys[bad])} > {int(keys[bad + 1])}."
+        )
+
+
+def test_crs_total_bounds_geographic():
+    # Geographic CRS without a more restrictive area_of_use falls back to world.
+    assert crs_total_bounds("EPSG:4326") == (-180.0, -90.0, 180.0, 90.0)
+
+
+def test_crs_total_bounds_etrs89():
+    # EPSG:4258 is geographic with a European area_of_use; the returned bounds
+    # must reflect that AoU rather than world bounds (otherwise Hilbert keys
+    # for European data would all be quantised into one tiny cell).
+    bounds = crs_total_bounds("EPSG:4258")
+    assert bounds[0] > -180.0 and bounds[2] < 180.0, bounds
+    assert bounds[1] > -90.0 and bounds[3] < 90.0, bounds
+
+
+def test_hilbert_distances_match_public_geopandas_api():
+    # hilbert_distances_from_bounds exists for callers that hold bbox columns
+    # without decoded geometry; it must stay identical to the public
+    # GeoSeries.hilbert_distance, which guards its private-API internals.
+    from shapely.geometry import box
+
+    rng = np.random.default_rng(42)
+    xs = rng.uniform(4.0, 6.0, 50)
+    ys = rng.uniform(51.0, 53.0, 50)
+    geoms = [box(x, y, x + 0.01, y + 0.01) for x, y in zip(xs, ys)]
+    gdf = gpd.GeoDataFrame(geometry=geoms, crs="EPSG:4326")
+    total = crs_total_bounds(gdf.crs)
+    public = gdf.geometry.hilbert_distance(total_bounds=list(total), level=16).to_numpy()
+    ours = hilbert_distances_from_bounds(
+        gdf.geometry.bounds.to_numpy(dtype=np.float64), total, level=16
+    )
+    assert np.array_equal(public.astype(np.uint64), ours)
+
+
+def test_crs_total_bounds_antimeridian():
+    # NZGD2000 / NZTM: the area of use spans the antimeridian (west > east);
+    # the projected bounds must still be finite and non-degenerate.
+    xmin, ymin, xmax, ymax = crs_total_bounds("EPSG:2193")
+    assert xmax > xmin and ymax > ymin
+    assert all(np.isfinite(v) for v in (xmin, ymin, xmax, ymax))
+
+
+def test_hilbert_sort_falls_back_without_area_of_use():
+    # A custom projected CRS without area_of_use must not fail the conversion;
+    # the sorter falls back to the dataset's own bounds.
+    from shapely.geometry import Point
+
+    from vecorel_cli.vecorel.hilbert import hilbert_sort_geodataframe
+
+    crs = "+proj=tmerc +lat_0=0 +lon_0=9 +k=1 +x_0=0 +y_0=0 +ellps=WGS84 +units=m +no_defs"
+    gdf = gpd.GeoDataFrame(geometry=[Point(0, 0), Point(1000, 1000), Point(10, 10)], crs=crs)
+    out = hilbert_sort_geodataframe(gdf)
+    assert len(out) == 3
+
+
+def test_not_existing_converter(tmp_folder):
+    with pytest.raises(Exception, match="Converter 'not_existing' not found"):
+        converter = ConvertData("not_existing")
+        converter.convert(tmp_folder / "converted.parquet")
+
+
+@pytest.mark.parametrize("choice", ["invalid_syntax", "invalid_name"])
+def test_invalid_converter(tmp_folder, choice):
+    with pytest.raises(Exception, match=f"Converter for '{choice}' not available or faulty:"):
+        converter = ConvertData(choice)
+        converter.convert(tmp_folder / "converted.parquet")
+
+
+def test_template_from_package_folder():
+    Registry.src_package = "vecorel_cli"
+    Registry.ignored_datasets = []
+    converter = ConvertData("template")
+    assert isinstance(converter, ConvertData), "Should succeed and not throw an exception"
+
+
+@pytest.mark.parametrize("bom", [False, True], ids=["without-bom", "with-bom"])
+def test_read_geojson_decodes_utf8(tmp_folder, cp1252_locale, bom):
+    file_path = tmp_folder / "umlaut.json"
+    feature = {
+        "type": "Feature",
+        "id": "1",
+        "properties": {"name": "Grünland"},
+        "geometry": {"type": "Point", "coordinates": [0, 0]},
+    }
+    file_path.write_text(
+        # ensure_ascii=False so the file really holds multi-byte UTF-8, not an escape sequence
+        json.dumps({"type": "FeatureCollection", "features": [feature]}, ensure_ascii=False),
+        encoding="utf-8-sig" if bom else "utf-8",
+    )
+
+    gdf = BaseConverter().read_geojson(str(file_path))
+
+    assert gdf["name"].iloc[0] == "Grünland"
+
+
+def test_data_access_exception(capsys, tmp_folder):
+    # todo: use fixture
+    logger.remove()
+    logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False)
+
+    with pytest.raises(Exception, match="Please provide the input data."):
+        converter = ConvertData("data_access")
+        converter.convert(tmp_folder / "converted.parquet")
+
+    out, err = capsys.readouterr()
+
+    assert isinstance(converter.converter, BaseConverter)
+    assert converter.converter.data_access in out

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/tests/test_create_geoparquet.py

@@ -40,3 +40,21 @@ def test_create_geoparquet_invalid_file(tmp_folder):
         gp.create("invalid.parquet", out)
     with pytest.raises(FileNotFoundError):
         gp.create(["invalid.json"], out)
+
+
+def test_create_geoparquet_collection_only_properties(tmp_parquet_file: Path):
+    # https://github.com/vecorel/cli/issues/26
+    inputs = ["tests/data-files/collection-only.json"]
+    creator = CreateGeoParquet()
+    creator.create(inputs, tmp_parquet_file)
+    assert tmp_parquet_file.exists()
+
+    gp = GeoParquet(tmp_parquet_file)
+
+    collection = gp.get_collection()
+    assert collection.get("source_name") == "Example Source"
+
+    data = gp.read()
+    assert len(data) == 2
+    # collection-only properties must not be added as columns
+    assert "source_name" not in data.columns

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/tests/test_encoding_geojson.py

@@ -1,3 +1,4 @@
+import json
 from pathlib import Path
 
 import pytest
@@ -55,3 +56,25 @@ def test_get_collection_returns_existing(tmp_folder):
 
     result = geojson.get_collection()
     assert result == expected
+
+
+@pytest.mark.parametrize("bom", [False, True], ids=["without-bom", "with-bom"])
+def test_read_geojson_decodes_utf8(tmp_folder, cp1252_locale, bom):
+    feature = {
+        "type": "Feature",
+        "id": "1",
+        "properties": {"name": "Grünland"},
+        "geometry": {"type": "Point", "coordinates": [0, 0]},
+    }
+    file_path = tmp_folder / "umlaut.json"
+    file_path.write_text(
+        # ensure_ascii=False so the file really holds multi-byte UTF-8, not an escape sequence
+        json.dumps({"type": "FeatureCollection", "features": [feature]}, ensure_ascii=False),
+        encoding="utf-8-sig" if bom else "utf-8",
+    )
+
+    geojson = GeoJSON(file_path)
+
+    assert geojson.read_geojson()["features"][0]["properties"]["name"] == "Grünland"
+    # read(num=...) takes the streaming branch, which opens the file separately
+    assert geojson.read(num=1)["name"].iloc[0] == "Grünland"

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/tests/test_ops.py

@@ -55,3 +55,40 @@ def test_merge_collections():
             },
         }
     )
+
+
+def test_merge_collections_keeps_collection_properties():
+    # https://github.com/vecorel/cli/issues/26
+    schemas = {"c1": ["https://example.com/schema1.yaml"]}
+    collection1 = Collection(
+        {
+            "schemas": schemas,
+            "source_name": "Example",
+            "source_version": "1.0",
+            "only_in_first": "x",
+        }
+    )
+    collection2 = Collection(
+        {
+            "schemas": schemas,
+            "source_name": "Example",
+            "source_version": "2.0",
+        }
+    )
+
+    merged = merge_collections([collection1])
+    assert merged.get("source_name") == "Example"
+    assert merged.get("source_version") == "1.0"
+    assert merged.get("only_in_first") == "x"
+
+    merged = merge_collections([collection1, collection2])
+    # present in all collections with the same value => kept
+    assert merged.get("source_name") == "Example"
+    # conflicting values => dropped
+    assert "source_version" not in merged
+    # missing in one collection => dropped
+    assert "only_in_first" not in merged
+
+    merged = merge_collections([collection1], properties=["only_in_first"])
+    assert "source_name" not in merged
+    assert merged.get("only_in_first") == "x"

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/tests/test_validate_schema.py

@@ -1,8 +1,16 @@
+from click.testing import CliRunner
 from jsonschema.exceptions import ValidationError
 
 from vecorel_cli.validate_schema import ValidateSchema
 
 
+def get_cli_command():
+    cmd = ValidateSchema.get_cli_command(ValidateSchema)
+    for arg in ValidateSchema.get_cli_args().values():
+        cmd = arg(cmd)
+    return cmd
+
+
 def test_validate_schema_valid():
     path = "tests/data-files/sdl/inspire-schema.yaml"
     cmd = ValidateSchema()
@@ -20,3 +28,15 @@ def test_validate_schema_invalid():
     error = result[0]
     assert isinstance(error, ValidationError)
     assert error.message.startswith("'STRING' is not one of [")
+
+
+def test_validate_schema_cli_valid_exit_code():
+    result = CliRunner().invoke(get_cli_command(), ["tests/data-files/sdl/inspire-schema.yaml"])
+    assert result.exit_code == 0
+
+
+def test_validate_schema_cli_invalid_exit_code():
+    result = CliRunner().invoke(
+        get_cli_command(), ["tests/data-files/sdl/inspire-schema-invalid.yaml"]
+    )
+    assert result.exit_code != 0

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli/basecommand.py

@@ -61,6 +61,10 @@ class BaseCommand(LoggerMixin):
                 self.exception(e)
                 sys.exit(1)
 
+        # Exit with a non-zero exit code if the runnable reported a failure
+        if result is False:
+            sys.exit(1)
+
         # Report command as finished
         if self.cmd_final_report:
             if isinstance(result, Path):

vecorel_cli-0.2.17/vecorel_cli/const.py

@@ -0,0 +1,12 @@
+SUPPORTED_PROTOCOLS = ["http", "https", "s3", "gs"]
+
+COMPRESSION_METHODS = ["brotli", "gzip", "lz4", "snappy", "zstd", "none"]
+
+GEOPARQUET_VERSIONS = ["1.0.0", "1.1.0"]
+GEOPARQUET_DEFAULT_VERSION = "1.1.0"
+
+# Default level used when sorting features along a Hilbert curve. A level of N
+# splits each axis of the reference bounds into 2**N cells; 16 gives a
+# 65,536 x 65,536 grid, fine enough for world-scale references at sub-km
+# resolution near the equator.
+HILBERT_DEFAULT_LEVEL = 16

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli/conversion/base.py

@@ -24,8 +24,10 @@ from geopandas import GeoDataFrame
 
 from ..cli.logger import LoggerMixin
 from ..cli.util import display_pandas_unrestricted
+from ..encoding.geojson import READ_ENCODING
 from ..encoding.geoparquet import GeoParquet
 from ..vecorel.collection import Collection
+from ..vecorel.hilbert import hilbert_sort_geodataframe
 from ..vecorel.schemas import Schemas
 from ..vecorel.typing import Sources
 from ..vecorel.util import get_fs, name_from_uri, stream_file
@@ -217,6 +219,9 @@ class BaseConverter(LoggerMixin):
                 yield GeoDataFrame(data), path, uri, layer
 
     def read_geojson(self, path, **kwargs):
+        # open_options is shared with gpd.read_parquet() and gpd.read_file(), so a converter
+        # cannot set a default encoding there without breaking the other two readers.
+        kwargs.setdefault("encoding", READ_ENCODING)
         with open(path, **kwargs) as f:
             obj = json.load(f)
 
@@ -408,7 +413,12 @@ class BaseConverter(LoggerMixin):
                 self.info("Removing Z geometry dimension")
                 gdf.geometry = gdf.geometry.force_2d()
 
-        gdf.sort_values("geometry", inplace=True, ignore_index=True)
+        # Sort by Hilbert distance against the CRS's total bounds. This gives
+        # row groups good spatial locality, and — crucially — produces the same
+        # ordering for any independently-converted partition of the same dataset
+        # (they share a CRS, therefore a Hilbert reference grid), so per-file
+        # outputs can be merged later without re-sorting.
+        gdf = hilbert_sort_geodataframe(gdf)
 
         # 8. Remove all columns that are not listed
         drop_columns = list(set(gdf.columns) - set(actual_columns.values()))

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli/create_stac.py

@@ -212,7 +212,7 @@ class CreateStacCollection(BaseCommand):
                 )
 
         # Add provider information
-        provider = collection.get("provider", "").strip()
+        provider = (collection.get("provider") or "").strip()
         if len(provider) > 0:
             provider_name, provider_url = parse_link_str(provider)
             stac["providers"] = [

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli/encoding/geojson.py

@@ -17,6 +17,11 @@ from ..vecorel.util import load_file, to_iso8601
 from ..vecorel.version import vecorel_version
 from .base import BaseEncoding
 
+# GeoJSON is UTF-8 (RFC 8259, section 8.1). Reading tolerates a byte order mark, which some
+# producers emit; writing must not add one, so the two differ.
+READ_ENCODING = "utf-8-sig"
+WRITE_ENCODING = "utf-8"
+
 
 class GeoJSON(BaseEncoding):
     feature_properties = {"type", "id", "geometry", "bbox", "properties"}
@@ -152,7 +157,7 @@ class GeoJSON(BaseEncoding):
         enforce_featurecollection: bool = False,
     ) -> Union[FeatureCollection, Feature]:
         # num only applies to FeatureCollections
-        with open(self.uri, "r") as f:
+        with open(self.uri, "r", encoding=READ_ENCODING) as f:
             obj = json.load(f)
 
         if not isinstance(obj, dict):
@@ -233,7 +238,7 @@ class GeoJSON(BaseEncoding):
         schema_map: SchemaMapping = {},
         properties: Optional[list[str]] = None,
     ) -> GeoDataFrame:
-        with open(self.uri, "r") as f:
+        with open(self.uri, "r", encoding=READ_ENCODING) as f:
             stream = json_stream.load(f)
             data = {
                 "id": [],
@@ -291,7 +296,7 @@ class GeoJSON(BaseEncoding):
             return gdf
 
     def _write_json(self, obj, path, indent=None) -> bool:
-        with open(path, "w") as f:
+        with open(path, "w", encoding=WRITE_ENCODING) as f:
             json.dump(obj, f, allow_nan=False, indent=indent, cls=VecorelJSONEncoder)
         return True
 

vecorel_cli-0.2.17/vecorel_cli/vecorel/hilbert.py

@@ -0,0 +1,142 @@
+"""Hilbert curve sorting against CRS-stable reference bounds.
+
+Sorting by Hilbert distance gives row groups good spatial locality, which
+improves bbox-pushdown query performance and downstream compression. We
+deliberately sort against the *CRS's* declared extent rather than the
+dataframe's own bbox: that way, two independently produced files (for
+example two provincial parts that will be merged later) end up using the
+same Hilbert reference grid, and their orderings are comparable.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from loguru import logger
+
+from ..const import HILBERT_DEFAULT_LEVEL
+
+# Geographic world bounds, used as the fallback when a CRS exposes no
+# `area_of_use` (e.g. some custom or aggregated CRSs).
+_GEOGRAPHIC_WORLD_BOUNDS = (-180.0, -90.0, 180.0, 90.0)
+
+
+def crs_total_bounds(crs) -> tuple[float, float, float, float]:
+    """Return a stable, deterministic ``(xmin, ymin, xmax, ymax)`` covering the
+    CRS's valid extent expressed in the CRS's own coordinate units.
+
+    The CRS's ``area_of_use`` (which pyproj always reports as WGS84 degrees,
+    regardless of the CRS's own units or prime meridian) is projected onto the
+    CRS axes, sampled along a 25x25 grid to capture curvature. Antimeridian-
+    spanning areas of use (``west > east``, e.g. New Zealand) are sampled
+    across the wrapped longitude span.
+
+    The result is independent of any particular dataset: it depends only on
+    the CRS. Two independently-converted parts of the same dataset therefore
+    share the same Hilbert reference grid and their orderings can be merged
+    without recomputation.
+
+    Raises ``ValueError`` for a projected CRS without a declared area of use;
+    callers that can tolerate a dataset-dependent grid should catch it and
+    substitute their own reference extent.
+    """
+    from pyproj import CRS, Transformer
+
+    if crs is None:
+        return _GEOGRAPHIC_WORLD_BOUNDS
+
+    c = CRS.from_user_input(crs)
+    aou = c.area_of_use
+
+    if aou is None:
+        if c.is_geographic:
+            return _GEOGRAPHIC_WORLD_BOUNDS
+        raise ValueError(
+            f"Cannot derive total bounds for projected CRS {crs!r}: no area_of_use defined"
+        )
+
+    west, south, east, north = (
+        float(aou.west),
+        float(aou.south),
+        float(aou.east),
+        float(aou.north),
+    )
+    # Antimeridian-spanning area of use: sample across the wrapped span.
+    # pyproj accepts longitudes beyond 180 and wraps them itself.
+    if west > east:
+        east += 360.0
+
+    # Sample the (WGS84) area of use into the target CRS. This also applies to
+    # geographic target CRSs: their own axes may use different units (grads)
+    # or a non-Greenwich prime meridian, so the AoU degrees cannot be returned
+    # verbatim.
+    transformer = Transformer.from_crs("EPSG:4326", c, always_xy=True)
+    n = 25
+    xs = np.linspace(west, east, n)
+    ys = np.linspace(south, north, n)
+    grid_x, grid_y = np.meshgrid(xs, ys)
+    px, py = transformer.transform(grid_x.ravel(), grid_y.ravel())
+    px = np.asarray(px, dtype=np.float64)
+    py = np.asarray(py, dtype=np.float64)
+    mask = np.isfinite(px) & np.isfinite(py)
+    if not mask.any():
+        raise ValueError(f"Could not project area_of_use into CRS {crs!r}")
+    return (
+        float(px[mask].min()),
+        float(py[mask].min()),
+        float(px[mask].max()),
+        float(py[mask].max()),
+    )
+
+
+def hilbert_distances_from_bounds(
+    bounds: np.ndarray,
+    total_bounds: tuple[float, float, float, float],
+    level: int = HILBERT_DEFAULT_LEVEL,
+) -> np.ndarray:
+    """Compute per-row Hilbert distances from feature bbox rows + CRS bounds.
+
+    ``bounds`` must be an ``(N, 4)`` float64 array of ``[xmin, ymin, xmax, ymax]``
+    per feature. Returns a ``(N,)`` uint64 array of Hilbert keys; sorting by
+    this key gives the Hilbert-curve ordering at the given resolution.
+
+    This is the bbox-column twin of the public
+    ``GeoSeries.hilbert_distance(total_bounds=..., level=...)`` (for callers
+    that hold covering bboxes without decoded geometry, e.g. streaming
+    parquet merges); a test asserts the two stay identical, guarding the
+    private-API import below against geopandas changes.
+    """
+    from geopandas.tools.hilbert_curve import _continuous_to_discrete_coords, _encode
+
+    x, y = _continuous_to_discrete_coords(bounds, level, list(total_bounds))
+    return _encode(level, x, y).astype(np.uint64, copy=False)
+
+
+def hilbert_sort_geodataframe(gdf, level: int = HILBERT_DEFAULT_LEVEL):
+    """Return a copy of ``gdf`` sorted by Hilbert distance against the CRS's
+    total bounds. The original index is reset.
+
+    If the CRS provides no stable extent (custom projected CRS without an
+    area of use), the dataframe's own bounds are used instead: the order is
+    still spatially local, but not comparable across independently-produced
+    files.
+    """
+    if gdf.empty:
+        return gdf.reset_index(drop=True)
+
+    try:
+        total = crs_total_bounds(gdf.crs)
+    except ValueError as e:
+        logger.warning(f"{e}; sorting against the dataset's own bounds instead")
+        total = tuple(float(v) for v in gdf.total_bounds)
+
+    # public geopandas API; level 16 = a 65,536 x 65,536 reference grid
+    keys = gdf.geometry.hilbert_distance(total_bounds=list(total), level=level).to_numpy()
+    order = np.argsort(keys, kind="stable")
+    return gdf.iloc[order].reset_index(drop=True)
+
+
+__all__ = [
+    "crs_total_bounds",
+    "hilbert_distances_from_bounds",
+    "hilbert_sort_geodataframe",
+]

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli/vecorel/ops.py

@@ -45,22 +45,31 @@ def merge(
 def merge_collections(collections: list[Collection], properties=None) -> Collection:
     schemas = Schemas()
     custom_schemas = VecorelSchema()
+    handled_separately = ("schemas", "schemas:custom")
 
+    # Additional collection metadata (e.g. collection-only properties) is kept
+    # if it's present in all collections with the same value, i.e. it still
+    # applies to the merged dataset as a whole.
+    other_props = None
     for collection in collections:
         schemas.add_all(collection.get_schemas())
 
         custom = collection.get_custom_schemas()
         custom_schemas.merge(custom)
 
-    collection = Collection(
-        {
-            "schemas": schemas,
-        }
-    )
+        props = {k: v for k, v in collection.items() if k not in handled_separately}
+        if other_props is None:
+            other_props = props
+        else:
+            other_props = {k: v for k, v in other_props.items() if k in props and props[k] == v}
 
+    if other_props is None:
+        other_props = {}
     if properties is not None:
+        other_props = {k: v for k, v in other_props.items() if k in properties}
         custom_schemas = custom_schemas.pick(properties)
 
+    collection = Collection({"schemas": schemas, **other_props})
     collection.set_custom_schemas(custom_schemas)
 
     return collection

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vecorel-cli
-Version: 0.2.15
+Version: 0.2.17
 Summary: CLI tools such as validation and file format conversion for vecorel.
 Author: Matthias Mohr
 License-Expression: Apache-2.0
@@ -24,7 +24,7 @@ Requires-Dist: shapely<3.0,>=2.1
 Requires-Dist: numpy<3.0,>=2.0
 Requires-Dist: pyarrow<24.0,>=21.0
 Requires-Dist: py7zr<2.0,>=1.0
-Requires-Dist: fsspec==2025.7.0
+Requires-Dist: fsspec>=2025.7.0
 Requires-Dist: jsonschema[format]<5.0,>=4.20
 Requires-Dist: aiohttp<4.0,>=3.9
 Requires-Dist: yarl<2.0,>=1.20
@@ -32,6 +32,13 @@ Requires-Dist: rarfile<5.0,>=4.0
 Requires-Dist: semantic-version<3.0,>=2.10.0
 Requires-Dist: json-stream<3.0,>=2.3.0
 Requires-Dist: loguru==0.7.3
+Provides-Extra: s3
+Requires-Dist: s3fs>=2025.7.0; extra == "s3"
+Provides-Extra: gcs
+Requires-Dist: gcsfs>=2025.7.0; extra == "gcs"
+Provides-Extra: cloud
+Requires-Dist: s3fs>=2025.7.0; extra == "cloud"
+Requires-Dist: gcsfs>=2025.7.0; extra == "cloud"
 Dynamic: license-file
 
 # Vecorel CLI
@@ -65,12 +72,14 @@ pixi install
 pixi run vec
 ```
 
-#### Using pip
+#### Using pip or uv
 
 Alternatively, you can install from PyPI with **Python 3.11** or any later version:
 
 ```bash
 pip install vecorel-cli
+# or
+uv pip install vecorel-cli
 ```
 
 ### Execute a command
@@ -115,9 +124,9 @@ Check `vec validate --help` for more details.
 The validator also supports remote files.
 
 - `http://` or `https://`: no further configuration is needed.
-- `s3://`: With Pixi, run `pixi install -e s3` or with pip, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
+- `s3://`: With Pixi, run `pixi install -e s3` or with pip/uv, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
   Refer to the [s3fs credentials documentation](https://s3fs.readthedocs.io/en/latest/#credentials) for how to define credentials.
-- `gs://`: With Pixi, run `pixi install -e gcs` or with pip, run `pip install vecorel-cli[gcs]`.
+- `gs://`: With Pixi, run `pixi install -e gcs` or with pip/uv, run `pip install vecorel-cli[gcs]`.
   By default, `gcsfs` will attempt to use your default gcloud credentials or, attempt to get credentials from the google metadata service, or fall back to anonymous access.
 
 ### Create Vecorel GeoParquet from GeoJSON
@@ -237,14 +246,14 @@ pixi install -e dev
 pixi run install-dev
 
 # Run tests
-pixi run test
+pixi run -e dev test
 
 # Format and lint code
 pixi run format
 pixi run lint
 
 # Run all checks (lint, format, test)
-pixi run check
+pixi run -e dev check
 
 # Install and run pre-commit
 pixi run pre-commit-install
@@ -257,6 +266,6 @@ The following high-level description gives an idea how to implement a converter
 
 1. Create a new file in `vecorel_cli/datasets` based on the `template.py`
 2. Fill in the required variables / test it / run it
-3. Add missing dependencies into the appropriate feature group in `pixi.toml` (or `setup.py` for pip users)
+3. Add missing dependencies into `pyproject.toml` (and the appropriate pixi feature group if needed)
 4. Add the converter to the list above
 5. Create a PR to submit your converter for review

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli.egg-info/SOURCES.txt

@@ -69,6 +69,7 @@ vecorel_cli/validation/geoparquet.py
 vecorel_cli/vecorel/__init__.py
 vecorel_cli/vecorel/collection.py
 vecorel_cli/vecorel/extensions.py
+vecorel_cli/vecorel/hilbert.py
 vecorel_cli/vecorel/ops.py
 vecorel_cli/vecorel/schemas.py
 vecorel_cli/vecorel/typing.py

{vecorel_cli-0.2.15 → vecorel_cli-0.2.17}/vecorel_cli.egg-info/requires.txt

@@ -6,7 +6,7 @@ shapely<3.0,>=2.1
 numpy<3.0,>=2.0
 pyarrow<24.0,>=21.0
 py7zr<2.0,>=1.0
-fsspec==2025.7.0
+fsspec>=2025.7.0
 jsonschema[format]<5.0,>=4.20
 aiohttp<4.0,>=3.9
 yarl<2.0,>=1.20
@@ -14,3 +14,13 @@ rarfile<5.0,>=4.0
 semantic-version<3.0,>=2.10.0
 json-stream<3.0,>=2.3.0
 loguru==0.7.3
+
+[cloud]
+s3fs>=2025.7.0
+gcsfs>=2025.7.0
+
+[gcs]
+gcsfs>=2025.7.0
+
+[s3]
+s3fs>=2025.7.0

vecorel_cli-0.2.15/tests/test_convert.py

@@ -1,76 +0,0 @@
-import sys
-from pathlib import Path
-
-import pytest
-from loguru import logger
-
-from vecorel_cli.conversion.base import BaseConverter
-from vecorel_cli.convert import ConvertData
-from vecorel_cli.registry import Registry
-from vecorel_cli.validate import ValidateData
-
-
-@pytest.fixture(autouse=True)
-def registry_reset():
-    ignored = Registry.ignored_datasets
-    src_package = Registry.src_package
-    Registry.src_package = "tests"
-    yield
-    Registry.ignore_datasets = ignored
-    Registry.src_package = src_package
-
-
-test_path = Path("tests/data-files/convert")
-
-
-@pytest.mark.parametrize("choice", ["example"])
-def test_converter(tmp_folder, choice):
-    dest = tmp_folder / "converted.parquet"
-    converter = ConvertData(choice)
-    converter.convert(dest, cache=(test_path / choice))
-
-    assert dest.exists(), f"Expected file {dest} to be created."
-
-    # todo: Validation works, but fails for the created file
-    validator = ValidateData()
-    validation_result = validator.validate(
-        dest,
-        num=100,
-        schema_map={},
-    )
-    assert validation_result.errors == []
-
-
-def test_not_existing_converter(tmp_folder):
-    with pytest.raises(Exception, match="Converter 'not_existing' not found"):
-        converter = ConvertData("not_existing")
-        converter.convert(tmp_folder / "converted.parquet")
-
-
-@pytest.mark.parametrize("choice", ["invalid_syntax", "invalid_name"])
-def test_invalid_converter(tmp_folder, choice):
-    with pytest.raises(Exception, match=f"Converter for '{choice}' not available or faulty:"):
-        converter = ConvertData(choice)
-        converter.convert(tmp_folder / "converted.parquet")
-
-
-def test_template_from_package_folder():
-    Registry.src_package = "vecorel_cli"
-    Registry.ignored_datasets = []
-    converter = ConvertData("template")
-    assert isinstance(converter, ConvertData), "Should succeed and not throw an exception"
-
-
-def test_data_access_exception(capsys, tmp_folder):
-    # todo: use fixture
-    logger.remove()
-    logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False)
-
-    with pytest.raises(Exception, match="Please provide the input data."):
-        converter = ConvertData("data_access")
-        converter.convert(tmp_folder / "converted.parquet")
-
-    out, err = capsys.readouterr()
-
-    assert isinstance(converter.converter, BaseConverter)
-    assert converter.converter.data_access in out

vecorel_cli-0.2.15/vecorel_cli/const.py

@@ -1,6 +0,0 @@
-SUPPORTED_PROTOCOLS = ["http", "https", "s3", "gs"]
-
-COMPRESSION_METHODS = ["brotli", "gzip", "lz4", "snappy", "zstd", "none"]
-
-GEOPARQUET_VERSIONS = ["1.0.0", "1.1.0"]
-GEOPARQUET_DEFAULT_VERSION = "1.1.0"