vecorel-cli 0.2.14__tar.gz → 0.2.16__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/PKG-INFO +17 -8
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/README.md +8 -6
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/pyproject.toml +16 -7
- vecorel_cli-0.2.16/tests/test_convert.py +161 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_create_geoparquet.py +18 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_ops.py +37 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_validate_schema.py +20 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/basecommand.py +4 -0
- vecorel_cli-0.2.16/vecorel_cli/const.py +12 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/conversion/base.py +7 -1
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/conversion/flatdict.py +17 -25
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/create_stac.py +1 -1
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/registry.py +8 -0
- vecorel_cli-0.2.16/vecorel_cli/vecorel/hilbert.py +142 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/ops.py +14 -5
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli.egg-info/PKG-INFO +17 -8
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli.egg-info/SOURCES.txt +1 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli.egg-info/requires.txt +11 -1
- vecorel_cli-0.2.14/tests/test_convert.py +0 -76
- vecorel_cli-0.2.14/vecorel_cli/const.py +0 -6
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/LICENSE +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/setup.cfg +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_converters.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_create_geojson.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_create_stac.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_describe.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_encoding_auto.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_encoding_geojson.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_encoding_geoparquet.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_improve.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_jsonschema.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_merge.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_rename_extension.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/tests/test_validate.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/cli/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/cli/logger.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/cli/options.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/cli/path_url.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/cli/setup.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/cli/util.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/conversion/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/conversion/admin.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/convert.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/converters.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/create_geojson.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/create_geoparquet.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/create_jsonschema.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/datasets/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/datasets/template.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/describe.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/encoding/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/encoding/auto.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/encoding/base.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/encoding/geojson.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/encoding/geoparquet.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/improve.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/jsonschema/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/jsonschema/template.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/merge.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/parquet/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/parquet/geopandas.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/parquet/types.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/rename_extension.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validate.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validate_schema.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validation/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validation/base.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validation/data.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validation/geojson.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/validation/geoparquet.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/__init__.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/collection.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/extensions.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/schemas.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/typing.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/util.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli/vecorel/version.py +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli.egg-info/dependency_links.txt +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli.egg-info/entry_points.txt +0 -0
- {vecorel_cli-0.2.14 → vecorel_cli-0.2.16}/vecorel_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vecorel-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.16
|
|
4
4
|
Summary: CLI tools such as validation and file format conversion for vecorel.
|
|
5
5
|
Author: Matthias Mohr
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -24,7 +24,7 @@ Requires-Dist: shapely<3.0,>=2.1
|
|
|
24
24
|
Requires-Dist: numpy<3.0,>=2.0
|
|
25
25
|
Requires-Dist: pyarrow<24.0,>=21.0
|
|
26
26
|
Requires-Dist: py7zr<2.0,>=1.0
|
|
27
|
-
Requires-Dist: fsspec
|
|
27
|
+
Requires-Dist: fsspec>=2025.7.0
|
|
28
28
|
Requires-Dist: jsonschema[format]<5.0,>=4.20
|
|
29
29
|
Requires-Dist: aiohttp<4.0,>=3.9
|
|
30
30
|
Requires-Dist: yarl<2.0,>=1.20
|
|
@@ -32,6 +32,13 @@ Requires-Dist: rarfile<5.0,>=4.0
|
|
|
32
32
|
Requires-Dist: semantic-version<3.0,>=2.10.0
|
|
33
33
|
Requires-Dist: json-stream<3.0,>=2.3.0
|
|
34
34
|
Requires-Dist: loguru==0.7.3
|
|
35
|
+
Provides-Extra: s3
|
|
36
|
+
Requires-Dist: s3fs>=2025.7.0; extra == "s3"
|
|
37
|
+
Provides-Extra: gcs
|
|
38
|
+
Requires-Dist: gcsfs>=2025.7.0; extra == "gcs"
|
|
39
|
+
Provides-Extra: cloud
|
|
40
|
+
Requires-Dist: s3fs>=2025.7.0; extra == "cloud"
|
|
41
|
+
Requires-Dist: gcsfs>=2025.7.0; extra == "cloud"
|
|
35
42
|
Dynamic: license-file
|
|
36
43
|
|
|
37
44
|
# Vecorel CLI
|
|
@@ -65,12 +72,14 @@ pixi install
|
|
|
65
72
|
pixi run vec
|
|
66
73
|
```
|
|
67
74
|
|
|
68
|
-
#### Using pip
|
|
75
|
+
#### Using pip or uv
|
|
69
76
|
|
|
70
77
|
Alternatively, you can install from PyPI with **Python 3.11** or any later version:
|
|
71
78
|
|
|
72
79
|
```bash
|
|
73
80
|
pip install vecorel-cli
|
|
81
|
+
# or
|
|
82
|
+
uv pip install vecorel-cli
|
|
74
83
|
```
|
|
75
84
|
|
|
76
85
|
### Execute a command
|
|
@@ -115,9 +124,9 @@ Check `vec validate --help` for more details.
|
|
|
115
124
|
The validator also supports remote files.
|
|
116
125
|
|
|
117
126
|
- `http://` or `https://`: no further configuration is needed.
|
|
118
|
-
- `s3://`: With Pixi, run `pixi install -e s3` or with pip, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
|
|
127
|
+
- `s3://`: With Pixi, run `pixi install -e s3` or with pip/uv, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
|
|
119
128
|
Refer to the [s3fs credentials documentation](https://s3fs.readthedocs.io/en/latest/#credentials) for how to define credentials.
|
|
120
|
-
- `gs://`: With Pixi, run `pixi install -e gcs` or with pip, run `pip install vecorel-cli[gcs]`.
|
|
129
|
+
- `gs://`: With Pixi, run `pixi install -e gcs` or with pip/uv, run `pip install vecorel-cli[gcs]`.
|
|
121
130
|
By default, `gcsfs` will attempt to use your default gcloud credentials or, attempt to get credentials from the google metadata service, or fall back to anonymous access.
|
|
122
131
|
|
|
123
132
|
### Create Vecorel GeoParquet from GeoJSON
|
|
@@ -237,14 +246,14 @@ pixi install -e dev
|
|
|
237
246
|
pixi run install-dev
|
|
238
247
|
|
|
239
248
|
# Run tests
|
|
240
|
-
pixi run test
|
|
249
|
+
pixi run -e dev test
|
|
241
250
|
|
|
242
251
|
# Format and lint code
|
|
243
252
|
pixi run format
|
|
244
253
|
pixi run lint
|
|
245
254
|
|
|
246
255
|
# Run all checks (lint, format, test)
|
|
247
|
-
pixi run check
|
|
256
|
+
pixi run -e dev check
|
|
248
257
|
|
|
249
258
|
# Install and run pre-commit
|
|
250
259
|
pixi run pre-commit-install
|
|
@@ -257,6 +266,6 @@ The following high-level description gives an idea how to implement a converter
|
|
|
257
266
|
|
|
258
267
|
1. Create a new file in `vecorel_cli/datasets` based on the `template.py`
|
|
259
268
|
2. Fill in the required variables / test it / run it
|
|
260
|
-
3. Add missing dependencies into
|
|
269
|
+
3. Add missing dependencies into `pyproject.toml` (and the appropriate pixi feature group if needed)
|
|
261
270
|
4. Add the converter to the list above
|
|
262
271
|
5. Create a PR to submit your converter for review
|
|
@@ -29,12 +29,14 @@ pixi install
|
|
|
29
29
|
pixi run vec
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
-
#### Using pip
|
|
32
|
+
#### Using pip or uv
|
|
33
33
|
|
|
34
34
|
Alternatively, you can install from PyPI with **Python 3.11** or any later version:
|
|
35
35
|
|
|
36
36
|
```bash
|
|
37
37
|
pip install vecorel-cli
|
|
38
|
+
# or
|
|
39
|
+
uv pip install vecorel-cli
|
|
38
40
|
```
|
|
39
41
|
|
|
40
42
|
### Execute a command
|
|
@@ -79,9 +81,9 @@ Check `vec validate --help` for more details.
|
|
|
79
81
|
The validator also supports remote files.
|
|
80
82
|
|
|
81
83
|
- `http://` or `https://`: no further configuration is needed.
|
|
82
|
-
- `s3://`: With Pixi, run `pixi install -e s3` or with pip, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
|
|
84
|
+
- `s3://`: With Pixi, run `pixi install -e s3` or with pip/uv, run `pip install vecorel-cli[s3]` and you may need to set environment variables.
|
|
83
85
|
Refer to the [s3fs credentials documentation](https://s3fs.readthedocs.io/en/latest/#credentials) for how to define credentials.
|
|
84
|
-
- `gs://`: With Pixi, run `pixi install -e gcs` or with pip, run `pip install vecorel-cli[gcs]`.
|
|
86
|
+
- `gs://`: With Pixi, run `pixi install -e gcs` or with pip/uv, run `pip install vecorel-cli[gcs]`.
|
|
85
87
|
By default, `gcsfs` will attempt to use your default gcloud credentials or, attempt to get credentials from the google metadata service, or fall back to anonymous access.
|
|
86
88
|
|
|
87
89
|
### Create Vecorel GeoParquet from GeoJSON
|
|
@@ -201,14 +203,14 @@ pixi install -e dev
|
|
|
201
203
|
pixi run install-dev
|
|
202
204
|
|
|
203
205
|
# Run tests
|
|
204
|
-
pixi run test
|
|
206
|
+
pixi run -e dev test
|
|
205
207
|
|
|
206
208
|
# Format and lint code
|
|
207
209
|
pixi run format
|
|
208
210
|
pixi run lint
|
|
209
211
|
|
|
210
212
|
# Run all checks (lint, format, test)
|
|
211
|
-
pixi run check
|
|
213
|
+
pixi run -e dev check
|
|
212
214
|
|
|
213
215
|
# Install and run pre-commit
|
|
214
216
|
pixi run pre-commit-install
|
|
@@ -221,6 +223,6 @@ The following high-level description gives an idea how to implement a converter
|
|
|
221
223
|
|
|
222
224
|
1. Create a new file in `vecorel_cli/datasets` based on the `template.py`
|
|
223
225
|
2. Fill in the required variables / test it / run it
|
|
224
|
-
3. Add missing dependencies into
|
|
226
|
+
3. Add missing dependencies into `pyproject.toml` (and the appropriate pixi feature group if needed)
|
|
225
227
|
4. Add the converter to the list above
|
|
226
228
|
5. Create a PR to submit your converter for review
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vecorel-cli"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.16"
|
|
8
8
|
description = "CLI tools such as validation and file format conversion for vecorel."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -30,7 +30,7 @@ dependencies = [
|
|
|
30
30
|
"numpy>=2.0,<3.0",
|
|
31
31
|
"pyarrow>=21.0,<24.0",
|
|
32
32
|
"py7zr>=1.0,<2.0",
|
|
33
|
-
"fsspec
|
|
33
|
+
"fsspec>=2025.7.0",
|
|
34
34
|
"jsonschema[format]>=4.20,<5.0",
|
|
35
35
|
"aiohttp>=3.9,<4.0",
|
|
36
36
|
"yarl>=1.20,<2.0",
|
|
@@ -40,6 +40,11 @@ dependencies = [
|
|
|
40
40
|
"loguru==0.7.3",
|
|
41
41
|
]
|
|
42
42
|
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
s3 = ["s3fs>=2025.7.0"]
|
|
45
|
+
gcs = ["gcsfs>=2025.7.0"]
|
|
46
|
+
cloud = ["s3fs>=2025.7.0", "gcsfs>=2025.7.0"]
|
|
47
|
+
|
|
43
48
|
[project.scripts]
|
|
44
49
|
vec = "vecorel_cli.cli.setup:run"
|
|
45
50
|
|
|
@@ -81,15 +86,19 @@ GitPython = ">=3.0.0,<4.0.0"
|
|
|
81
86
|
[tool.pixi.feature.dev.pypi-dependencies]
|
|
82
87
|
build = ">=1.0.0,<2.0.0"
|
|
83
88
|
|
|
84
|
-
[tool.pixi.feature.
|
|
85
|
-
s3fs = "
|
|
86
|
-
|
|
89
|
+
[tool.pixi.feature.s3.dependencies]
|
|
90
|
+
s3fs = ">=2025.7.0"
|
|
91
|
+
|
|
92
|
+
[tool.pixi.feature.gcs.dependencies]
|
|
93
|
+
gcsfs = ">=2025.7.0"
|
|
87
94
|
|
|
88
95
|
[tool.pixi.environments]
|
|
89
96
|
default = {solve-group = "default"}
|
|
90
97
|
dev = {features = ["dev"], solve-group = "default"}
|
|
91
|
-
|
|
92
|
-
|
|
98
|
+
s3 = {features = ["s3"], solve-group = "default"}
|
|
99
|
+
gcs = {features = ["gcs"], solve-group = "default"}
|
|
100
|
+
cloud = {features = ["s3", "gcs"], solve-group = "default"}
|
|
101
|
+
all = {features = ["dev", "s3", "gcs"], solve-group = "default"}
|
|
93
102
|
|
|
94
103
|
[tool.pixi.tasks]
|
|
95
104
|
# Development tasks
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import geopandas as gpd
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pytest
|
|
7
|
+
from loguru import logger
|
|
8
|
+
|
|
9
|
+
from vecorel_cli.conversion.base import BaseConverter
|
|
10
|
+
from vecorel_cli.convert import ConvertData
|
|
11
|
+
from vecorel_cli.registry import Registry
|
|
12
|
+
from vecorel_cli.validate import ValidateData
|
|
13
|
+
from vecorel_cli.vecorel.hilbert import crs_total_bounds, hilbert_distances_from_bounds
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@pytest.fixture(autouse=True)
|
|
17
|
+
def registry_reset():
|
|
18
|
+
ignored = Registry.ignored_datasets
|
|
19
|
+
src_package = Registry.src_package
|
|
20
|
+
Registry.src_package = "tests"
|
|
21
|
+
yield
|
|
22
|
+
Registry.ignore_datasets = ignored
|
|
23
|
+
Registry.src_package = src_package
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
test_path = Path("tests/data-files/convert")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.mark.parametrize("choice", ["example"])
|
|
30
|
+
def test_converter(tmp_folder, choice):
|
|
31
|
+
dest = tmp_folder / "converted.parquet"
|
|
32
|
+
converter = ConvertData(choice)
|
|
33
|
+
converter.convert(dest, cache=(test_path / choice))
|
|
34
|
+
|
|
35
|
+
assert dest.exists(), f"Expected file {dest} to be created."
|
|
36
|
+
|
|
37
|
+
# todo: Validation works, but fails for the created file
|
|
38
|
+
validator = ValidateData()
|
|
39
|
+
validation_result = validator.validate(
|
|
40
|
+
dest,
|
|
41
|
+
num=100,
|
|
42
|
+
schema_map={},
|
|
43
|
+
)
|
|
44
|
+
assert validation_result.errors == []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.mark.parametrize("choice", ["example"])
|
|
48
|
+
def test_converter_hilbert_curve_order(tmp_folder, choice):
|
|
49
|
+
"""The converter must emit rows in Hilbert-curve order against the CRS's
|
|
50
|
+
total bounds. This is what makes per-partition outputs of the same dataset
|
|
51
|
+
mergeable without re-sorting: they all share a CRS-derived Hilbert grid."""
|
|
52
|
+
dest = tmp_folder / "converted.parquet"
|
|
53
|
+
ConvertData(choice).convert(dest, cache=(test_path / choice))
|
|
54
|
+
|
|
55
|
+
gdf = gpd.read_parquet(dest)
|
|
56
|
+
assert len(gdf) > 1, "Need at least 2 rows to verify ordering."
|
|
57
|
+
|
|
58
|
+
total_bounds = crs_total_bounds(gdf.crs)
|
|
59
|
+
bounds = gdf.geometry.bounds.to_numpy(dtype=np.float64, copy=False)
|
|
60
|
+
keys = hilbert_distances_from_bounds(bounds, total_bounds)
|
|
61
|
+
|
|
62
|
+
# NB: keys are uint64 — np.diff would wrap descents into huge positives
|
|
63
|
+
# and never flag them; compare adjacent keys directly instead.
|
|
64
|
+
descents = keys[1:] < keys[:-1]
|
|
65
|
+
if np.any(descents):
|
|
66
|
+
bad = int(np.argmax(descents))
|
|
67
|
+
raise AssertionError(
|
|
68
|
+
"Output rows are not in non-decreasing Hilbert order against the "
|
|
69
|
+
f"CRS total bounds ({total_bounds}).\n"
|
|
70
|
+
f" First out-of-order pair at index {bad}->{bad + 1}: "
|
|
71
|
+
f"key {int(keys[bad])} > {int(keys[bad + 1])}."
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_crs_total_bounds_geographic():
|
|
76
|
+
# Geographic CRS without a more restrictive area_of_use falls back to world.
|
|
77
|
+
assert crs_total_bounds("EPSG:4326") == (-180.0, -90.0, 180.0, 90.0)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_crs_total_bounds_etrs89():
|
|
81
|
+
# EPSG:4258 is geographic with a European area_of_use; the returned bounds
|
|
82
|
+
# must reflect that AoU rather than world bounds (otherwise Hilbert keys
|
|
83
|
+
# for European data would all be quantised into one tiny cell).
|
|
84
|
+
bounds = crs_total_bounds("EPSG:4258")
|
|
85
|
+
assert bounds[0] > -180.0 and bounds[2] < 180.0, bounds
|
|
86
|
+
assert bounds[1] > -90.0 and bounds[3] < 90.0, bounds
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_hilbert_distances_match_public_geopandas_api():
|
|
90
|
+
# hilbert_distances_from_bounds exists for callers that hold bbox columns
|
|
91
|
+
# without decoded geometry; it must stay identical to the public
|
|
92
|
+
# GeoSeries.hilbert_distance, which guards its private-API internals.
|
|
93
|
+
from shapely.geometry import box
|
|
94
|
+
|
|
95
|
+
rng = np.random.default_rng(42)
|
|
96
|
+
xs = rng.uniform(4.0, 6.0, 50)
|
|
97
|
+
ys = rng.uniform(51.0, 53.0, 50)
|
|
98
|
+
geoms = [box(x, y, x + 0.01, y + 0.01) for x, y in zip(xs, ys)]
|
|
99
|
+
gdf = gpd.GeoDataFrame(geometry=geoms, crs="EPSG:4326")
|
|
100
|
+
total = crs_total_bounds(gdf.crs)
|
|
101
|
+
public = gdf.geometry.hilbert_distance(total_bounds=list(total), level=16).to_numpy()
|
|
102
|
+
ours = hilbert_distances_from_bounds(
|
|
103
|
+
gdf.geometry.bounds.to_numpy(dtype=np.float64), total, level=16
|
|
104
|
+
)
|
|
105
|
+
assert np.array_equal(public.astype(np.uint64), ours)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_crs_total_bounds_antimeridian():
|
|
109
|
+
# NZGD2000 / NZTM: the area of use spans the antimeridian (west > east);
|
|
110
|
+
# the projected bounds must still be finite and non-degenerate.
|
|
111
|
+
xmin, ymin, xmax, ymax = crs_total_bounds("EPSG:2193")
|
|
112
|
+
assert xmax > xmin and ymax > ymin
|
|
113
|
+
assert all(np.isfinite(v) for v in (xmin, ymin, xmax, ymax))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_hilbert_sort_falls_back_without_area_of_use():
|
|
117
|
+
# A custom projected CRS without area_of_use must not fail the conversion;
|
|
118
|
+
# the sorter falls back to the dataset's own bounds.
|
|
119
|
+
from shapely.geometry import Point
|
|
120
|
+
|
|
121
|
+
from vecorel_cli.vecorel.hilbert import hilbert_sort_geodataframe
|
|
122
|
+
|
|
123
|
+
crs = "+proj=tmerc +lat_0=0 +lon_0=9 +k=1 +x_0=0 +y_0=0 +ellps=WGS84 +units=m +no_defs"
|
|
124
|
+
gdf = gpd.GeoDataFrame(geometry=[Point(0, 0), Point(1000, 1000), Point(10, 10)], crs=crs)
|
|
125
|
+
out = hilbert_sort_geodataframe(gdf)
|
|
126
|
+
assert len(out) == 3
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_not_existing_converter(tmp_folder):
|
|
130
|
+
with pytest.raises(Exception, match="Converter 'not_existing' not found"):
|
|
131
|
+
converter = ConvertData("not_existing")
|
|
132
|
+
converter.convert(tmp_folder / "converted.parquet")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@pytest.mark.parametrize("choice", ["invalid_syntax", "invalid_name"])
|
|
136
|
+
def test_invalid_converter(tmp_folder, choice):
|
|
137
|
+
with pytest.raises(Exception, match=f"Converter for '{choice}' not available or faulty:"):
|
|
138
|
+
converter = ConvertData(choice)
|
|
139
|
+
converter.convert(tmp_folder / "converted.parquet")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_template_from_package_folder():
|
|
143
|
+
Registry.src_package = "vecorel_cli"
|
|
144
|
+
Registry.ignored_datasets = []
|
|
145
|
+
converter = ConvertData("template")
|
|
146
|
+
assert isinstance(converter, ConvertData), "Should succeed and not throw an exception"
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_data_access_exception(capsys, tmp_folder):
|
|
150
|
+
# todo: use fixture
|
|
151
|
+
logger.remove()
|
|
152
|
+
logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False)
|
|
153
|
+
|
|
154
|
+
with pytest.raises(Exception, match="Please provide the input data."):
|
|
155
|
+
converter = ConvertData("data_access")
|
|
156
|
+
converter.convert(tmp_folder / "converted.parquet")
|
|
157
|
+
|
|
158
|
+
out, err = capsys.readouterr()
|
|
159
|
+
|
|
160
|
+
assert isinstance(converter.converter, BaseConverter)
|
|
161
|
+
assert converter.converter.data_access in out
|
|
@@ -40,3 +40,21 @@ def test_create_geoparquet_invalid_file(tmp_folder):
|
|
|
40
40
|
gp.create("invalid.parquet", out)
|
|
41
41
|
with pytest.raises(FileNotFoundError):
|
|
42
42
|
gp.create(["invalid.json"], out)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_create_geoparquet_collection_only_properties(tmp_parquet_file: Path):
|
|
46
|
+
# https://github.com/vecorel/cli/issues/26
|
|
47
|
+
inputs = ["tests/data-files/collection-only.json"]
|
|
48
|
+
creator = CreateGeoParquet()
|
|
49
|
+
creator.create(inputs, tmp_parquet_file)
|
|
50
|
+
assert tmp_parquet_file.exists()
|
|
51
|
+
|
|
52
|
+
gp = GeoParquet(tmp_parquet_file)
|
|
53
|
+
|
|
54
|
+
collection = gp.get_collection()
|
|
55
|
+
assert collection.get("source_name") == "Example Source"
|
|
56
|
+
|
|
57
|
+
data = gp.read()
|
|
58
|
+
assert len(data) == 2
|
|
59
|
+
# collection-only properties must not be added as columns
|
|
60
|
+
assert "source_name" not in data.columns
|
|
@@ -55,3 +55,40 @@ def test_merge_collections():
|
|
|
55
55
|
},
|
|
56
56
|
}
|
|
57
57
|
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_merge_collections_keeps_collection_properties():
|
|
61
|
+
# https://github.com/vecorel/cli/issues/26
|
|
62
|
+
schemas = {"c1": ["https://example.com/schema1.yaml"]}
|
|
63
|
+
collection1 = Collection(
|
|
64
|
+
{
|
|
65
|
+
"schemas": schemas,
|
|
66
|
+
"source_name": "Example",
|
|
67
|
+
"source_version": "1.0",
|
|
68
|
+
"only_in_first": "x",
|
|
69
|
+
}
|
|
70
|
+
)
|
|
71
|
+
collection2 = Collection(
|
|
72
|
+
{
|
|
73
|
+
"schemas": schemas,
|
|
74
|
+
"source_name": "Example",
|
|
75
|
+
"source_version": "2.0",
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
merged = merge_collections([collection1])
|
|
80
|
+
assert merged.get("source_name") == "Example"
|
|
81
|
+
assert merged.get("source_version") == "1.0"
|
|
82
|
+
assert merged.get("only_in_first") == "x"
|
|
83
|
+
|
|
84
|
+
merged = merge_collections([collection1, collection2])
|
|
85
|
+
# present in all collections with the same value => kept
|
|
86
|
+
assert merged.get("source_name") == "Example"
|
|
87
|
+
# conflicting values => dropped
|
|
88
|
+
assert "source_version" not in merged
|
|
89
|
+
# missing in one collection => dropped
|
|
90
|
+
assert "only_in_first" not in merged
|
|
91
|
+
|
|
92
|
+
merged = merge_collections([collection1], properties=["only_in_first"])
|
|
93
|
+
assert "source_name" not in merged
|
|
94
|
+
assert merged.get("only_in_first") == "x"
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
+
from click.testing import CliRunner
|
|
1
2
|
from jsonschema.exceptions import ValidationError
|
|
2
3
|
|
|
3
4
|
from vecorel_cli.validate_schema import ValidateSchema
|
|
4
5
|
|
|
5
6
|
|
|
7
|
+
def get_cli_command():
|
|
8
|
+
cmd = ValidateSchema.get_cli_command(ValidateSchema)
|
|
9
|
+
for arg in ValidateSchema.get_cli_args().values():
|
|
10
|
+
cmd = arg(cmd)
|
|
11
|
+
return cmd
|
|
12
|
+
|
|
13
|
+
|
|
6
14
|
def test_validate_schema_valid():
|
|
7
15
|
path = "tests/data-files/sdl/inspire-schema.yaml"
|
|
8
16
|
cmd = ValidateSchema()
|
|
@@ -20,3 +28,15 @@ def test_validate_schema_invalid():
|
|
|
20
28
|
error = result[0]
|
|
21
29
|
assert isinstance(error, ValidationError)
|
|
22
30
|
assert error.message.startswith("'STRING' is not one of [")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_validate_schema_cli_valid_exit_code():
|
|
34
|
+
result = CliRunner().invoke(get_cli_command(), ["tests/data-files/sdl/inspire-schema.yaml"])
|
|
35
|
+
assert result.exit_code == 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_validate_schema_cli_invalid_exit_code():
|
|
39
|
+
result = CliRunner().invoke(
|
|
40
|
+
get_cli_command(), ["tests/data-files/sdl/inspire-schema-invalid.yaml"]
|
|
41
|
+
)
|
|
42
|
+
assert result.exit_code != 0
|
|
@@ -61,6 +61,10 @@ class BaseCommand(LoggerMixin):
|
|
|
61
61
|
self.exception(e)
|
|
62
62
|
sys.exit(1)
|
|
63
63
|
|
|
64
|
+
# Exit with a non-zero exit code if the runnable reported a failure
|
|
65
|
+
if result is False:
|
|
66
|
+
sys.exit(1)
|
|
67
|
+
|
|
64
68
|
# Report command as finished
|
|
65
69
|
if self.cmd_final_report:
|
|
66
70
|
if isinstance(result, Path):
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
SUPPORTED_PROTOCOLS = ["http", "https", "s3", "gs"]
|
|
2
|
+
|
|
3
|
+
COMPRESSION_METHODS = ["brotli", "gzip", "lz4", "snappy", "zstd", "none"]
|
|
4
|
+
|
|
5
|
+
GEOPARQUET_VERSIONS = ["1.0.0", "1.1.0"]
|
|
6
|
+
GEOPARQUET_DEFAULT_VERSION = "1.1.0"
|
|
7
|
+
|
|
8
|
+
# Default level used when sorting features along a Hilbert curve. A level of N
|
|
9
|
+
# splits each axis of the reference bounds into 2**N cells; 16 gives a
|
|
10
|
+
# 65,536 x 65,536 grid, fine enough for world-scale references at sub-km
|
|
11
|
+
# resolution near the equator.
|
|
12
|
+
HILBERT_DEFAULT_LEVEL = 16
|
|
@@ -26,6 +26,7 @@ from ..cli.logger import LoggerMixin
|
|
|
26
26
|
from ..cli.util import display_pandas_unrestricted
|
|
27
27
|
from ..encoding.geoparquet import GeoParquet
|
|
28
28
|
from ..vecorel.collection import Collection
|
|
29
|
+
from ..vecorel.hilbert import hilbert_sort_geodataframe
|
|
29
30
|
from ..vecorel.schemas import Schemas
|
|
30
31
|
from ..vecorel.typing import Sources
|
|
31
32
|
from ..vecorel.util import get_fs, name_from_uri, stream_file
|
|
@@ -408,7 +409,12 @@ class BaseConverter(LoggerMixin):
|
|
|
408
409
|
self.info("Removing Z geometry dimension")
|
|
409
410
|
gdf.geometry = gdf.geometry.force_2d()
|
|
410
411
|
|
|
411
|
-
|
|
412
|
+
# Sort by Hilbert distance against the CRS's total bounds. This gives
|
|
413
|
+
# row groups good spatial locality, and — crucially — produces the same
|
|
414
|
+
# ordering for any independently-converted partition of the same dataset
|
|
415
|
+
# (they share a CRS, therefore a Hilbert reference grid), so per-file
|
|
416
|
+
# outputs can be merged later without re-sorting.
|
|
417
|
+
gdf = hilbert_sort_geodataframe(gdf)
|
|
412
418
|
|
|
413
419
|
# 8. Remove all columns that are not listed
|
|
414
420
|
drop_columns = list(set(gdf.columns) - set(actual_columns.values()))
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
1
|
"""
|
|
4
2
|
FlatDict is a dict object that allows for single level, delimited
|
|
5
3
|
key/value pair mapping of nested dictionaries.
|
|
@@ -11,6 +9,7 @@ Author: Gavin M. Roy (@gmr)
|
|
|
11
9
|
|
|
12
10
|
Code slightly adapted for linting purposes.
|
|
13
11
|
"""
|
|
12
|
+
|
|
14
13
|
try:
|
|
15
14
|
from collections.abc import MutableMapping
|
|
16
15
|
except ImportError: # pragma: nocover
|
|
@@ -27,9 +26,10 @@ class FlatDict(MutableMapping):
|
|
|
27
26
|
or by calling :meth:`FlatDict.set_delimiter`.
|
|
28
27
|
|
|
29
28
|
"""
|
|
29
|
+
|
|
30
30
|
_COERCE = dict
|
|
31
31
|
|
|
32
|
-
def __init__(self, value=None, delimiter=
|
|
32
|
+
def __init__(self, value=None, delimiter=":", dict_class=dict):
|
|
33
33
|
super(FlatDict, self).__init__()
|
|
34
34
|
self._values = dict_class()
|
|
35
35
|
self._delimiter = delimiter
|
|
@@ -136,8 +136,7 @@ class FlatDict(MutableMapping):
|
|
|
136
136
|
:rtype: str
|
|
137
137
|
|
|
138
138
|
"""
|
|
139
|
-
return '<{} id={} {}>"'.format(self.__class__.__name__, id(self),
|
|
140
|
-
str(self))
|
|
139
|
+
return '<{} id={} {}>"'.format(self.__class__.__name__, id(self), str(self))
|
|
141
140
|
|
|
142
141
|
def __setitem__(self, key, value):
|
|
143
142
|
"""Assign the value to the key, dynamically building nested
|
|
@@ -156,8 +155,7 @@ class FlatDict(MutableMapping):
|
|
|
156
155
|
self._values[pk] = self.__class__({ck: value}, self._delimiter)
|
|
157
156
|
return
|
|
158
157
|
elif not isinstance(self._values[pk], FlatDict):
|
|
159
|
-
raise TypeError(
|
|
160
|
-
'Assignment to invalid type for key {}'.format(pk))
|
|
158
|
+
raise TypeError("Assignment to invalid type for key {}".format(pk))
|
|
161
159
|
self._values[pk][ck] = value
|
|
162
160
|
else:
|
|
163
161
|
self._values[key] = value
|
|
@@ -168,8 +166,7 @@ class FlatDict(MutableMapping):
|
|
|
168
166
|
:rtype: str
|
|
169
167
|
|
|
170
168
|
"""
|
|
171
|
-
return
|
|
172
|
-
['{!r}: {!r}'.format(k, self[k]) for k in self.keys()]))
|
|
169
|
+
return "{{{}}}".format(", ".join(["{!r}: {!r}".format(k, self[k]) for k in self.keys()]))
|
|
173
170
|
|
|
174
171
|
def as_dict(self):
|
|
175
172
|
"""Return the :class:`~flatdict.FlatDict` as a :class:`dict`
|
|
@@ -290,9 +287,7 @@ class FlatDict(MutableMapping):
|
|
|
290
287
|
|
|
291
288
|
for key, value in self._values.items():
|
|
292
289
|
if isinstance(value, (FlatDict, dict)):
|
|
293
|
-
nested = [
|
|
294
|
-
self._delimiter.join([str(key), str(k)])
|
|
295
|
-
for k in value.keys()]
|
|
290
|
+
nested = [self._delimiter.join([str(key), str(k)]) for k in value.keys()]
|
|
296
291
|
keys += nested if nested else [key]
|
|
297
292
|
else:
|
|
298
293
|
keys.append(key)
|
|
@@ -340,8 +335,7 @@ class FlatDict(MutableMapping):
|
|
|
340
335
|
"""
|
|
341
336
|
for key in self.keys():
|
|
342
337
|
if delimiter in key:
|
|
343
|
-
raise ValueError(
|
|
344
|
-
delimiter)
|
|
338
|
+
raise ValueError("Key {!r} collides with delimiter {!r}", key, delimiter)
|
|
345
339
|
self._delimiter = delimiter
|
|
346
340
|
for key in self._values.keys():
|
|
347
341
|
if isinstance(self._values[key], FlatDict):
|
|
@@ -382,14 +376,15 @@ class FlatDict(MutableMapping):
|
|
|
382
376
|
|
|
383
377
|
class FlatterDict(FlatDict):
|
|
384
378
|
"""Like :class:`~flatdict.FlatDict` but also coerces lists and sets
|
|
385
|
-
|
|
386
|
-
|
|
379
|
+
to child-dict instances with the offset as the key. Alternative to
|
|
380
|
+
the implementation added in v1.2 of FlatDict.
|
|
387
381
|
|
|
388
382
|
"""
|
|
383
|
+
|
|
389
384
|
_COERCE = list, tuple, set, dict, FlatDict
|
|
390
385
|
_ARRAYS = list, set, tuple
|
|
391
386
|
|
|
392
|
-
def __init__(self, value=None, delimiter=
|
|
387
|
+
def __init__(self, value=None, delimiter=":", dict_class=dict):
|
|
393
388
|
self.original_type = type(value)
|
|
394
389
|
if self.original_type in self._ARRAYS:
|
|
395
390
|
value = {str(i): v for i, v in enumerate(value)}
|
|
@@ -404,28 +399,25 @@ class FlatterDict(FlatDict):
|
|
|
404
399
|
:raises: TypeError
|
|
405
400
|
|
|
406
401
|
"""
|
|
407
|
-
if isinstance(value, self._COERCE) and
|
|
408
|
-
not isinstance(value, FlatterDict):
|
|
402
|
+
if isinstance(value, self._COERCE) and not isinstance(value, FlatterDict):
|
|
409
403
|
value = self.__class__(value, self._delimiter)
|
|
410
404
|
if self._has_delimiter(key):
|
|
411
405
|
pk, ck = key.split(self._delimiter, 1)
|
|
412
406
|
if pk not in self._values:
|
|
413
407
|
self._values[pk] = self.__class__({ck: value}, self._delimiter)
|
|
414
408
|
return
|
|
415
|
-
if getattr(self._values[pk],
|
|
416
|
-
None) in self._ARRAYS:
|
|
409
|
+
if getattr(self._values[pk], "original_type", None) in self._ARRAYS:
|
|
417
410
|
try:
|
|
418
411
|
k, cck = ck.split(self._delimiter, 1)
|
|
419
412
|
int(k)
|
|
420
413
|
except ValueError:
|
|
421
414
|
raise TypeError(
|
|
422
|
-
|
|
423
|
-
|
|
415
|
+
"Assignment to invalid type for key {}{}{}".format(pk, self._delimiter, ck)
|
|
416
|
+
)
|
|
424
417
|
self._values[pk][k][cck] = value
|
|
425
418
|
return
|
|
426
419
|
elif not isinstance(self._values[pk], FlatterDict):
|
|
427
|
-
raise TypeError(
|
|
428
|
-
'Assignment to invalid type for key {}'.format(pk))
|
|
420
|
+
raise TypeError("Assignment to invalid type for key {}".format(pk))
|
|
429
421
|
self._values[pk][ck] = value
|
|
430
422
|
else:
|
|
431
423
|
self._values[key] = value
|
|
@@ -212,7 +212,7 @@ class CreateStacCollection(BaseCommand):
|
|
|
212
212
|
)
|
|
213
213
|
|
|
214
214
|
# Add provider information
|
|
215
|
-
provider = collection.get("provider"
|
|
215
|
+
provider = (collection.get("provider") or "").strip()
|
|
216
216
|
if len(provider) > 0:
|
|
217
217
|
provider_name, provider_url = parse_link_str(provider)
|
|
218
218
|
stac["providers"] = [
|
|
@@ -2,6 +2,8 @@ import importlib.metadata
|
|
|
2
2
|
import re
|
|
3
3
|
from typing import Union
|
|
4
4
|
|
|
5
|
+
from .vecorel.schemas import Schemas
|
|
6
|
+
|
|
5
7
|
|
|
6
8
|
class VecorelRegistry:
|
|
7
9
|
"""
|
|
@@ -77,6 +79,12 @@ class VecorelRegistry:
|
|
|
77
79
|
|
|
78
80
|
return extensions
|
|
79
81
|
|
|
82
|
+
def get_default_collection(self, id: str, extensions: set | list | None = None) -> dict:
|
|
83
|
+
schemas = {Schemas.get_core_uri()}
|
|
84
|
+
if extensions:
|
|
85
|
+
schemas |= set(extensions)
|
|
86
|
+
return {"schemas": {id: list(schemas)}, "collection": id}
|
|
87
|
+
|
|
80
88
|
def remove_command(self, command):
|
|
81
89
|
from .basecommand import BaseCommand
|
|
82
90
|
|