pyogrio 0.9.0__cp39-cp39-manylinux_2_28_aarch64.whl → 0.11.0__cp39-cp39-manylinux_2_28_aarch64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyogrio might be problematic. Click here for more details.
- pyogrio/__init__.py +28 -21
- pyogrio/_compat.py +15 -1
- pyogrio/_env.py +4 -6
- pyogrio/_err.cpython-39-aarch64-linux-gnu.so +0 -0
- pyogrio/_geometry.cpython-39-aarch64-linux-gnu.so +0 -0
- pyogrio/_io.cpython-39-aarch64-linux-gnu.so +0 -0
- pyogrio/_ogr.cpython-39-aarch64-linux-gnu.so +0 -0
- pyogrio/_version.py +3 -3
- pyogrio/_vsi.cpython-39-aarch64-linux-gnu.so +0 -0
- pyogrio/core.py +86 -20
- pyogrio/errors.py +9 -16
- pyogrio/gdal_data/GDAL-targets-release.cmake +3 -3
- pyogrio/gdal_data/GDAL-targets.cmake +2 -2
- pyogrio/gdal_data/GDALConfig.cmake +0 -1
- pyogrio/gdal_data/GDALConfigVersion.cmake +3 -3
- pyogrio/gdal_data/MM_m_idofic.csv +321 -0
- pyogrio/gdal_data/gdalinfo_output.schema.json +3 -3
- pyogrio/gdal_data/gdaltileindex.xsd +253 -0
- pyogrio/gdal_data/gdalvrt.xsd +178 -63
- pyogrio/gdal_data/nitf_spec.xml +1 -17
- pyogrio/gdal_data/nitf_spec.xsd +1 -17
- pyogrio/gdal_data/ogrinfo_output.schema.json +23 -0
- pyogrio/gdal_data/ogrvrt.xsd +4 -17
- pyogrio/gdal_data/osmconf.ini +3 -1
- pyogrio/gdal_data/pci_datum.txt +222 -155
- pyogrio/gdal_data/pci_ellips.txt +90 -38
- pyogrio/gdal_data/pdfcomposition.xsd +1 -17
- pyogrio/gdal_data/vcpkg.spdx.json +32 -27
- pyogrio/gdal_data/vcpkg_abi_info.txt +30 -29
- pyogrio/gdal_data/vdv452.xml +1 -17
- pyogrio/gdal_data/vdv452.xsd +1 -17
- pyogrio/geopandas.py +122 -66
- pyogrio/proj_data/ITRF2014 +1 -1
- pyogrio/proj_data/ITRF2020 +91 -0
- pyogrio/proj_data/proj-config-version.cmake +2 -2
- pyogrio/proj_data/proj-config.cmake +1 -1
- pyogrio/proj_data/proj-targets.cmake +3 -3
- pyogrio/proj_data/proj.db +0 -0
- pyogrio/proj_data/proj.ini +11 -3
- pyogrio/proj_data/proj4-targets.cmake +3 -3
- pyogrio/proj_data/projjson.schema.json +1 -1
- pyogrio/proj_data/usage +7 -2
- pyogrio/proj_data/vcpkg.spdx.json +27 -22
- pyogrio/proj_data/vcpkg_abi_info.txt +17 -16
- pyogrio/raw.py +46 -30
- pyogrio/tests/conftest.py +214 -12
- pyogrio/tests/fixtures/README.md +32 -13
- pyogrio/tests/fixtures/curve.gpkg +0 -0
- pyogrio/tests/fixtures/{test_multisurface.gpkg → curvepolygon.gpkg} +0 -0
- pyogrio/tests/fixtures/line_zm.gpkg +0 -0
- pyogrio/tests/fixtures/multisurface.gpkg +0 -0
- pyogrio/tests/test_arrow.py +181 -24
- pyogrio/tests/test_core.py +170 -76
- pyogrio/tests/test_geopandas_io.py +483 -135
- pyogrio/tests/test_path.py +39 -17
- pyogrio/tests/test_raw_io.py +170 -55
- pyogrio/tests/test_util.py +56 -0
- pyogrio/util.py +69 -32
- pyogrio-0.11.0.dist-info/METADATA +124 -0
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info}/RECORD +200 -214
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info}/WHEEL +1 -1
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info/licenses}/LICENSE +1 -1
- pyogrio.libs/{libgdal-6ff0914e.so.34.3.8.5 → libgdal-4bc0d15f.so.36.3.10.3} +0 -0
- pyogrio/_err.pxd +0 -4
- pyogrio/_err.pyx +0 -250
- pyogrio/_geometry.pxd +0 -4
- pyogrio/_geometry.pyx +0 -129
- pyogrio/_io.pxd +0 -0
- pyogrio/_io.pyx +0 -2742
- pyogrio/_ogr.pxd +0 -444
- pyogrio/_ogr.pyx +0 -346
- pyogrio/_vsi.pxd +0 -4
- pyogrio/_vsi.pyx +0 -140
- pyogrio/arrow_bridge.h +0 -115
- pyogrio/gdal_data/bag_template.xml +0 -201
- pyogrio/gdal_data/gmlasconf.xml +0 -169
- pyogrio/gdal_data/gmlasconf.xsd +0 -1066
- pyogrio/gdal_data/netcdf_config.xsd +0 -143
- pyogrio/tests/fixtures/poly_not_enough_points.shp.zip +0 -0
- pyogrio/tests/fixtures/test_datetime.geojson +0 -7
- pyogrio/tests/fixtures/test_datetime_tz.geojson +0 -8
- pyogrio/tests/fixtures/test_fgdb.gdb.zip +0 -0
- pyogrio/tests/fixtures/test_nested.geojson +0 -18
- pyogrio/tests/fixtures/test_ogr_types_list.geojson +0 -12
- pyogrio-0.9.0.dist-info/METADATA +0 -100
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info}/top_level.txt +0 -0
pyogrio/tests/test_path.py
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
|
+
import contextlib
|
|
1
2
|
import os
|
|
2
3
|
from pathlib import Path
|
|
3
|
-
import
|
|
4
|
-
from zipfile import ZipFile, ZIP_DEFLATED
|
|
5
|
-
|
|
6
|
-
import pytest
|
|
4
|
+
from zipfile import ZIP_DEFLATED, ZipFile
|
|
7
5
|
|
|
8
6
|
import pyogrio
|
|
9
7
|
import pyogrio.raw
|
|
10
|
-
from pyogrio.
|
|
8
|
+
from pyogrio._compat import HAS_PYPROJ
|
|
9
|
+
from pyogrio.util import get_vsi_path_or_buffer, vsi_path
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
11
12
|
|
|
12
13
|
try:
|
|
13
|
-
import geopandas #
|
|
14
|
+
import geopandas # noqa: F401
|
|
14
15
|
|
|
15
16
|
has_geopandas = True
|
|
16
17
|
except ImportError:
|
|
@@ -32,9 +33,20 @@ def change_cwd(path):
|
|
|
32
33
|
[
|
|
33
34
|
# local file paths that should be passed through as is
|
|
34
35
|
("data.gpkg", "data.gpkg"),
|
|
36
|
+
("data.gpkg.zip", "data.gpkg.zip"),
|
|
37
|
+
("data.shp.zip", "data.shp.zip"),
|
|
38
|
+
(Path("data.gpkg"), "data.gpkg"),
|
|
39
|
+
(Path("data.gpkg.zip"), "data.gpkg.zip"),
|
|
40
|
+
(Path("data.shp.zip"), "data.shp.zip"),
|
|
35
41
|
("/home/user/data.gpkg", "/home/user/data.gpkg"),
|
|
42
|
+
("/home/user/data.gpkg.zip", "/home/user/data.gpkg.zip"),
|
|
43
|
+
("/home/user/data.shp.zip", "/home/user/data.shp.zip"),
|
|
36
44
|
(r"C:\User\Documents\data.gpkg", r"C:\User\Documents\data.gpkg"),
|
|
45
|
+
(r"C:\User\Documents\data.gpkg.zip", r"C:\User\Documents\data.gpkg.zip"),
|
|
46
|
+
(r"C:\User\Documents\data.shp.zip", r"C:\User\Documents\data.shp.zip"),
|
|
37
47
|
("file:///home/user/data.gpkg", "/home/user/data.gpkg"),
|
|
48
|
+
("file:///home/user/data.gpkg.zip", "/home/user/data.gpkg.zip"),
|
|
49
|
+
("file:///home/user/data.shp.zip", "/home/user/data.shp.zip"),
|
|
38
50
|
("/home/folder # with hash/data.gpkg", "/home/folder # with hash/data.gpkg"),
|
|
39
51
|
# cloud URIs
|
|
40
52
|
("https://testing/data.gpkg", "/vsicurl/https://testing/data.gpkg"),
|
|
@@ -84,6 +96,8 @@ def change_cwd(path):
|
|
|
84
96
|
"s3://testing/test.zip!a/b/item.shp",
|
|
85
97
|
"/vsizip/vsis3/testing/test.zip/a/b/item.shp",
|
|
86
98
|
),
|
|
99
|
+
("/vsimem/data.gpkg", "/vsimem/data.gpkg"),
|
|
100
|
+
(Path("/vsimem/data.gpkg"), "/vsimem/data.gpkg"),
|
|
87
101
|
],
|
|
88
102
|
)
|
|
89
103
|
def test_vsi_path(path, expected):
|
|
@@ -238,6 +252,9 @@ def test_detect_zip_path(tmp_path, naturalearth_lowres):
|
|
|
238
252
|
path = tmp_path / "test.zip"
|
|
239
253
|
with ZipFile(path, mode="w", compression=ZIP_DEFLATED, compresslevel=5) as out:
|
|
240
254
|
for ext in ["dbf", "prj", "shp", "shx"]:
|
|
255
|
+
if not HAS_PYPROJ and ext == "prj":
|
|
256
|
+
continue
|
|
257
|
+
|
|
241
258
|
filename = f"test1.{ext}"
|
|
242
259
|
out.write(tmp_path / filename, filename)
|
|
243
260
|
|
|
@@ -279,6 +296,7 @@ def test_url():
|
|
|
279
296
|
assert len(result[0]) == 177
|
|
280
297
|
|
|
281
298
|
|
|
299
|
+
@pytest.mark.network
|
|
282
300
|
@pytest.mark.skipif(not has_geopandas, reason="GeoPandas not available")
|
|
283
301
|
def test_url_dataframe():
|
|
284
302
|
url = "https://raw.githubusercontent.com/geopandas/pyogrio/main/pyogrio/tests/fixtures/naturalearth_lowres/naturalearth_lowres.shp"
|
|
@@ -334,19 +352,23 @@ def test_uri_s3_dataframe(aws_env_setup):
|
|
|
334
352
|
assert len(df) == 67
|
|
335
353
|
|
|
336
354
|
|
|
337
|
-
|
|
338
|
-
path
|
|
339
|
-
|
|
355
|
+
@pytest.mark.parametrize(
|
|
356
|
+
"path, expected",
|
|
357
|
+
[
|
|
358
|
+
(Path("/tmp/test.gpkg"), str(Path("/tmp/test.gpkg"))),
|
|
359
|
+
(Path("/vsimem/test.gpkg"), "/vsimem/test.gpkg"),
|
|
360
|
+
],
|
|
361
|
+
)
|
|
362
|
+
def test_get_vsi_path_or_buffer_obj_to_string(path, expected):
|
|
363
|
+
"""Verify that get_vsi_path_or_buffer retains forward slashes in /vsimem paths.
|
|
364
|
+
|
|
365
|
+
The /vsimem paths should keep forward slashes for GDAL to recognize them as such.
|
|
366
|
+
However, on Windows systems, forward slashes are by default replaced by backslashes,
|
|
367
|
+
so this test verifies that this doesn't happen for /vsimem paths.
|
|
368
|
+
"""
|
|
369
|
+
assert get_vsi_path_or_buffer(path) == expected
|
|
340
370
|
|
|
341
371
|
|
|
342
372
|
def test_get_vsi_path_or_buffer_fixtures_to_string(tmp_path):
|
|
343
373
|
path = tmp_path / "test.gpkg"
|
|
344
374
|
assert get_vsi_path_or_buffer(path) == str(path)
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
@pytest.mark.parametrize(
|
|
348
|
-
"raw_path", ["/vsimem/test.shp.zip", "/vsizip//vsimem/test.shp.zip"]
|
|
349
|
-
)
|
|
350
|
-
def test_vsimem_path_exception(raw_path):
|
|
351
|
-
with pytest.raises(ValueError, match=""):
|
|
352
|
-
vsi_path(raw_path)
|
pyogrio/tests/test_raw_io.py
CHANGED
|
@@ -1,33 +1,36 @@
|
|
|
1
1
|
import contextlib
|
|
2
2
|
import ctypes
|
|
3
|
-
from io import BytesIO
|
|
4
3
|
import json
|
|
5
4
|
import sys
|
|
5
|
+
from io import BytesIO
|
|
6
|
+
from zipfile import ZipFile
|
|
6
7
|
|
|
7
8
|
import numpy as np
|
|
8
9
|
from numpy import array_equal
|
|
9
|
-
import pytest
|
|
10
10
|
|
|
11
11
|
import pyogrio
|
|
12
12
|
from pyogrio import (
|
|
13
|
-
|
|
13
|
+
__gdal_version__,
|
|
14
|
+
get_gdal_config_option,
|
|
14
15
|
list_drivers,
|
|
16
|
+
list_layers,
|
|
15
17
|
read_info,
|
|
16
18
|
set_gdal_config_options,
|
|
17
|
-
get_gdal_config_option,
|
|
18
|
-
__gdal_version__,
|
|
19
19
|
)
|
|
20
|
-
from pyogrio._compat import
|
|
21
|
-
from pyogrio.
|
|
22
|
-
from pyogrio.
|
|
20
|
+
from pyogrio._compat import GDAL_GE_37, HAS_PYARROW, HAS_SHAPELY
|
|
21
|
+
from pyogrio.errors import DataLayerError, DataSourceError, FeatureError
|
|
22
|
+
from pyogrio.raw import open_arrow, read, write
|
|
23
23
|
from pyogrio.tests.conftest import (
|
|
24
|
-
DRIVERS,
|
|
25
24
|
DRIVER_EXT,
|
|
25
|
+
DRIVERS,
|
|
26
26
|
prepare_testfile,
|
|
27
|
-
requires_pyarrow_api,
|
|
28
27
|
requires_arrow_api,
|
|
28
|
+
requires_pyarrow_api,
|
|
29
|
+
requires_shapely,
|
|
29
30
|
)
|
|
30
31
|
|
|
32
|
+
import pytest
|
|
33
|
+
|
|
31
34
|
try:
|
|
32
35
|
import shapely
|
|
33
36
|
except ImportError:
|
|
@@ -60,9 +63,10 @@ def test_read(naturalearth_lowres):
|
|
|
60
63
|
@pytest.mark.parametrize("ext", DRIVERS)
|
|
61
64
|
def test_read_autodetect_driver(tmp_path, naturalearth_lowres, ext):
|
|
62
65
|
# Test all supported autodetect drivers
|
|
66
|
+
if ext == ".gpkg.zip" and not GDAL_GE_37:
|
|
67
|
+
pytest.skip(".gpkg.zip not supported for gdal < 3.7.0")
|
|
63
68
|
testfile = prepare_testfile(naturalearth_lowres, dst_dir=tmp_path, ext=ext)
|
|
64
69
|
|
|
65
|
-
assert testfile.suffix == ext
|
|
66
70
|
assert testfile.exists()
|
|
67
71
|
meta, _, geometry, fields = read(testfile)
|
|
68
72
|
|
|
@@ -116,6 +120,29 @@ def test_read_no_geometry(naturalearth_lowres):
|
|
|
116
120
|
assert geometry is None
|
|
117
121
|
|
|
118
122
|
|
|
123
|
+
@requires_shapely
|
|
124
|
+
def test_read_no_geometry__mask(naturalearth_lowres):
|
|
125
|
+
geometry, fields = read(
|
|
126
|
+
naturalearth_lowres,
|
|
127
|
+
read_geometry=False,
|
|
128
|
+
mask=shapely.Point(-105, 55),
|
|
129
|
+
)[2:]
|
|
130
|
+
|
|
131
|
+
assert np.array_equal(fields[3], ["CAN"])
|
|
132
|
+
assert geometry is None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def test_read_no_geometry__bbox(naturalearth_lowres):
|
|
136
|
+
geometry, fields = read(
|
|
137
|
+
naturalearth_lowres,
|
|
138
|
+
read_geometry=False,
|
|
139
|
+
bbox=(-109.0, 55.0, -109.0, 55.0),
|
|
140
|
+
)[2:]
|
|
141
|
+
|
|
142
|
+
assert np.array_equal(fields[3], ["CAN"])
|
|
143
|
+
assert geometry is None
|
|
144
|
+
|
|
145
|
+
|
|
119
146
|
def test_read_no_geometry_no_columns_no_fids(naturalearth_lowres):
|
|
120
147
|
with pytest.raises(
|
|
121
148
|
ValueError,
|
|
@@ -255,9 +282,7 @@ def test_read_bbox_where(naturalearth_lowres_all_ext):
|
|
|
255
282
|
assert np.array_equal(fields[3], ["CAN"])
|
|
256
283
|
|
|
257
284
|
|
|
258
|
-
@
|
|
259
|
-
not HAS_SHAPELY, reason="Shapely is required for mask functionality"
|
|
260
|
-
)
|
|
285
|
+
@requires_shapely
|
|
261
286
|
@pytest.mark.parametrize(
|
|
262
287
|
"mask",
|
|
263
288
|
[
|
|
@@ -271,17 +296,13 @@ def test_read_mask_invalid(naturalearth_lowres, mask):
|
|
|
271
296
|
read(naturalearth_lowres, mask=mask)
|
|
272
297
|
|
|
273
298
|
|
|
274
|
-
@
|
|
275
|
-
not HAS_SHAPELY, reason="Shapely is required for mask functionality"
|
|
276
|
-
)
|
|
299
|
+
@requires_shapely
|
|
277
300
|
def test_read_bbox_mask_invalid(naturalearth_lowres):
|
|
278
301
|
with pytest.raises(ValueError, match="cannot set both 'bbox' and 'mask'"):
|
|
279
302
|
read(naturalearth_lowres, bbox=(-85, 8, -80, 10), mask=shapely.Point(-105, 55))
|
|
280
303
|
|
|
281
304
|
|
|
282
|
-
@
|
|
283
|
-
not HAS_SHAPELY, reason="Shapely is required for mask functionality"
|
|
284
|
-
)
|
|
305
|
+
@requires_shapely
|
|
285
306
|
@pytest.mark.parametrize(
|
|
286
307
|
"mask,expected",
|
|
287
308
|
[
|
|
@@ -316,9 +337,7 @@ def test_read_mask(naturalearth_lowres_all_ext, mask, expected):
|
|
|
316
337
|
assert len(geometry) == len(expected)
|
|
317
338
|
|
|
318
339
|
|
|
319
|
-
@
|
|
320
|
-
not HAS_SHAPELY, reason="Shapely is required for mask functionality"
|
|
321
|
-
)
|
|
340
|
+
@requires_shapely
|
|
322
341
|
def test_read_mask_sql(naturalearth_lowres_all_ext):
|
|
323
342
|
fields = read(
|
|
324
343
|
naturalearth_lowres_all_ext,
|
|
@@ -329,9 +348,7 @@ def test_read_mask_sql(naturalearth_lowres_all_ext):
|
|
|
329
348
|
assert np.array_equal(fields[3], ["CAN"])
|
|
330
349
|
|
|
331
350
|
|
|
332
|
-
@
|
|
333
|
-
not HAS_SHAPELY, reason="Shapely is required for mask functionality"
|
|
334
|
-
)
|
|
351
|
+
@requires_shapely
|
|
335
352
|
def test_read_mask_where(naturalearth_lowres_all_ext):
|
|
336
353
|
fields = read(
|
|
337
354
|
naturalearth_lowres_all_ext,
|
|
@@ -603,13 +620,80 @@ def test_write_no_geom_no_fields():
|
|
|
603
620
|
__gdal_version__ < (3, 6, 0),
|
|
604
621
|
reason="OpenFileGDB write support only available for GDAL >= 3.6.0",
|
|
605
622
|
)
|
|
606
|
-
|
|
607
|
-
|
|
623
|
+
@pytest.mark.parametrize(
|
|
624
|
+
"write_int64",
|
|
625
|
+
[
|
|
626
|
+
False,
|
|
627
|
+
pytest.param(
|
|
628
|
+
True,
|
|
629
|
+
marks=pytest.mark.skipif(
|
|
630
|
+
__gdal_version__ < (3, 9, 0),
|
|
631
|
+
reason="OpenFileGDB write support for int64 values for GDAL >= 3.9.0",
|
|
632
|
+
),
|
|
633
|
+
),
|
|
634
|
+
],
|
|
635
|
+
)
|
|
636
|
+
def test_write_openfilegdb(tmp_path, write_int64):
|
|
637
|
+
# Point(0, 0)
|
|
638
|
+
expected_geometry = np.array(
|
|
639
|
+
[bytes.fromhex("010100000000000000000000000000000000000000")] * 3, dtype=object
|
|
640
|
+
)
|
|
641
|
+
expected_field_data = [
|
|
642
|
+
np.array([True, False, True], dtype="bool"),
|
|
643
|
+
np.array([1, 2, 3], dtype="int16"),
|
|
644
|
+
np.array([1, 2, 3], dtype="int32"),
|
|
645
|
+
np.array([1, 2, 3], dtype="int64"),
|
|
646
|
+
np.array([1, 2, 3], dtype="float32"),
|
|
647
|
+
np.array([1, 2, 3], dtype="float64"),
|
|
648
|
+
]
|
|
649
|
+
expected_fields = ["bool", "int16", "int32", "int64", "float32", "float64"]
|
|
650
|
+
expected_meta = {
|
|
651
|
+
"geometry_type": "Point",
|
|
652
|
+
"crs": "EPSG:4326",
|
|
653
|
+
"fields": expected_fields,
|
|
654
|
+
}
|
|
608
655
|
|
|
609
656
|
filename = tmp_path / "test.gdb"
|
|
610
|
-
write(filename, geometry, field_data, driver="OpenFileGDB", **meta)
|
|
611
657
|
|
|
612
|
-
|
|
658
|
+
# int64 is not supported without additional config: https://gdal.org/en/latest/drivers/vector/openfilegdb.html#bit-integer-field-support
|
|
659
|
+
# it is converted to float64 by default and raises a warning
|
|
660
|
+
# (for GDAL >= 3.9.0 only)
|
|
661
|
+
write_params = (
|
|
662
|
+
{"TARGET_ARCGIS_VERSION": "ARCGIS_PRO_3_2_OR_LATER"} if write_int64 else {}
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
if write_int64 or __gdal_version__ < (3, 9, 0):
|
|
666
|
+
ctx = contextlib.nullcontext()
|
|
667
|
+
else:
|
|
668
|
+
ctx = pytest.warns(
|
|
669
|
+
RuntimeWarning, match="Integer64 will be written as a Float64"
|
|
670
|
+
)
|
|
671
|
+
|
|
672
|
+
with ctx:
|
|
673
|
+
write(
|
|
674
|
+
filename,
|
|
675
|
+
expected_geometry,
|
|
676
|
+
expected_field_data,
|
|
677
|
+
driver="OpenFileGDB",
|
|
678
|
+
**expected_meta,
|
|
679
|
+
**write_params,
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
meta, _, geometry, field_data = read(filename)
|
|
683
|
+
|
|
684
|
+
if not write_int64:
|
|
685
|
+
expected_field_data[3] = expected_field_data[3].astype("float64")
|
|
686
|
+
|
|
687
|
+
# bool types are converted to int32
|
|
688
|
+
expected_field_data[0] = expected_field_data[0].astype("int32")
|
|
689
|
+
|
|
690
|
+
assert meta["crs"] == expected_meta["crs"]
|
|
691
|
+
assert np.array_equal(meta["fields"], expected_meta["fields"])
|
|
692
|
+
|
|
693
|
+
assert np.array_equal(geometry, expected_geometry)
|
|
694
|
+
for i in range(len(expected_field_data)):
|
|
695
|
+
assert field_data[i].dtype == expected_field_data[i].dtype
|
|
696
|
+
assert np.array_equal(field_data[i], expected_field_data[i])
|
|
613
697
|
|
|
614
698
|
|
|
615
699
|
@pytest.mark.parametrize("ext", DRIVERS)
|
|
@@ -620,6 +704,9 @@ def test_write_append(tmp_path, naturalearth_lowres, ext):
|
|
|
620
704
|
if ext in (".geojsonl", ".geojsons") and __gdal_version__ < (3, 6, 0):
|
|
621
705
|
pytest.skip("Append to GeoJSONSeq only available for GDAL >= 3.6.0")
|
|
622
706
|
|
|
707
|
+
if ext == ".gpkg.zip":
|
|
708
|
+
pytest.skip("Append to .gpkg.zip is not supported")
|
|
709
|
+
|
|
623
710
|
meta, _, geometry, field_data = read(naturalearth_lowres)
|
|
624
711
|
|
|
625
712
|
# coerce output layer to MultiPolygon to avoid mixed type errors
|
|
@@ -743,7 +830,7 @@ def assert_equal_result(result1, result2):
|
|
|
743
830
|
|
|
744
831
|
assert np.array_equal(meta1["fields"], meta2["fields"])
|
|
745
832
|
assert np.array_equal(index1, index2)
|
|
746
|
-
assert all(
|
|
833
|
+
assert all(np.array_equal(f1, f2) for f1, f2 in zip(field_data1, field_data2))
|
|
747
834
|
|
|
748
835
|
if HAS_SHAPELY:
|
|
749
836
|
# a plain `assert np.array_equal(geometry1, geometry2)` doesn't work
|
|
@@ -794,6 +881,12 @@ def test_read_from_file_like(tmp_path, naturalearth_lowres, driver, ext):
|
|
|
794
881
|
assert_equal_result((meta, index, geometry, field_data), result2)
|
|
795
882
|
|
|
796
883
|
|
|
884
|
+
def test_read_from_nonseekable_bytes(nonseekable_bytes):
|
|
885
|
+
meta, _, geometry, _ = read(nonseekable_bytes)
|
|
886
|
+
assert meta["fields"].shape == (0,)
|
|
887
|
+
assert len(geometry) == 1
|
|
888
|
+
|
|
889
|
+
|
|
797
890
|
@pytest.mark.parametrize("ext", ["gpkg", "fgb"])
|
|
798
891
|
def test_read_write_data_types_numeric(tmp_path, ext):
|
|
799
892
|
# Point(0, 0)
|
|
@@ -809,13 +902,13 @@ def test_read_write_data_types_numeric(tmp_path, ext):
|
|
|
809
902
|
np.array([1, 2, 3], dtype="float64"),
|
|
810
903
|
]
|
|
811
904
|
fields = ["bool", "int16", "int32", "int64", "float32", "float64"]
|
|
812
|
-
meta =
|
|
905
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "spatial_index": False}
|
|
813
906
|
|
|
814
907
|
filename = tmp_path / f"test.{ext}"
|
|
815
908
|
write(filename, geometry, field_data, fields, **meta)
|
|
816
909
|
result = read(filename)[3]
|
|
817
|
-
assert all(
|
|
818
|
-
assert all(
|
|
910
|
+
assert all(np.array_equal(f1, f2) for f1, f2 in zip(result, field_data))
|
|
911
|
+
assert all(f1.dtype == f2.dtype for f1, f2 in zip(result, field_data))
|
|
819
912
|
|
|
820
913
|
# other integer data types that don't roundtrip exactly
|
|
821
914
|
# these are generally promoted to a larger integer type except for uint64
|
|
@@ -866,7 +959,7 @@ def test_read_write_datetime(tmp_path):
|
|
|
866
959
|
geometry = np.array(
|
|
867
960
|
[bytes.fromhex("010100000000000000000000000000000000000000")] * 2, dtype=object
|
|
868
961
|
)
|
|
869
|
-
meta =
|
|
962
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "spatial_index": False}
|
|
870
963
|
|
|
871
964
|
filename = tmp_path / "test.gpkg"
|
|
872
965
|
write(filename, geometry, field_data, fields, **meta)
|
|
@@ -889,7 +982,7 @@ def test_read_write_int64_large(tmp_path, ext):
|
|
|
889
982
|
)
|
|
890
983
|
field_data = [np.array([1, 2192502720, -5], dtype="int64")]
|
|
891
984
|
fields = ["overflow_int64"]
|
|
892
|
-
meta =
|
|
985
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "spatial_index": False}
|
|
893
986
|
|
|
894
987
|
filename = tmp_path / f"test.{ext}"
|
|
895
988
|
write(filename, geometry, field_data, fields, **meta)
|
|
@@ -912,17 +1005,17 @@ def test_read_data_types_numeric_with_null(test_gpkg_nulls):
|
|
|
912
1005
|
assert field.dtype == "float64"
|
|
913
1006
|
|
|
914
1007
|
|
|
915
|
-
def test_read_unsupported_types(
|
|
916
|
-
fields = read(
|
|
1008
|
+
def test_read_unsupported_types(list_field_values_file):
|
|
1009
|
+
fields = read(list_field_values_file)[3]
|
|
917
1010
|
# list field gets skipped, only integer field is read
|
|
918
1011
|
assert len(fields) == 1
|
|
919
1012
|
|
|
920
|
-
fields = read(
|
|
1013
|
+
fields = read(list_field_values_file, columns=["int64"])[3]
|
|
921
1014
|
assert len(fields) == 1
|
|
922
1015
|
|
|
923
1016
|
|
|
924
|
-
def test_read_datetime_millisecond(
|
|
925
|
-
field = read(
|
|
1017
|
+
def test_read_datetime_millisecond(datetime_file):
|
|
1018
|
+
field = read(datetime_file)[3][0]
|
|
926
1019
|
assert field.dtype == "datetime64[ms]"
|
|
927
1020
|
assert field[0] == np.datetime64("2020-01-01 09:00:00.123")
|
|
928
1021
|
assert field[1] == np.datetime64("2020-01-01 10:00:00.000")
|
|
@@ -951,13 +1044,14 @@ def test_read_unsupported_ext_with_prefix(tmp_path):
|
|
|
951
1044
|
assert field_data[0] == "data1"
|
|
952
1045
|
|
|
953
1046
|
|
|
954
|
-
def test_read_datetime_as_string(
|
|
955
|
-
field = read(
|
|
1047
|
+
def test_read_datetime_as_string(datetime_tz_file):
|
|
1048
|
+
field = read(datetime_tz_file)[3][0]
|
|
956
1049
|
assert field.dtype == "datetime64[ms]"
|
|
957
1050
|
# timezone is ignored in numpy layer
|
|
958
1051
|
assert field[0] == np.datetime64("2020-01-01 09:00:00.123")
|
|
959
1052
|
assert field[1] == np.datetime64("2020-01-01 10:00:00.000")
|
|
960
|
-
|
|
1053
|
+
|
|
1054
|
+
field = read(datetime_tz_file, datetime_as_string=True)[3][0]
|
|
961
1055
|
assert field.dtype == "object"
|
|
962
1056
|
# GDAL doesn't return strings in ISO format (yet)
|
|
963
1057
|
assert field[0] == "2020/01/01 09:00:00.123-05"
|
|
@@ -973,7 +1067,7 @@ def test_read_write_null_geometry(tmp_path, ext):
|
|
|
973
1067
|
)
|
|
974
1068
|
field_data = [np.array([1, 2], dtype="int32")]
|
|
975
1069
|
fields = ["col"]
|
|
976
|
-
meta =
|
|
1070
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326"}
|
|
977
1071
|
if ext == "gpkg":
|
|
978
1072
|
meta["spatial_index"] = False
|
|
979
1073
|
|
|
@@ -993,12 +1087,12 @@ def test_write_float_nan_null(tmp_path, dtype):
|
|
|
993
1087
|
)
|
|
994
1088
|
field_data = [np.array([1.5, np.nan], dtype=dtype)]
|
|
995
1089
|
fields = ["col"]
|
|
996
|
-
meta =
|
|
1090
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326"}
|
|
997
1091
|
filename = tmp_path / "test.geojson"
|
|
998
1092
|
|
|
999
1093
|
# default nan_as_null=True
|
|
1000
1094
|
write(filename, geometry, field_data, fields, **meta)
|
|
1001
|
-
with open(filename
|
|
1095
|
+
with open(filename) as f:
|
|
1002
1096
|
content = f.read()
|
|
1003
1097
|
assert '{ "col": null }' in content
|
|
1004
1098
|
|
|
@@ -1010,7 +1104,7 @@ def test_write_float_nan_null(tmp_path, dtype):
|
|
|
1010
1104
|
ctx = contextlib.nullcontext()
|
|
1011
1105
|
with ctx:
|
|
1012
1106
|
write(filename, geometry, field_data, fields, **meta, nan_as_null=False)
|
|
1013
|
-
with open(filename
|
|
1107
|
+
with open(filename) as f:
|
|
1014
1108
|
content = f.read()
|
|
1015
1109
|
assert '"properties": { }' in content
|
|
1016
1110
|
|
|
@@ -1024,7 +1118,7 @@ def test_write_float_nan_null(tmp_path, dtype):
|
|
|
1024
1118
|
nan_as_null=False,
|
|
1025
1119
|
WRITE_NON_FINITE_VALUES="YES",
|
|
1026
1120
|
)
|
|
1027
|
-
with open(filename
|
|
1121
|
+
with open(filename) as f:
|
|
1028
1122
|
content = f.read()
|
|
1029
1123
|
assert '{ "col": NaN }' in content
|
|
1030
1124
|
|
|
@@ -1043,7 +1137,7 @@ def test_write_float_nan_null_arrow(tmp_path):
|
|
|
1043
1137
|
)
|
|
1044
1138
|
field_data = [np.array([1.5, np.nan], dtype="float64")]
|
|
1045
1139
|
fields = ["col"]
|
|
1046
|
-
meta =
|
|
1140
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326"}
|
|
1047
1141
|
fname = tmp_path / "test.arrow"
|
|
1048
1142
|
|
|
1049
1143
|
# default nan_as_null=True
|
|
@@ -1146,6 +1240,27 @@ def test_write_memory_existing_unsupported(naturalearth_lowres):
|
|
|
1146
1240
|
write(buffer, geometry, field_data, driver="GeoJSON", layer="test", **meta)
|
|
1147
1241
|
|
|
1148
1242
|
|
|
1243
|
+
def test_write_open_file_handle(tmp_path, naturalearth_lowres):
|
|
1244
|
+
"""Verify that writing to an open file handle is not currently supported"""
|
|
1245
|
+
|
|
1246
|
+
meta, _, geometry, field_data = read(naturalearth_lowres)
|
|
1247
|
+
|
|
1248
|
+
# verify it fails for regular file handle
|
|
1249
|
+
with pytest.raises(
|
|
1250
|
+
NotImplementedError, match="writing to an open file handle is not yet supported"
|
|
1251
|
+
):
|
|
1252
|
+
with open(tmp_path / "test.geojson", "wb") as f:
|
|
1253
|
+
write(f, geometry, field_data, driver="GeoJSON", layer="test", **meta)
|
|
1254
|
+
|
|
1255
|
+
# verify it fails for ZipFile
|
|
1256
|
+
with pytest.raises(
|
|
1257
|
+
NotImplementedError, match="writing to an open file handle is not yet supported"
|
|
1258
|
+
):
|
|
1259
|
+
with ZipFile(tmp_path / "test.geojson.zip", "w") as z:
|
|
1260
|
+
with z.open("test.geojson", "w") as f:
|
|
1261
|
+
write(f, geometry, field_data, driver="GeoJSON", layer="test", **meta)
|
|
1262
|
+
|
|
1263
|
+
|
|
1149
1264
|
@pytest.mark.parametrize("ext", ["fgb", "gpkg", "geojson"])
|
|
1150
1265
|
@pytest.mark.parametrize(
|
|
1151
1266
|
"read_encoding,write_encoding",
|
|
@@ -1182,7 +1297,7 @@ def test_encoding_io(tmp_path, ext, read_encoding, write_encoding):
|
|
|
1182
1297
|
np.array([mandarin], dtype=object),
|
|
1183
1298
|
]
|
|
1184
1299
|
fields = [arabic, cree, mandarin]
|
|
1185
|
-
meta =
|
|
1300
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "encoding": write_encoding}
|
|
1186
1301
|
|
|
1187
1302
|
filename = tmp_path / f"test.{ext}"
|
|
1188
1303
|
write(filename, geometry, field_data, fields, **meta)
|
|
@@ -1232,7 +1347,7 @@ def test_encoding_io_shapefile(tmp_path, read_encoding, write_encoding):
|
|
|
1232
1347
|
# character level) by GDAL when output to shapefile, so we have to truncate
|
|
1233
1348
|
# before writing
|
|
1234
1349
|
fields = [arabic[:5], cree[:3], mandarin]
|
|
1235
|
-
meta =
|
|
1350
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "encoding": "UTF-8"}
|
|
1236
1351
|
|
|
1237
1352
|
filename = tmp_path / "test.shp"
|
|
1238
1353
|
# NOTE: GDAL automatically creates a cpg file with the encoding name, which
|
|
@@ -1277,7 +1392,7 @@ def test_non_utf8_encoding_io(tmp_path, ext, encoded_text):
|
|
|
1277
1392
|
field_data = [np.array([text], dtype=object)]
|
|
1278
1393
|
|
|
1279
1394
|
fields = [text]
|
|
1280
|
-
meta =
|
|
1395
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "encoding": encoding}
|
|
1281
1396
|
|
|
1282
1397
|
filename = tmp_path / f"test.{ext}"
|
|
1283
1398
|
write(filename, geometry, field_data, fields, **meta)
|
|
@@ -1307,7 +1422,7 @@ def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text):
|
|
|
1307
1422
|
field_data = [np.array([text], dtype=object)]
|
|
1308
1423
|
|
|
1309
1424
|
fields = [text]
|
|
1310
|
-
meta =
|
|
1425
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326", "encoding": encoding}
|
|
1311
1426
|
|
|
1312
1427
|
filename = tmp_path / "test.shp"
|
|
1313
1428
|
write(filename, geometry, field_data, fields, **meta)
|
|
@@ -1357,7 +1472,7 @@ def test_write_with_mask(tmp_path):
|
|
|
1357
1472
|
field_data = [np.array([1, 2, 3], dtype="int32")]
|
|
1358
1473
|
field_mask = [np.array([False, True, False])]
|
|
1359
1474
|
fields = ["col"]
|
|
1360
|
-
meta =
|
|
1475
|
+
meta = {"geometry_type": "Point", "crs": "EPSG:4326"}
|
|
1361
1476
|
|
|
1362
1477
|
filename = tmp_path / "test.geojson"
|
|
1363
1478
|
write(filename, geometry, field_data, fields, field_mask, **meta)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from pyogrio import vsi_listtree, vsi_unlink
|
|
4
|
+
from pyogrio.raw import read, write
|
|
5
|
+
from pyogrio.util import vsimem_rmtree_toplevel
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_vsimem_rmtree_toplevel(naturalearth_lowres):
|
|
11
|
+
# Prepare test data in /vsimem/
|
|
12
|
+
meta, _, geometry, field_data = read(naturalearth_lowres)
|
|
13
|
+
meta["spatial_index"] = False
|
|
14
|
+
meta["geometry_type"] = "MultiPolygon"
|
|
15
|
+
test_dir_path = Path(f"/vsimem/test/{naturalearth_lowres.stem}.gpkg")
|
|
16
|
+
test_dir2_path = Path(f"/vsimem/test2/test2/{naturalearth_lowres.stem}.gpkg")
|
|
17
|
+
|
|
18
|
+
write(test_dir_path, geometry, field_data, **meta)
|
|
19
|
+
write(test_dir2_path, geometry, field_data, **meta)
|
|
20
|
+
|
|
21
|
+
# Check if everything was created properly with listtree
|
|
22
|
+
files = vsi_listtree("/vsimem/")
|
|
23
|
+
assert test_dir_path.as_posix() in files
|
|
24
|
+
assert test_dir2_path.as_posix() in files
|
|
25
|
+
|
|
26
|
+
# Test deleting parent dir of file in single directory
|
|
27
|
+
vsimem_rmtree_toplevel(test_dir_path)
|
|
28
|
+
files = vsi_listtree("/vsimem/")
|
|
29
|
+
assert test_dir_path.parent.as_posix() not in files
|
|
30
|
+
assert test_dir2_path.as_posix() in files
|
|
31
|
+
|
|
32
|
+
# Test deleting top-level dir of file in a subdirectory
|
|
33
|
+
vsimem_rmtree_toplevel(test_dir2_path)
|
|
34
|
+
assert test_dir2_path.as_posix() not in vsi_listtree("/vsimem/")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_vsimem_rmtree_toplevel_error(naturalearth_lowres):
|
|
38
|
+
# Prepare test data in /vsimem
|
|
39
|
+
meta, _, geometry, field_data = read(naturalearth_lowres)
|
|
40
|
+
meta["spatial_index"] = False
|
|
41
|
+
meta["geometry_type"] = "MultiPolygon"
|
|
42
|
+
test_file_path = Path(f"/vsimem/pyogrio_test_{naturalearth_lowres.stem}.gpkg")
|
|
43
|
+
|
|
44
|
+
write(test_file_path, geometry, field_data, **meta)
|
|
45
|
+
assert test_file_path.as_posix() in vsi_listtree("/vsimem/")
|
|
46
|
+
|
|
47
|
+
# Deleting parent dir of non-existent file should raise an error.
|
|
48
|
+
with pytest.raises(FileNotFoundError, match="Path does not exist"):
|
|
49
|
+
vsimem_rmtree_toplevel("/vsimem/test/non-existent.gpkg")
|
|
50
|
+
|
|
51
|
+
# File should still be there
|
|
52
|
+
assert test_file_path.as_posix() in vsi_listtree("/vsimem/")
|
|
53
|
+
|
|
54
|
+
# Cleanup.
|
|
55
|
+
vsi_unlink(test_file_path)
|
|
56
|
+
assert test_file_path not in vsi_listtree("/vsimem/")
|