pyogrio 0.9.0__cp312-cp312-win_amd64.whl → 0.11.0__cp312-cp312-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyogrio might be problematic. Click here for more details.
- pyogrio/__init__.py +35 -28
- pyogrio/_compat.py +15 -1
- pyogrio/_env.py +4 -6
- pyogrio/_err.c +8941 -3106
- pyogrio/_err.cp312-win_amd64.pyd +0 -0
- pyogrio/_geometry.c +1008 -807
- pyogrio/_geometry.cp312-win_amd64.pyd +0 -0
- pyogrio/_io.c +10678 -9440
- pyogrio/_io.cp312-win_amd64.pyd +0 -0
- pyogrio/_ogr.c +1950 -1873
- pyogrio/_ogr.cp312-win_amd64.pyd +0 -0
- pyogrio/_version.py +3 -3
- pyogrio/_vsi.c +7558 -2509
- pyogrio/_vsi.cp312-win_amd64.pyd +0 -0
- pyogrio/core.py +86 -20
- pyogrio/errors.py +9 -16
- pyogrio/gdal_data/GDAL-targets.cmake +2 -2
- pyogrio/gdal_data/GDALConfig.cmake +0 -1
- pyogrio/gdal_data/GDALConfigVersion.cmake +3 -3
- pyogrio/gdal_data/MM_m_idofic.csv +321 -0
- pyogrio/gdal_data/gdalinfo_output.schema.json +3 -3
- pyogrio/gdal_data/gdaltileindex.xsd +253 -0
- pyogrio/gdal_data/gdalvrt.xsd +178 -63
- pyogrio/gdal_data/nitf_spec.xml +1 -17
- pyogrio/gdal_data/nitf_spec.xsd +1 -17
- pyogrio/gdal_data/ogrinfo_output.schema.json +23 -0
- pyogrio/gdal_data/ogrvrt.xsd +4 -17
- pyogrio/gdal_data/osmconf.ini +3 -1
- pyogrio/gdal_data/pci_datum.txt +222 -155
- pyogrio/gdal_data/pci_ellips.txt +90 -38
- pyogrio/gdal_data/pdfcomposition.xsd +1 -17
- pyogrio/gdal_data/vcpkg.spdx.json +29 -24
- pyogrio/gdal_data/vcpkg_abi_info.txt +32 -31
- pyogrio/gdal_data/vdv452.xml +1 -17
- pyogrio/gdal_data/vdv452.xsd +1 -17
- pyogrio/geopandas.py +122 -66
- pyogrio/proj_data/ITRF2014 +1 -1
- pyogrio/proj_data/ITRF2020 +91 -0
- pyogrio/proj_data/proj-config-version.cmake +2 -2
- pyogrio/proj_data/proj-config.cmake +1 -1
- pyogrio/proj_data/proj-targets-release.cmake +3 -2
- pyogrio/proj_data/proj-targets.cmake +2 -2
- pyogrio/proj_data/proj.db +0 -0
- pyogrio/proj_data/proj.ini +11 -3
- pyogrio/proj_data/proj4-targets-release.cmake +3 -2
- pyogrio/proj_data/proj4-targets.cmake +2 -2
- pyogrio/proj_data/projjson.schema.json +1 -1
- pyogrio/proj_data/usage +7 -2
- pyogrio/proj_data/vcpkg.spdx.json +26 -21
- pyogrio/proj_data/vcpkg_abi_info.txt +19 -18
- pyogrio/raw.py +46 -30
- pyogrio/tests/conftest.py +214 -12
- pyogrio/tests/fixtures/README.md +32 -13
- pyogrio/tests/fixtures/curve.gpkg +0 -0
- pyogrio/tests/fixtures/{test_multisurface.gpkg → curvepolygon.gpkg} +0 -0
- pyogrio/tests/fixtures/line_zm.gpkg +0 -0
- pyogrio/tests/fixtures/multisurface.gpkg +0 -0
- pyogrio/tests/test_arrow.py +181 -24
- pyogrio/tests/test_core.py +170 -76
- pyogrio/tests/test_geopandas_io.py +483 -135
- pyogrio/tests/test_path.py +39 -17
- pyogrio/tests/test_raw_io.py +170 -55
- pyogrio/tests/test_util.py +56 -0
- pyogrio/util.py +69 -32
- pyogrio-0.11.0.dist-info/DELVEWHEEL +2 -0
- pyogrio-0.11.0.dist-info/METADATA +124 -0
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info}/RECORD +90 -102
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info}/WHEEL +1 -1
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info/licenses}/LICENSE +1 -1
- pyogrio.libs/{Lerc-5e4d8cbeeabca06f95e2270792304dc3.dll → Lerc-0a4c85fb364eca6153da109568898e6c.dll} +0 -0
- pyogrio.libs/{gdal-b434963605a006e01c486c0df6dea4e0.dll → gdal-34e3e080255b205ef706390e097fa4dc.dll} +0 -0
- pyogrio.libs/geos-d8f20037634583c2efcd6ea1f4153169.dll +0 -0
- pyogrio.libs/geos_c-046e8885887192fced9516bda554471d.dll +0 -0
- pyogrio.libs/geotiff-1f2f76a5d30685a8e0497d9dbf8a79cf.dll +0 -0
- pyogrio.libs/{iconv-2-8fcc23ddc6f096c45871011b6e008b44.dll → iconv-2-4b71ebbdf6834234e0c64cb2439b77cf.dll} +0 -0
- pyogrio.libs/{jpeg62-2f9b7af22d78338e8f0be0058503dc35.dll → jpeg62-bf2a75b1f2695748cd86238ddb19c65b.dll} +0 -0
- pyogrio.libs/json-c-4bc6781090e73c9b22d8dc057618a277.dll +0 -0
- pyogrio.libs/libcrypto-3-x64-c8f1692ed45cf55faecce2c448056b2e.dll +0 -0
- pyogrio.libs/libcurl-f3604410cd467a44d927794ebdce81b8.dll +0 -0
- pyogrio.libs/libexpat-385074fd54deb4b640baafa42cbb3146.dll +0 -0
- pyogrio.libs/liblzma-8968f0bfd463b7fe612b20d07adc3c1e.dll +0 -0
- pyogrio.libs/libpng16-44105208fe941d03e9f0c17bdbb952f7.dll +0 -0
- pyogrio.libs/libssl-3-x64-58c364315f431ab1f0e48b311b8cf105.dll +0 -0
- pyogrio.libs/msvcp140-99aa35e2033bb8d388bc35c8a68b77e3.dll +0 -0
- pyogrio.libs/proj_9-ee59474f99643c112eb02aa34a910237.dll +0 -0
- pyogrio.libs/{qhull_r-c45abde5d0c92faf723cc2942138af77.dll → qhull_r-eaac2f11a3d8241f082e54447c7504d7.dll} +0 -0
- pyogrio.libs/sqlite3-dc748e3452944fd41001abacdd783569.dll +0 -0
- pyogrio.libs/tiff-c409ddbe87b39639b83fee50d4aea318.dll +0 -0
- pyogrio.libs/{zlib1-e1272810861a13dd8d6cff3beac47f17.dll → zlib1-094085b7b78666197dcc8e1fce2d835d.dll} +0 -0
- pyogrio/_err.pxd +0 -4
- pyogrio/_err.pyx +0 -250
- pyogrio/_geometry.pxd +0 -4
- pyogrio/_geometry.pyx +0 -129
- pyogrio/_io.pxd +0 -0
- pyogrio/_io.pyx +0 -2742
- pyogrio/_ogr.pxd +0 -444
- pyogrio/_ogr.pyx +0 -346
- pyogrio/_vsi.pxd +0 -4
- pyogrio/_vsi.pyx +0 -140
- pyogrio/arrow_bridge.h +0 -115
- pyogrio/gdal_data/bag_template.xml +0 -201
- pyogrio/gdal_data/gmlasconf.xml +0 -169
- pyogrio/gdal_data/gmlasconf.xsd +0 -1066
- pyogrio/gdal_data/netcdf_config.xsd +0 -143
- pyogrio/tests/fixtures/poly_not_enough_points.shp.zip +0 -0
- pyogrio/tests/fixtures/test_datetime.geojson +0 -7
- pyogrio/tests/fixtures/test_datetime_tz.geojson +0 -8
- pyogrio/tests/fixtures/test_fgdb.gdb.zip +0 -0
- pyogrio/tests/fixtures/test_nested.geojson +0 -18
- pyogrio/tests/fixtures/test_ogr_types_list.geojson +0 -12
- pyogrio-0.9.0.dist-info/DELVEWHEEL +0 -2
- pyogrio-0.9.0.dist-info/METADATA +0 -100
- pyogrio.libs/geos-f0622d0794b81c937a851b2e6fa9b712.dll +0 -0
- pyogrio.libs/geos_c-0e16bf70612fc3301d077b9d863a3fdb.dll +0 -0
- pyogrio.libs/geotiff-772e7c705fb15ddf91b432adb4eb1f6c.dll +0 -0
- pyogrio.libs/json-c-e52a077545e4057de42beb4948289b41.dll +0 -0
- pyogrio.libs/libcurl-bc81cd8afe15b10c0821b181b6af8bd0.dll +0 -0
- pyogrio.libs/libexpat-fbe03ca8917dfda776562d4338b289b8.dll +0 -0
- pyogrio.libs/liblzma-6b36f24d54d3dd45f274a2aebef81085.dll +0 -0
- pyogrio.libs/libpng16-13928571ad910705eae8d7dd8eef8b11.dll +0 -0
- pyogrio.libs/msvcp140-46db46e967c8db2cb7a20fc75872a57e.dll +0 -0
- pyogrio.libs/proj-8a30239ef2dfc3b9dd2bb48e8abb330f.dll +0 -0
- pyogrio.libs/sqlite3-df30c3cf230727e23c43c40126a530f7.dll +0 -0
- pyogrio.libs/tiff-43630f30487a9015213475ae86ed3fa3.dll +0 -0
- {pyogrio-0.9.0.dist-info → pyogrio-0.11.0.dist-info}/top_level.txt +0 -0
pyogrio/tests/conftest.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
from io import BytesIO
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
from zipfile import ZIP_DEFLATED, ZipFile
|
|
3
4
|
|
|
4
|
-
import
|
|
5
|
+
import numpy as np
|
|
5
6
|
|
|
6
7
|
from pyogrio import (
|
|
7
8
|
__gdal_version_string__,
|
|
@@ -13,10 +14,14 @@ from pyogrio._compat import (
|
|
|
13
14
|
HAS_ARROW_WRITE_API,
|
|
14
15
|
HAS_GDAL_GEOS,
|
|
15
16
|
HAS_PYARROW,
|
|
17
|
+
HAS_PYPROJ,
|
|
16
18
|
HAS_SHAPELY,
|
|
17
19
|
)
|
|
20
|
+
from pyogrio.core import vsi_rmtree
|
|
18
21
|
from pyogrio.raw import read, write
|
|
19
22
|
|
|
23
|
+
import pytest
|
|
24
|
+
|
|
20
25
|
_data_dir = Path(__file__).parent.resolve() / "fixtures"
|
|
21
26
|
|
|
22
27
|
# mapping of driver extension to driver name for well-supported drivers
|
|
@@ -26,7 +31,10 @@ DRIVERS = {
|
|
|
26
31
|
".geojsonl": "GeoJSONSeq",
|
|
27
32
|
".geojsons": "GeoJSONSeq",
|
|
28
33
|
".gpkg": "GPKG",
|
|
34
|
+
".gpkg.zip": "GPKG",
|
|
29
35
|
".shp": "ESRI Shapefile",
|
|
36
|
+
".shp.zip": "ESRI Shapefile",
|
|
37
|
+
".shz": "ESRI Shapefile",
|
|
30
38
|
}
|
|
31
39
|
|
|
32
40
|
# mapping of driver name to extension
|
|
@@ -34,6 +42,15 @@ DRIVER_EXT = {driver: ext for ext, driver in DRIVERS.items()}
|
|
|
34
42
|
|
|
35
43
|
ALL_EXTS = [".fgb", ".geojson", ".geojsonl", ".gpkg", ".shp"]
|
|
36
44
|
|
|
45
|
+
START_FID = {
|
|
46
|
+
".fgb": 0,
|
|
47
|
+
".geojson": 0,
|
|
48
|
+
".geojsonl": 0,
|
|
49
|
+
".geojsons": 0,
|
|
50
|
+
".gpkg": 1,
|
|
51
|
+
".shp": 0,
|
|
52
|
+
}
|
|
53
|
+
|
|
37
54
|
|
|
38
55
|
def pytest_report_header(config):
|
|
39
56
|
drivers = ", ".join(
|
|
@@ -53,6 +70,8 @@ requires_pyarrow_api = pytest.mark.skipif(
|
|
|
53
70
|
not HAS_ARROW_API or not HAS_PYARROW, reason="GDAL>=3.6 and pyarrow required"
|
|
54
71
|
)
|
|
55
72
|
|
|
73
|
+
requires_pyproj = pytest.mark.skipif(not HAS_PYPROJ, reason="pyproj required")
|
|
74
|
+
|
|
56
75
|
requires_arrow_write_api = pytest.mark.skipif(
|
|
57
76
|
not HAS_ARROW_WRITE_API or not HAS_PYARROW,
|
|
58
77
|
reason="GDAL>=3.8 required for Arrow write API",
|
|
@@ -108,9 +127,14 @@ def naturalearth_lowres_all_ext(tmp_path, naturalearth_lowres, request):
|
|
|
108
127
|
return prepare_testfile(naturalearth_lowres, tmp_path, request.param)
|
|
109
128
|
|
|
110
129
|
|
|
130
|
+
@pytest.fixture(scope="function", params=[".geojson"])
|
|
131
|
+
def naturalearth_lowres_geojson(tmp_path, naturalearth_lowres, request):
|
|
132
|
+
return prepare_testfile(naturalearth_lowres, tmp_path, request.param)
|
|
133
|
+
|
|
134
|
+
|
|
111
135
|
@pytest.fixture(scope="function")
|
|
112
136
|
def naturalearth_lowres_vsi(tmp_path, naturalearth_lowres):
|
|
113
|
-
"""Wrap naturalearth_lowres as a zip file for
|
|
137
|
+
"""Wrap naturalearth_lowres as a zip file for VSI tests"""
|
|
114
138
|
|
|
115
139
|
path = tmp_path / f"{naturalearth_lowres.name}.zip"
|
|
116
140
|
with ZipFile(path, mode="w", compression=ZIP_DEFLATED, compresslevel=5) as out:
|
|
@@ -121,29 +145,182 @@ def naturalearth_lowres_vsi(tmp_path, naturalearth_lowres):
|
|
|
121
145
|
return path, f"/vsizip/{path}/{naturalearth_lowres.name}"
|
|
122
146
|
|
|
123
147
|
|
|
148
|
+
@pytest.fixture(scope="function")
|
|
149
|
+
def naturalearth_lowres_vsimem(naturalearth_lowres):
|
|
150
|
+
"""Write naturalearth_lowres to a vsimem file for VSI tests"""
|
|
151
|
+
|
|
152
|
+
meta, _, geometry, field_data = read(naturalearth_lowres)
|
|
153
|
+
name = f"pyogrio_fixture_{naturalearth_lowres.stem}"
|
|
154
|
+
dst_path = Path(f"/vsimem/{name}/{name}.gpkg")
|
|
155
|
+
meta["spatial_index"] = False
|
|
156
|
+
meta["geometry_type"] = "MultiPolygon"
|
|
157
|
+
|
|
158
|
+
write(dst_path, geometry, field_data, layer="naturalearth_lowres", **meta)
|
|
159
|
+
yield dst_path
|
|
160
|
+
|
|
161
|
+
vsi_rmtree(dst_path.parent)
|
|
162
|
+
|
|
163
|
+
|
|
124
164
|
@pytest.fixture(scope="session")
|
|
125
|
-
def
|
|
126
|
-
return
|
|
165
|
+
def line_zm_file():
|
|
166
|
+
return _data_dir / "line_zm.gpkg"
|
|
127
167
|
|
|
128
168
|
|
|
129
169
|
@pytest.fixture(scope="session")
|
|
130
|
-
def
|
|
131
|
-
return _data_dir / "
|
|
170
|
+
def curve_file():
|
|
171
|
+
return _data_dir / "curve.gpkg"
|
|
132
172
|
|
|
133
173
|
|
|
134
174
|
@pytest.fixture(scope="session")
|
|
135
|
-
def
|
|
136
|
-
return _data_dir / "
|
|
175
|
+
def curve_polygon_file():
|
|
176
|
+
return _data_dir / "curvepolygon.gpkg"
|
|
137
177
|
|
|
138
178
|
|
|
139
179
|
@pytest.fixture(scope="session")
|
|
140
|
-
def
|
|
141
|
-
return _data_dir / "
|
|
180
|
+
def multisurface_file():
|
|
181
|
+
return _data_dir / "multisurface.gpkg"
|
|
142
182
|
|
|
143
183
|
|
|
144
184
|
@pytest.fixture(scope="session")
|
|
145
|
-
def
|
|
146
|
-
return _data_dir / "
|
|
185
|
+
def test_gpkg_nulls():
|
|
186
|
+
return _data_dir / "test_gpkg_nulls.gpkg"
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@pytest.fixture(scope="function")
|
|
190
|
+
def no_geometry_file(tmp_path):
|
|
191
|
+
# create a GPKG layer that does not include geometry
|
|
192
|
+
filename = tmp_path / "test_no_geometry.gpkg"
|
|
193
|
+
write(
|
|
194
|
+
filename,
|
|
195
|
+
layer="no_geometry",
|
|
196
|
+
geometry=None,
|
|
197
|
+
field_data=[np.array(["a", "b", "c"])],
|
|
198
|
+
fields=["col"],
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
return filename
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@pytest.fixture(scope="function")
|
|
205
|
+
def list_field_values_file(tmp_path):
|
|
206
|
+
# Create a GeoJSON file with list values in a property
|
|
207
|
+
list_geojson = """{
|
|
208
|
+
"type": "FeatureCollection",
|
|
209
|
+
"features": [
|
|
210
|
+
{
|
|
211
|
+
"type": "Feature",
|
|
212
|
+
"properties": { "int64": 1, "list_int64": [0, 1] },
|
|
213
|
+
"geometry": { "type": "Point", "coordinates": [0, 2] }
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
"type": "Feature",
|
|
217
|
+
"properties": { "int64": 2, "list_int64": [2, 3] },
|
|
218
|
+
"geometry": { "type": "Point", "coordinates": [1, 2] }
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
"type": "Feature",
|
|
222
|
+
"properties": { "int64": 3, "list_int64": [4, 5] },
|
|
223
|
+
"geometry": { "type": "Point", "coordinates": [2, 2] }
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
"type": "Feature",
|
|
227
|
+
"properties": { "int64": 4, "list_int64": [6, 7] },
|
|
228
|
+
"geometry": { "type": "Point", "coordinates": [3, 2] }
|
|
229
|
+
},
|
|
230
|
+
{
|
|
231
|
+
"type": "Feature",
|
|
232
|
+
"properties": { "int64": 5, "list_int64": [8, 9] },
|
|
233
|
+
"geometry": { "type": "Point", "coordinates": [4, 2] }
|
|
234
|
+
}
|
|
235
|
+
]
|
|
236
|
+
}"""
|
|
237
|
+
|
|
238
|
+
filename = tmp_path / "test_ogr_types_list.geojson"
|
|
239
|
+
with open(filename, "w") as f:
|
|
240
|
+
_ = f.write(list_geojson)
|
|
241
|
+
|
|
242
|
+
return filename
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@pytest.fixture(scope="function")
|
|
246
|
+
def nested_geojson_file(tmp_path):
|
|
247
|
+
# create GeoJSON file with nested properties
|
|
248
|
+
nested_geojson = """{
|
|
249
|
+
"type": "FeatureCollection",
|
|
250
|
+
"features": [
|
|
251
|
+
{
|
|
252
|
+
"type": "Feature",
|
|
253
|
+
"geometry": {
|
|
254
|
+
"type": "Point",
|
|
255
|
+
"coordinates": [0, 0]
|
|
256
|
+
},
|
|
257
|
+
"properties": {
|
|
258
|
+
"top_level": "A",
|
|
259
|
+
"intermediate_level": {
|
|
260
|
+
"bottom_level": "B"
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
]
|
|
265
|
+
}"""
|
|
266
|
+
|
|
267
|
+
filename = tmp_path / "test_nested.geojson"
|
|
268
|
+
with open(filename, "w") as f:
|
|
269
|
+
_ = f.write(nested_geojson)
|
|
270
|
+
|
|
271
|
+
return filename
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
@pytest.fixture(scope="function")
|
|
275
|
+
def datetime_file(tmp_path):
|
|
276
|
+
# create GeoJSON file with millisecond precision
|
|
277
|
+
datetime_geojson = """{
|
|
278
|
+
"type": "FeatureCollection",
|
|
279
|
+
"features": [
|
|
280
|
+
{
|
|
281
|
+
"type": "Feature",
|
|
282
|
+
"properties": { "col": "2020-01-01T09:00:00.123" },
|
|
283
|
+
"geometry": { "type": "Point", "coordinates": [1, 1] }
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
"type": "Feature",
|
|
287
|
+
"properties": { "col": "2020-01-01T10:00:00" },
|
|
288
|
+
"geometry": { "type": "Point", "coordinates": [2, 2] }
|
|
289
|
+
}
|
|
290
|
+
]
|
|
291
|
+
}"""
|
|
292
|
+
|
|
293
|
+
filename = tmp_path / "test_datetime.geojson"
|
|
294
|
+
with open(filename, "w") as f:
|
|
295
|
+
_ = f.write(datetime_geojson)
|
|
296
|
+
|
|
297
|
+
return filename
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@pytest.fixture(scope="function")
|
|
301
|
+
def datetime_tz_file(tmp_path):
|
|
302
|
+
# create GeoJSON file with datetimes with timezone
|
|
303
|
+
datetime_tz_geojson = """{
|
|
304
|
+
"type": "FeatureCollection",
|
|
305
|
+
"features": [
|
|
306
|
+
{
|
|
307
|
+
"type": "Feature",
|
|
308
|
+
"properties": { "datetime_col": "2020-01-01T09:00:00.123-05:00" },
|
|
309
|
+
"geometry": { "type": "Point", "coordinates": [1, 1] }
|
|
310
|
+
},
|
|
311
|
+
{
|
|
312
|
+
"type": "Feature",
|
|
313
|
+
"properties": { "datetime_col": "2020-01-01T10:00:00-05:00" },
|
|
314
|
+
"geometry": { "type": "Point", "coordinates": [2, 2] }
|
|
315
|
+
}
|
|
316
|
+
]
|
|
317
|
+
}"""
|
|
318
|
+
|
|
319
|
+
filename = tmp_path / "test_datetime_tz.geojson"
|
|
320
|
+
with open(filename, "w") as f:
|
|
321
|
+
f.write(datetime_tz_geojson)
|
|
322
|
+
|
|
323
|
+
return filename
|
|
147
324
|
|
|
148
325
|
|
|
149
326
|
@pytest.fixture(scope="function")
|
|
@@ -178,6 +355,31 @@ def geojson_filelike(tmp_path):
|
|
|
178
355
|
yield f
|
|
179
356
|
|
|
180
357
|
|
|
358
|
+
@pytest.fixture(scope="function")
|
|
359
|
+
def nonseekable_bytes(tmp_path):
|
|
360
|
+
# mock a non-seekable byte stream, such as a zstandard handle
|
|
361
|
+
class NonSeekableBytesIO(BytesIO):
|
|
362
|
+
def seekable(self):
|
|
363
|
+
return False
|
|
364
|
+
|
|
365
|
+
def seek(self, *args, **kwargs):
|
|
366
|
+
raise OSError("cannot seek")
|
|
367
|
+
|
|
368
|
+
# wrap GeoJSON into a non-seekable BytesIO
|
|
369
|
+
geojson = """{
|
|
370
|
+
"type": "FeatureCollection",
|
|
371
|
+
"features": [
|
|
372
|
+
{
|
|
373
|
+
"type": "Feature",
|
|
374
|
+
"properties": { },
|
|
375
|
+
"geometry": { "type": "Point", "coordinates": [1, 1] }
|
|
376
|
+
}
|
|
377
|
+
]
|
|
378
|
+
}"""
|
|
379
|
+
|
|
380
|
+
return NonSeekableBytesIO(geojson.encode("UTF-8"))
|
|
381
|
+
|
|
382
|
+
|
|
181
383
|
@pytest.fixture(
|
|
182
384
|
scope="session",
|
|
183
385
|
params=[
|
pyogrio/tests/fixtures/README.md
CHANGED
|
@@ -1,13 +1,28 @@
|
|
|
1
1
|
# Test datasets
|
|
2
2
|
|
|
3
|
-
##
|
|
3
|
+
## Obtaining / creating test datasets
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
If a test dataset can be created in code, do that instead. If it is used in a
|
|
6
|
+
single test, create the test dataset as part of that test. If it is used in
|
|
7
|
+
more than a single test, add it to `pyogrio/tests/conftest.py` instead, as a
|
|
8
|
+
function-scoped test fixture.
|
|
9
|
+
|
|
10
|
+
If you need to obtain 3rd party test files:
|
|
11
|
+
|
|
12
|
+
- add a section below that describes the source location and processing steps
|
|
13
|
+
to derive that dataset
|
|
14
|
+
- make sure the license is compatible with including in Pyogrio (public domain or open-source)
|
|
15
|
+
and record that license below
|
|
16
|
+
|
|
17
|
+
Please keep the test files no larger than necessary to use in tests.
|
|
6
18
|
|
|
7
|
-
##
|
|
19
|
+
## Included test datasets
|
|
20
|
+
|
|
21
|
+
### Natural Earth lowres
|
|
22
|
+
|
|
23
|
+
`naturalearth_lowres.shp` was copied from GeoPandas.
|
|
8
24
|
|
|
9
|
-
|
|
10
|
-
Downloaded from http://trac.osgeo.org/gdal/raw-attachment/wiki/FileGDB/test_fgdb.gdb.zip
|
|
25
|
+
License: public domain
|
|
11
26
|
|
|
12
27
|
### GPKG test dataset with null values
|
|
13
28
|
|
|
@@ -75,15 +90,19 @@ NOTE: Reading boolean values into GeoPandas using Fiona backend treats those
|
|
|
75
90
|
values as `None` and column dtype as `object`; Pyogrio treats those values as
|
|
76
91
|
`np.nan` and column dtype as `float64`.
|
|
77
92
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
This was extracted from https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_0308_HU4_GDB.zip
|
|
81
|
-
`NHDWaterbody` layer using ogr2ogr:
|
|
82
|
-
|
|
83
|
-
```bash
|
|
84
|
-
ogr2ogr test_mixed_surface.gpkg NHDPLUS_H_0308_HU4_GDB.gdb NHDWaterbody -where '"NHDPlusID" = 15000300070477' -select "NHDPlusID"
|
|
85
|
-
```
|
|
93
|
+
License: same as Pyogrio
|
|
86
94
|
|
|
87
95
|
### OSM PBF test
|
|
88
96
|
|
|
89
97
|
This was downloaded from https://github.com/openstreetmap/OSM-binary/blob/master/resources/sample.pbf
|
|
98
|
+
|
|
99
|
+
License: [Open Data Commons Open Database License (ODbL)](https://opendatacommons.org/licenses/odbl/)
|
|
100
|
+
|
|
101
|
+
### Test files for geometry types that are downgraded on read
|
|
102
|
+
|
|
103
|
+
`line_zm.gpkg` was created using QGIS to digitize a LineString GPKG layer with Z and M enabled. Downgraded to LineString Z on read.
|
|
104
|
+
`curve.gpkg` was created using QGIS to digitize a Curve GPKG layer. Downgraded to LineString on read.
|
|
105
|
+
`curvepolygon.gpkg` was created using QGIS to digitize a CurvePolygon GPKG layer. Downgraded to Polygon on read.
|
|
106
|
+
`multisurface.gpkg` was created using QGIS to digitize a MultiSurface GPKG layer. Downgraded to MultiPolygon on read.
|
|
107
|
+
|
|
108
|
+
License: same as Pyogrio
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
pyogrio/tests/test_arrow.py
CHANGED
|
@@ -1,39 +1,42 @@
|
|
|
1
1
|
import contextlib
|
|
2
|
-
from io import BytesIO
|
|
3
2
|
import json
|
|
4
3
|
import math
|
|
5
4
|
import os
|
|
6
|
-
from packaging.version import Version
|
|
7
5
|
import sys
|
|
6
|
+
from io import BytesIO
|
|
7
|
+
from packaging.version import Version
|
|
8
|
+
from zipfile import ZipFile
|
|
8
9
|
|
|
9
|
-
import pytest
|
|
10
10
|
import numpy as np
|
|
11
11
|
|
|
12
12
|
import pyogrio
|
|
13
13
|
from pyogrio import (
|
|
14
14
|
__gdal_version__,
|
|
15
|
+
get_gdal_config_option,
|
|
16
|
+
list_layers,
|
|
15
17
|
read_dataframe,
|
|
16
18
|
read_info,
|
|
17
|
-
list_layers,
|
|
18
|
-
get_gdal_config_option,
|
|
19
19
|
set_gdal_config_options,
|
|
20
|
+
vsi_listtree,
|
|
20
21
|
)
|
|
22
|
+
from pyogrio.errors import DataLayerError, DataSourceError, FieldError
|
|
21
23
|
from pyogrio.raw import open_arrow, read_arrow, write, write_arrow
|
|
22
|
-
from pyogrio.errors import DataSourceError, FieldError, DataLayerError
|
|
23
24
|
from pyogrio.tests.conftest import (
|
|
24
25
|
ALL_EXTS,
|
|
25
|
-
DRIVERS,
|
|
26
26
|
DRIVER_EXT,
|
|
27
|
+
DRIVERS,
|
|
27
28
|
requires_arrow_write_api,
|
|
28
29
|
requires_pyarrow_api,
|
|
29
30
|
)
|
|
30
31
|
|
|
32
|
+
import pytest
|
|
33
|
+
|
|
31
34
|
try:
|
|
32
35
|
import pandas as pd
|
|
33
|
-
from pandas.testing import assert_frame_equal, assert_index_equal
|
|
34
|
-
from geopandas.testing import assert_geodataframe_equal
|
|
35
|
-
|
|
36
36
|
import pyarrow
|
|
37
|
+
|
|
38
|
+
from geopandas.testing import assert_geodataframe_equal
|
|
39
|
+
from pandas.testing import assert_frame_equal, assert_index_equal
|
|
37
40
|
except ImportError:
|
|
38
41
|
pass
|
|
39
42
|
|
|
@@ -130,24 +133,24 @@ def test_read_arrow_ignore_geometry(naturalearth_lowres):
|
|
|
130
133
|
assert_frame_equal(result, expected)
|
|
131
134
|
|
|
132
135
|
|
|
133
|
-
def test_read_arrow_nested_types(
|
|
136
|
+
def test_read_arrow_nested_types(list_field_values_file):
|
|
134
137
|
# with arrow, list types are supported
|
|
135
|
-
result = read_dataframe(
|
|
138
|
+
result = read_dataframe(list_field_values_file, use_arrow=True)
|
|
136
139
|
assert "list_int64" in result.columns
|
|
137
140
|
assert result["list_int64"][0].tolist() == [0, 1]
|
|
138
141
|
|
|
139
142
|
|
|
140
|
-
def test_read_arrow_to_pandas_kwargs(
|
|
143
|
+
def test_read_arrow_to_pandas_kwargs(no_geometry_file):
|
|
141
144
|
# with arrow, list types are supported
|
|
142
145
|
arrow_to_pandas_kwargs = {"strings_to_categorical": True}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
+
df = read_dataframe(
|
|
147
|
+
no_geometry_file,
|
|
148
|
+
read_geometry=False,
|
|
146
149
|
use_arrow=True,
|
|
147
150
|
arrow_to_pandas_kwargs=arrow_to_pandas_kwargs,
|
|
148
151
|
)
|
|
149
|
-
assert "
|
|
150
|
-
assert
|
|
152
|
+
assert df.col.dtype.name == "category"
|
|
153
|
+
assert np.array_equal(df.col.values.categories, ["a", "b", "c"])
|
|
151
154
|
|
|
152
155
|
|
|
153
156
|
def test_read_arrow_raw(naturalearth_lowres):
|
|
@@ -160,6 +163,10 @@ def test_read_arrow_vsi(naturalearth_lowres_vsi):
|
|
|
160
163
|
table = read_arrow(naturalearth_lowres_vsi[1])[1]
|
|
161
164
|
assert len(table) == 177
|
|
162
165
|
|
|
166
|
+
# Check temp file was cleaned up. Filter to files created by pyogrio, as GDAL keeps
|
|
167
|
+
# cache files in /vsimem/.
|
|
168
|
+
assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
|
|
169
|
+
|
|
163
170
|
|
|
164
171
|
def test_read_arrow_bytes(geojson_bytes):
|
|
165
172
|
meta, table = read_arrow(geojson_bytes)
|
|
@@ -167,6 +174,18 @@ def test_read_arrow_bytes(geojson_bytes):
|
|
|
167
174
|
assert meta["fields"].shape == (5,)
|
|
168
175
|
assert len(table) == 3
|
|
169
176
|
|
|
177
|
+
# Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
|
|
178
|
+
assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_read_arrow_nonseekable_bytes(nonseekable_bytes):
|
|
182
|
+
meta, table = read_arrow(nonseekable_bytes)
|
|
183
|
+
assert meta["fields"].shape == (0,)
|
|
184
|
+
assert len(table) == 1
|
|
185
|
+
|
|
186
|
+
# Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
|
|
187
|
+
assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
|
|
188
|
+
|
|
170
189
|
|
|
171
190
|
def test_read_arrow_filelike(geojson_filelike):
|
|
172
191
|
meta, table = read_arrow(geojson_filelike)
|
|
@@ -174,6 +193,9 @@ def test_read_arrow_filelike(geojson_filelike):
|
|
|
174
193
|
assert meta["fields"].shape == (5,)
|
|
175
194
|
assert len(table) == 3
|
|
176
195
|
|
|
196
|
+
# Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
|
|
197
|
+
assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
|
|
198
|
+
|
|
177
199
|
|
|
178
200
|
def test_open_arrow_pyarrow(naturalearth_lowres):
|
|
179
201
|
with open_arrow(naturalearth_lowres, use_pyarrow=True) as (meta, reader):
|
|
@@ -183,7 +205,7 @@ def test_open_arrow_pyarrow(naturalearth_lowres):
|
|
|
183
205
|
|
|
184
206
|
|
|
185
207
|
def test_open_arrow_batch_size(naturalearth_lowres):
|
|
186
|
-
|
|
208
|
+
_, table = read_arrow(naturalearth_lowres)
|
|
187
209
|
batch_size = math.ceil(len(table) / 2)
|
|
188
210
|
|
|
189
211
|
with open_arrow(naturalearth_lowres, batch_size=batch_size, use_pyarrow=True) as (
|
|
@@ -289,14 +311,15 @@ def use_arrow_context():
|
|
|
289
311
|
del os.environ["PYOGRIO_USE_ARROW"]
|
|
290
312
|
|
|
291
313
|
|
|
292
|
-
def test_enable_with_environment_variable(
|
|
314
|
+
def test_enable_with_environment_variable(list_field_values_file):
|
|
293
315
|
# list types are only supported with arrow, so don't work by default and work
|
|
294
316
|
# when arrow is enabled through env variable
|
|
295
|
-
result = read_dataframe(
|
|
317
|
+
result = read_dataframe(list_field_values_file)
|
|
296
318
|
assert "list_int64" not in result.columns
|
|
297
319
|
|
|
298
320
|
with use_arrow_context():
|
|
299
|
-
result = read_dataframe(
|
|
321
|
+
result = read_dataframe(list_field_values_file)
|
|
322
|
+
|
|
300
323
|
assert "list_int64" in result.columns
|
|
301
324
|
|
|
302
325
|
|
|
@@ -483,6 +506,88 @@ def test_write_geojson(tmp_path, naturalearth_lowres):
|
|
|
483
506
|
)
|
|
484
507
|
|
|
485
508
|
|
|
509
|
+
@requires_arrow_write_api
|
|
510
|
+
@pytest.mark.skipif(
|
|
511
|
+
__gdal_version__ < (3, 6, 0),
|
|
512
|
+
reason="OpenFileGDB write support only available for GDAL >= 3.6.0",
|
|
513
|
+
)
|
|
514
|
+
@pytest.mark.parametrize(
|
|
515
|
+
"write_int64",
|
|
516
|
+
[
|
|
517
|
+
False,
|
|
518
|
+
pytest.param(
|
|
519
|
+
True,
|
|
520
|
+
marks=pytest.mark.skipif(
|
|
521
|
+
__gdal_version__ < (3, 9, 0),
|
|
522
|
+
reason="OpenFileGDB write support for int64 values for GDAL >= 3.9.0",
|
|
523
|
+
),
|
|
524
|
+
),
|
|
525
|
+
],
|
|
526
|
+
)
|
|
527
|
+
def test_write_openfilegdb(tmp_path, write_int64):
|
|
528
|
+
expected_field_data = [
|
|
529
|
+
np.array([True, False, True], dtype="bool"),
|
|
530
|
+
np.array([1, 2, 3], dtype="int16"),
|
|
531
|
+
np.array([1, 2, 3], dtype="int32"),
|
|
532
|
+
np.array([1, 2, 3], dtype="int64"),
|
|
533
|
+
np.array([1, 2, 3], dtype="float32"),
|
|
534
|
+
np.array([1, 2, 3], dtype="float64"),
|
|
535
|
+
]
|
|
536
|
+
|
|
537
|
+
table = pa.table(
|
|
538
|
+
{
|
|
539
|
+
"geometry": points,
|
|
540
|
+
**{field.dtype.name: field for field in expected_field_data},
|
|
541
|
+
}
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
filename = tmp_path / "test.gdb"
|
|
545
|
+
|
|
546
|
+
expected_meta = {"crs": "EPSG:4326"}
|
|
547
|
+
|
|
548
|
+
# int64 is not supported without additional config: https://gdal.org/en/latest/drivers/vector/openfilegdb.html#bit-integer-field-support
|
|
549
|
+
# it is converted to float64 by default and raises a warning
|
|
550
|
+
# (for GDAL >= 3.9.0 only)
|
|
551
|
+
write_params = (
|
|
552
|
+
{"TARGET_ARCGIS_VERSION": "ARCGIS_PRO_3_2_OR_LATER"} if write_int64 else {}
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
if write_int64 or __gdal_version__ < (3, 9, 0):
|
|
556
|
+
ctx = contextlib.nullcontext()
|
|
557
|
+
else:
|
|
558
|
+
ctx = pytest.warns(
|
|
559
|
+
RuntimeWarning, match="Integer64 will be written as a Float64"
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
with ctx:
|
|
563
|
+
write_arrow(
|
|
564
|
+
table,
|
|
565
|
+
filename,
|
|
566
|
+
driver="OpenFileGDB",
|
|
567
|
+
geometry_type="Point",
|
|
568
|
+
geometry_name="geometry",
|
|
569
|
+
**expected_meta,
|
|
570
|
+
**write_params,
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
meta, table = read_arrow(filename)
|
|
574
|
+
|
|
575
|
+
if not write_int64:
|
|
576
|
+
expected_field_data[3] = expected_field_data[3].astype("float64")
|
|
577
|
+
|
|
578
|
+
# bool types are converted to int32
|
|
579
|
+
expected_field_data[0] = expected_field_data[0].astype("int32")
|
|
580
|
+
|
|
581
|
+
assert meta["crs"] == expected_meta["crs"]
|
|
582
|
+
|
|
583
|
+
# NOTE: geometry name is set to "SHAPE" by GDAL
|
|
584
|
+
assert np.array_equal(table[meta["geometry_name"]], points)
|
|
585
|
+
for i in range(len(expected_field_data)):
|
|
586
|
+
values = table[table.schema.names[i]].to_numpy()
|
|
587
|
+
assert values.dtype == expected_field_data[i].dtype
|
|
588
|
+
assert np.array_equal(values, expected_field_data[i])
|
|
589
|
+
|
|
590
|
+
|
|
486
591
|
@pytest.mark.parametrize(
|
|
487
592
|
"driver",
|
|
488
593
|
{
|
|
@@ -538,6 +643,9 @@ def test_write_append(request, tmp_path, naturalearth_lowres, ext):
|
|
|
538
643
|
pytest.mark.xfail(reason="Bugs with append when writing Arrow to GeoJSON")
|
|
539
644
|
)
|
|
540
645
|
|
|
646
|
+
if ext == ".gpkg.zip":
|
|
647
|
+
pytest.skip("Append is not supported for .gpkg.zip")
|
|
648
|
+
|
|
541
649
|
meta, table = read_arrow(naturalearth_lowres)
|
|
542
650
|
|
|
543
651
|
# coerce output layer to generic Geometry to avoid mixed type errors
|
|
@@ -877,6 +985,9 @@ def test_write_memory_driver_required(naturalearth_lowres):
|
|
|
877
985
|
geometry_name=meta["geometry_name"] or "wkb_geometry",
|
|
878
986
|
)
|
|
879
987
|
|
|
988
|
+
# Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
|
|
989
|
+
assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
|
|
990
|
+
|
|
880
991
|
|
|
881
992
|
@requires_arrow_write_api
|
|
882
993
|
@pytest.mark.parametrize("driver", ["ESRI Shapefile", "OpenFileGDB"])
|
|
@@ -945,6 +1056,48 @@ def test_write_memory_existing_unsupported(naturalearth_lowres):
|
|
|
945
1056
|
)
|
|
946
1057
|
|
|
947
1058
|
|
|
1059
|
+
@requires_arrow_write_api
|
|
1060
|
+
def test_write_open_file_handle(tmp_path, naturalearth_lowres):
|
|
1061
|
+
"""Verify that writing to an open file handle is not currently supported"""
|
|
1062
|
+
|
|
1063
|
+
meta, table = read_arrow(naturalearth_lowres, max_features=1)
|
|
1064
|
+
meta["geometry_type"] = "MultiPolygon"
|
|
1065
|
+
|
|
1066
|
+
# verify it fails for regular file handle
|
|
1067
|
+
with pytest.raises(
|
|
1068
|
+
NotImplementedError, match="writing to an open file handle is not yet supported"
|
|
1069
|
+
):
|
|
1070
|
+
with open(tmp_path / "test.geojson", "wb") as f:
|
|
1071
|
+
write_arrow(
|
|
1072
|
+
table,
|
|
1073
|
+
f,
|
|
1074
|
+
driver="GeoJSON",
|
|
1075
|
+
layer="test",
|
|
1076
|
+
crs=meta["crs"],
|
|
1077
|
+
geometry_type=meta["geometry_type"],
|
|
1078
|
+
geometry_name=meta["geometry_name"] or "wkb_geometry",
|
|
1079
|
+
)
|
|
1080
|
+
|
|
1081
|
+
# verify it fails for ZipFile
|
|
1082
|
+
with pytest.raises(
|
|
1083
|
+
NotImplementedError, match="writing to an open file handle is not yet supported"
|
|
1084
|
+
):
|
|
1085
|
+
with ZipFile(tmp_path / "test.geojson.zip", "w") as z:
|
|
1086
|
+
with z.open("test.geojson", "w") as f:
|
|
1087
|
+
write_arrow(
|
|
1088
|
+
table,
|
|
1089
|
+
f,
|
|
1090
|
+
driver="GeoJSON",
|
|
1091
|
+
layer="test",
|
|
1092
|
+
crs=meta["crs"],
|
|
1093
|
+
geometry_type=meta["geometry_type"],
|
|
1094
|
+
geometry_name=meta["geometry_name"] or "wkb_geometry",
|
|
1095
|
+
)
|
|
1096
|
+
|
|
1097
|
+
# Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
|
|
1098
|
+
assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
|
|
1099
|
+
|
|
1100
|
+
|
|
948
1101
|
@requires_arrow_write_api
|
|
949
1102
|
def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text):
|
|
950
1103
|
encoding, text = encoded_text
|
|
@@ -1004,13 +1157,17 @@ def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text):
|
|
|
1004
1157
|
|
|
1005
1158
|
@requires_arrow_write_api
|
|
1006
1159
|
def test_encoding_write_layer_option_collision_shapefile(tmp_path, naturalearth_lowres):
|
|
1007
|
-
"""Providing both encoding parameter and ENCODING layer creation option
|
|
1160
|
+
"""Providing both encoding parameter and ENCODING layer creation option
|
|
1161
|
+
(even if blank) is not allowed."""
|
|
1008
1162
|
|
|
1009
1163
|
meta, table = read_arrow(naturalearth_lowres)
|
|
1010
1164
|
|
|
1011
1165
|
with pytest.raises(
|
|
1012
1166
|
ValueError,
|
|
1013
|
-
match=
|
|
1167
|
+
match=(
|
|
1168
|
+
'cannot provide both encoding parameter and "ENCODING" layer creation '
|
|
1169
|
+
"option"
|
|
1170
|
+
),
|
|
1014
1171
|
):
|
|
1015
1172
|
write_arrow(
|
|
1016
1173
|
table,
|