pyogrio 0.8.0__cp312-cp312-win_amd64.whl → 0.10.0__cp312-cp312-win_amd64.whl

pyogrio/tests/conftest.py

@@ -1,7 +1,8 @@
+from io import BytesIO
 from pathlib import Path
 from zipfile import ZIP_DEFLATED, ZipFile
 
-import pytest
+import numpy as np
 
 from pyogrio import (
     __gdal_version_string__,
@@ -13,10 +14,14 @@ from pyogrio._compat import (
     HAS_ARROW_WRITE_API,
     HAS_GDAL_GEOS,
     HAS_PYARROW,
+    HAS_PYPROJ,
     HAS_SHAPELY,
 )
+from pyogrio.core import vsi_rmtree
 from pyogrio.raw import read, write
 
+import pytest
+
 _data_dir = Path(__file__).parent.resolve() / "fixtures"
 
 # mapping of driver extension to driver name for well-supported drivers
@@ -34,6 +39,15 @@ DRIVER_EXT = {driver: ext for ext, driver in DRIVERS.items()}
 
 ALL_EXTS = [".fgb", ".geojson", ".geojsonl", ".gpkg", ".shp"]
 
+START_FID = {
+    ".fgb": 0,
+    ".geojson": 0,
+    ".geojsonl": 0,
+    ".geojsons": 0,
+    ".gpkg": 1,
+    ".shp": 0,
+}
+
 
 def pytest_report_header(config):
     drivers = ", ".join(
@@ -53,6 +67,8 @@ requires_pyarrow_api = pytest.mark.skipif(
     not HAS_ARROW_API or not HAS_PYARROW, reason="GDAL>=3.6 and pyarrow required"
 )
 
+requires_pyproj = pytest.mark.skipif(not HAS_PYPROJ, reason="pyproj required")
+
 requires_arrow_write_api = pytest.mark.skipif(
     not HAS_ARROW_WRITE_API or not HAS_PYARROW,
     reason="GDAL>=3.8 required for Arrow write API",
@@ -110,7 +126,7 @@ def naturalearth_lowres_all_ext(tmp_path, naturalearth_lowres, request):
 
 @pytest.fixture(scope="function")
 def naturalearth_lowres_vsi(tmp_path, naturalearth_lowres):
-    """Wrap naturalearth_lowres as a zip file for vsi tests"""
+    """Wrap naturalearth_lowres as a zip file for VSI tests"""
 
     path = tmp_path / f"{naturalearth_lowres.name}.zip"
     with ZipFile(path, mode="w", compression=ZIP_DEFLATED, compresslevel=5) as out:
@@ -121,29 +137,182 @@ def naturalearth_lowres_vsi(tmp_path, naturalearth_lowres):
     return path, f"/vsizip/{path}/{naturalearth_lowres.name}"
 
 
+@pytest.fixture(scope="function")
+def naturalearth_lowres_vsimem(naturalearth_lowres):
+    """Write naturalearth_lowres to a vsimem file for VSI tests"""
+
+    meta, _, geometry, field_data = read(naturalearth_lowres)
+    name = f"pyogrio_fixture_{naturalearth_lowres.stem}"
+    dst_path = Path(f"/vsimem/{name}/{name}.gpkg")
+    meta["spatial_index"] = False
+    meta["geometry_type"] = "MultiPolygon"
+
+    write(dst_path, geometry, field_data, layer="naturalearth_lowres", **meta)
+    yield dst_path
+
+    vsi_rmtree(dst_path.parent)
+
+
 @pytest.fixture(scope="session")
-def test_fgdb_vsi():
-    return f"/vsizip/{_data_dir}/test_fgdb.gdb.zip"
+def line_zm_file():
+    return _data_dir / "line_zm.gpkg"
 
 
 @pytest.fixture(scope="session")
-def test_gpkg_nulls():
-    return _data_dir / "test_gpkg_nulls.gpkg"
+def curve_file():
+    return _data_dir / "curve.gpkg"
 
 
 @pytest.fixture(scope="session")
-def test_ogr_types_list():
-    return _data_dir / "test_ogr_types_list.geojson"
+def curve_polygon_file():
+    return _data_dir / "curvepolygon.gpkg"
 
 
 @pytest.fixture(scope="session")
-def test_datetime():
-    return _data_dir / "test_datetime.geojson"
+def multisurface_file():
+    return _data_dir / "multisurface.gpkg"
 
 
 @pytest.fixture(scope="session")
-def test_datetime_tz():
-    return _data_dir / "test_datetime_tz.geojson"
+def test_gpkg_nulls():
+    return _data_dir / "test_gpkg_nulls.gpkg"
+
+
+@pytest.fixture(scope="function")
+def no_geometry_file(tmp_path):
+    # create a GPKG layer that does not include geometry
+    filename = tmp_path / "test_no_geometry.gpkg"
+    write(
+        filename,
+        layer="no_geometry",
+        geometry=None,
+        field_data=[np.array(["a", "b", "c"])],
+        fields=["col"],
+    )
+
+    return filename
+
+
+@pytest.fixture(scope="function")
+def list_field_values_file(tmp_path):
+    # Create a GeoJSON file with list values in a property
+    list_geojson = """{
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "properties": { "int64": 1, "list_int64": [0, 1] },
+                "geometry": { "type": "Point", "coordinates": [0, 2] }
+            },
+            {
+                "type": "Feature",
+                "properties": { "int64": 2, "list_int64": [2, 3] },
+                "geometry": { "type": "Point", "coordinates": [1, 2] }
+            },
+            {
+                "type": "Feature",
+                "properties": { "int64": 3, "list_int64": [4, 5] },
+                "geometry": { "type": "Point", "coordinates": [2, 2] }
+            },
+            {
+                "type": "Feature",
+                "properties": { "int64": 4, "list_int64": [6, 7] },
+                "geometry": { "type": "Point", "coordinates": [3, 2] }
+            },
+            {
+                "type": "Feature",
+                "properties": { "int64": 5, "list_int64": [8, 9] },
+                "geometry": { "type": "Point", "coordinates": [4, 2] }
+            }
+        ]
+    }"""
+
+    filename = tmp_path / "test_ogr_types_list.geojson"
+    with open(filename, "w") as f:
+        _ = f.write(list_geojson)
+
+    return filename
+
+
+@pytest.fixture(scope="function")
+def nested_geojson_file(tmp_path):
+    # create GeoJSON file with nested properties
+    nested_geojson = """{
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [0, 0]
+                },
+                "properties": {
+                    "top_level": "A",
+                    "intermediate_level": {
+                        "bottom_level": "B"
+                    }
+                }
+            }
+        ]
+    }"""
+
+    filename = tmp_path / "test_nested.geojson"
+    with open(filename, "w") as f:
+        _ = f.write(nested_geojson)
+
+    return filename
+
+
+@pytest.fixture(scope="function")
+def datetime_file(tmp_path):
+    # create GeoJSON file with millisecond precision
+    datetime_geojson = """{
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "properties": { "col": "2020-01-01T09:00:00.123" },
+                "geometry": { "type": "Point", "coordinates": [1, 1] }
+            },
+            {
+                "type": "Feature",
+                "properties": { "col": "2020-01-01T10:00:00" },
+                "geometry": { "type": "Point", "coordinates": [2, 2] }
+            }
+        ]
+    }"""
+
+    filename = tmp_path / "test_datetime.geojson"
+    with open(filename, "w") as f:
+        _ = f.write(datetime_geojson)
+
+    return filename
+
+
+@pytest.fixture(scope="function")
+def datetime_tz_file(tmp_path):
+    # create GeoJSON file with datetimes with timezone
+    datetime_tz_geojson = """{
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "properties": { "datetime_col": "2020-01-01T09:00:00.123-05:00" },
+                "geometry": { "type": "Point", "coordinates": [1, 1] }
+            },
+            {
+                "type": "Feature",
+                "properties": { "datetime_col": "2020-01-01T10:00:00-05:00" },
+                "geometry": { "type": "Point", "coordinates": [2, 2] }
+            }
+        ]
+    }"""
+
+    filename = tmp_path / "test_datetime_tz.geojson"
+    with open(filename, "w") as f:
+        f.write(datetime_tz_geojson)
+
+    return filename
 
 
 @pytest.fixture(scope="function")
@@ -178,6 +347,31 @@ def geojson_filelike(tmp_path):
         yield f
 
 
+@pytest.fixture(scope="function")
+def nonseekable_bytes(tmp_path):
+    # mock a non-seekable byte stream, such as a zstandard handle
+    class NonSeekableBytesIO(BytesIO):
+        def seekable(self):
+            return False
+
+        def seek(self, *args, **kwargs):
+            raise OSError("cannot seek")
+
+    # wrap GeoJSON into a non-seekable BytesIO
+    geojson = """{
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "properties": { },
+                "geometry": { "type": "Point", "coordinates": [1, 1] }
+            }
+        ]
+    }"""
+
+    return NonSeekableBytesIO(geojson.encode("UTF-8"))
+
+
 @pytest.fixture(
     scope="session",
     params=[

pyogrio/tests/fixtures/README.md

@@ -1,13 +1,28 @@
 # Test datasets
 
-## Natural Earth lowres
+## Obtaining / creating test datasets
 
-`naturalearth_lowres.shp` was copied from GeoPandas.
+If a test dataset can be created in code, do that instead. If it is used in a
+single test, create the test dataset as part of that test. If it is used in
+more than a single test, add it to `pyogrio/tests/conftest.py` instead, as a
+function-scoped test fixture.
+
+If you need to obtain 3rd party test files:
+
+-   add a section below that describes the source location and processing steps
+    to derive that dataset
+-   make sure the license is compatible with including in Pyogrio (public domain or open-source)
+    and record that license below
+
+Please keep the test files no larger than necessary to use in tests.
 
-## FGDB test dataset
+## Included test datasets
+
+### Natural Earth lowres
+
+`naturalearth_lowres.shp` was copied from GeoPandas.
 
-`test_fgdb.gdb.zip`
-Downloaded from http://trac.osgeo.org/gdal/raw-attachment/wiki/FileGDB/test_fgdb.gdb.zip
+License: public domain
 
 ### GPKG test dataset with null values
 
@@ -75,15 +90,19 @@ NOTE: Reading boolean values into GeoPandas using Fiona backend treats those
 values as `None` and column dtype as `object`; Pyogrio treats those values as
 `np.nan` and column dtype as `float64`.
 
-### GPKG test with MultiSurface
-
-This was extracted from https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_0308_HU4_GDB.zip
-`NHDWaterbody` layer using ogr2ogr:
-
-```bash
-ogr2ogr test_mixed_surface.gpkg NHDPLUS_H_0308_HU4_GDB.gdb NHDWaterbody -where '"NHDPlusID" = 15000300070477' -select "NHDPlusID"
-```
+License: same as Pyogrio
 
 ### OSM PBF test
 
 This was downloaded from https://github.com/openstreetmap/OSM-binary/blob/master/resources/sample.pbf
+
+License: [Open Data Commons Open Database License (ODbL)](https://opendatacommons.org/licenses/odbl/)
+
+### Test files for geometry types that are downgraded on read
+
+`line_zm.gpkg` was created using QGIS to digitize a LineString GPKG layer with Z and M enabled. Downgraded to LineString Z on read.
+`curve.gpkg` was created using QGIS to digitize a Curve GPKG layer. Downgraded to LineString on read.
+`curvepolygon.gpkg` was created using QGIS to digitize a CurvePolygon GPKG layer. Downgraded to Polygon on read.
+`multisurface.gpkg` was created using QGIS to digitize a MultiSurface GPKG layer. Downgraded to MultiPolygon on read.
+
+License: same as Pyogrio

pyogrio/tests/test_arrow.py

@@ -1,39 +1,42 @@
 import contextlib
-from io import BytesIO
 import json
 import math
 import os
-from packaging.version import Version
 import sys
+from io import BytesIO
+from packaging.version import Version
+from zipfile import ZipFile
 
-import pytest
 import numpy as np
 
 import pyogrio
 from pyogrio import (
     __gdal_version__,
+    get_gdal_config_option,
+    list_layers,
     read_dataframe,
     read_info,
-    list_layers,
-    get_gdal_config_option,
     set_gdal_config_options,
+    vsi_listtree,
 )
+from pyogrio.errors import DataLayerError, DataSourceError, FieldError
 from pyogrio.raw import open_arrow, read_arrow, write, write_arrow
-from pyogrio.errors import DataSourceError, FieldError, DataLayerError
 from pyogrio.tests.conftest import (
     ALL_EXTS,
-    DRIVERS,
     DRIVER_EXT,
+    DRIVERS,
     requires_arrow_write_api,
     requires_pyarrow_api,
 )
 
+import pytest
+
 try:
     import pandas as pd
-    from pandas.testing import assert_frame_equal, assert_index_equal
-    from geopandas.testing import assert_geodataframe_equal
-
     import pyarrow
+
+    from geopandas.testing import assert_geodataframe_equal
+    from pandas.testing import assert_frame_equal, assert_index_equal
 except ImportError:
     pass
 
@@ -130,24 +133,24 @@ def test_read_arrow_ignore_geometry(naturalearth_lowres):
     assert_frame_equal(result, expected)
 
 
-def test_read_arrow_nested_types(test_ogr_types_list):
+def test_read_arrow_nested_types(list_field_values_file):
     # with arrow, list types are supported
-    result = read_dataframe(test_ogr_types_list, use_arrow=True)
+    result = read_dataframe(list_field_values_file, use_arrow=True)
     assert "list_int64" in result.columns
     assert result["list_int64"][0].tolist() == [0, 1]
 
 
-def test_read_arrow_to_pandas_kwargs(test_fgdb_vsi):
+def test_read_arrow_to_pandas_kwargs(no_geometry_file):
     # with arrow, list types are supported
     arrow_to_pandas_kwargs = {"strings_to_categorical": True}
-    result = read_dataframe(
-        test_fgdb_vsi,
-        layer="basetable_2",
+    df = read_dataframe(
+        no_geometry_file,
+        read_geometry=False,
         use_arrow=True,
         arrow_to_pandas_kwargs=arrow_to_pandas_kwargs,
     )
-    assert "SEGMENT_NAME" in result.columns
-    assert result["SEGMENT_NAME"].dtype.name == "category"
+    assert df.col.dtype.name == "category"
+    assert np.array_equal(df.col.values.categories, ["a", "b", "c"])
 
 
 def test_read_arrow_raw(naturalearth_lowres):
@@ -160,6 +163,10 @@ def test_read_arrow_vsi(naturalearth_lowres_vsi):
     table = read_arrow(naturalearth_lowres_vsi[1])[1]
     assert len(table) == 177
 
+    # Check temp file was cleaned up. Filter to files created by pyogrio, as GDAL keeps
+    # cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 def test_read_arrow_bytes(geojson_bytes):
     meta, table = read_arrow(geojson_bytes)
@@ -167,6 +174,18 @@ def test_read_arrow_bytes(geojson_bytes):
     assert meta["fields"].shape == (5,)
     assert len(table) == 3
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
+
+def test_read_arrow_nonseekable_bytes(nonseekable_bytes):
+    meta, table = read_arrow(nonseekable_bytes)
+    assert meta["fields"].shape == (0,)
+    assert len(table) == 1
+
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 def test_read_arrow_filelike(geojson_filelike):
     meta, table = read_arrow(geojson_filelike)
@@ -174,6 +193,9 @@ def test_read_arrow_filelike(geojson_filelike):
     assert meta["fields"].shape == (5,)
     assert len(table) == 3
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 def test_open_arrow_pyarrow(naturalearth_lowres):
     with open_arrow(naturalearth_lowres, use_pyarrow=True) as (meta, reader):
@@ -183,7 +205,7 @@ def test_open_arrow_pyarrow(naturalearth_lowres):
 
 
 def test_open_arrow_batch_size(naturalearth_lowres):
-    meta, table = read_arrow(naturalearth_lowres)
+    _, table = read_arrow(naturalearth_lowres)
     batch_size = math.ceil(len(table) / 2)
 
     with open_arrow(naturalearth_lowres, batch_size=batch_size, use_pyarrow=True) as (
@@ -289,14 +311,15 @@ def use_arrow_context():
         del os.environ["PYOGRIO_USE_ARROW"]
 
 
-def test_enable_with_environment_variable(test_ogr_types_list):
+def test_enable_with_environment_variable(list_field_values_file):
     # list types are only supported with arrow, so don't work by default and work
     # when arrow is enabled through env variable
-    result = read_dataframe(test_ogr_types_list)
+    result = read_dataframe(list_field_values_file)
     assert "list_int64" not in result.columns
 
     with use_arrow_context():
-        result = read_dataframe(test_ogr_types_list)
+        result = read_dataframe(list_field_values_file)
+
     assert "list_int64" in result.columns
 
 
@@ -483,6 +506,88 @@ def test_write_geojson(tmp_path, naturalearth_lowres):
     )
 
 
+@requires_arrow_write_api
+@pytest.mark.skipif(
+    __gdal_version__ < (3, 6, 0),
+    reason="OpenFileGDB write support only available for GDAL >= 3.6.0",
+)
+@pytest.mark.parametrize(
+    "write_int64",
+    [
+        False,
+        pytest.param(
+            True,
+            marks=pytest.mark.skipif(
+                __gdal_version__ < (3, 9, 0),
+                reason="OpenFileGDB write support for int64 values for GDAL >= 3.9.0",
+            ),
+        ),
+    ],
+)
+def test_write_openfilegdb(tmp_path, write_int64):
+    expected_field_data = [
+        np.array([True, False, True], dtype="bool"),
+        np.array([1, 2, 3], dtype="int16"),
+        np.array([1, 2, 3], dtype="int32"),
+        np.array([1, 2, 3], dtype="int64"),
+        np.array([1, 2, 3], dtype="float32"),
+        np.array([1, 2, 3], dtype="float64"),
+    ]
+
+    table = pa.table(
+        {
+            "geometry": points,
+            **{field.dtype.name: field for field in expected_field_data},
+        }
+    )
+
+    filename = tmp_path / "test.gdb"
+
+    expected_meta = {"crs": "EPSG:4326"}
+
+    # int64 is not supported without additional config: https://gdal.org/en/latest/drivers/vector/openfilegdb.html#bit-integer-field-support
+    # it is converted to float64 by default and raises a warning
+    # (for GDAL >= 3.9.0 only)
+    write_params = (
+        {"TARGET_ARCGIS_VERSION": "ARCGIS_PRO_3_2_OR_LATER"} if write_int64 else {}
+    )
+
+    if write_int64 or __gdal_version__ < (3, 9, 0):
+        ctx = contextlib.nullcontext()
+    else:
+        ctx = pytest.warns(
+            RuntimeWarning, match="Integer64 will be written as a Float64"
+        )
+
+    with ctx:
+        write_arrow(
+            table,
+            filename,
+            driver="OpenFileGDB",
+            geometry_type="Point",
+            geometry_name="geometry",
+            **expected_meta,
+            **write_params,
+        )
+
+    meta, table = read_arrow(filename)
+
+    if not write_int64:
+        expected_field_data[3] = expected_field_data[3].astype("float64")
+
+    # bool types are converted to int32
+    expected_field_data[0] = expected_field_data[0].astype("int32")
+
+    assert meta["crs"] == expected_meta["crs"]
+
+    # NOTE: geometry name is set to "SHAPE" by GDAL
+    assert np.array_equal(table[meta["geometry_name"]], points)
+    for i in range(len(expected_field_data)):
+        values = table[table.schema.names[i]].to_numpy()
+        assert values.dtype == expected_field_data[i].dtype
+        assert np.array_equal(values, expected_field_data[i])
+
+
 @pytest.mark.parametrize(
     "driver",
     {
@@ -877,6 +982,9 @@ def test_write_memory_driver_required(naturalearth_lowres):
             geometry_name=meta["geometry_name"] or "wkb_geometry",
         )
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 @requires_arrow_write_api
 @pytest.mark.parametrize("driver", ["ESRI Shapefile", "OpenFileGDB"])
@@ -945,6 +1053,48 @@ def test_write_memory_existing_unsupported(naturalearth_lowres):
         )
 
 
+@requires_arrow_write_api
+def test_write_open_file_handle(tmp_path, naturalearth_lowres):
+    """Verify that writing to an open file handle is not currently supported"""
+
+    meta, table = read_arrow(naturalearth_lowres, max_features=1)
+    meta["geometry_type"] = "MultiPolygon"
+
+    # verify it fails for regular file handle
+    with pytest.raises(
+        NotImplementedError, match="writing to an open file handle is not yet supported"
+    ):
+        with open(tmp_path / "test.geojson", "wb") as f:
+            write_arrow(
+                table,
+                f,
+                driver="GeoJSON",
+                layer="test",
+                crs=meta["crs"],
+                geometry_type=meta["geometry_type"],
+                geometry_name=meta["geometry_name"] or "wkb_geometry",
+            )
+
+    # verify it fails for ZipFile
+    with pytest.raises(
+        NotImplementedError, match="writing to an open file handle is not yet supported"
+    ):
+        with ZipFile(tmp_path / "test.geojson.zip", "w") as z:
+            with z.open("test.geojson", "w") as f:
+                write_arrow(
+                    table,
+                    f,
+                    driver="GeoJSON",
+                    layer="test",
+                    crs=meta["crs"],
+                    geometry_type=meta["geometry_type"],
+                    geometry_name=meta["geometry_name"] or "wkb_geometry",
+                )
+
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
+
 @requires_arrow_write_api
 def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text):
     encoding, text = encoded_text
@@ -1004,13 +1154,17 @@ def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text):
 
 @requires_arrow_write_api
 def test_encoding_write_layer_option_collision_shapefile(tmp_path, naturalearth_lowres):
-    """Providing both encoding parameter and ENCODING layer creation option (even if blank) is not allowed"""
+    """Providing both encoding parameter and ENCODING layer creation option
+    (even if blank) is not allowed."""
 
     meta, table = read_arrow(naturalearth_lowres)
 
     with pytest.raises(
         ValueError,
-        match='cannot provide both encoding parameter and "ENCODING" layer creation option',
+        match=(
+            'cannot provide both encoding parameter and "ENCODING" layer creation '
+            "option"
+        ),
     ):
         write_arrow(
             table,