pyogrio 0.9.0__cp39-cp39-manylinux_2_28_aarch64.whl → 0.11.0__cp39-cp39-manylinux_2_28_aarch64.whl

pyogrio/tests/test_geopandas_io.py

@@ -1,14 +1,33 @@
 import contextlib
+import locale
+import warnings
 from datetime import datetime
 from io import BytesIO
-import locale
+from zipfile import ZipFile
 
 import numpy as np
-import pytest
 
-from pyogrio import list_layers, list_drivers, read_info, __gdal_version__
+from pyogrio import (
+    __gdal_version__,
+    list_drivers,
+    list_layers,
+    read_info,
+    set_gdal_config_options,
+    vsi_listtree,
+    vsi_unlink,
+)
+from pyogrio._compat import (
+    GDAL_GE_37,
+    GDAL_GE_311,
+    GDAL_GE_352,
+    HAS_ARROW_WRITE_API,
+    HAS_PYPROJ,
+    PANDAS_GE_15,
+    PANDAS_GE_30,
+    SHAPELY_GE_21,
+)
 from pyogrio.errors import DataLayerError, DataSourceError, FeatureError, GeometryError
-from pyogrio.geopandas import read_dataframe, write_dataframe, PANDAS_GE_20
+from pyogrio.geopandas import PANDAS_GE_20, read_dataframe, write_dataframe
 from pyogrio.raw import (
     DRIVERS_NO_MIXED_DIMENSIONS,
     DRIVERS_NO_MIXED_SINGLE_MULTI,
@@ -16,27 +35,29 @@ from pyogrio.raw import (
 from pyogrio.tests.conftest import (
     ALL_EXTS,
     DRIVERS,
-    requires_pyarrow_api,
+    START_FID,
     requires_arrow_write_api,
     requires_gdal_geos,
+    requires_pyarrow_api,
+    requires_pyproj,
 )
-from pyogrio._compat import PANDAS_GE_15, HAS_ARROW_WRITE_API
 
-try:
-    import pandas as pd
-    from pandas.testing import (
-        assert_frame_equal,
-        assert_index_equal,
-        assert_series_equal,
-    )
+import pytest
 
+try:
     import geopandas as gp
+    import pandas as pd
     from geopandas.array import from_wkt
-    from geopandas.testing import assert_geodataframe_equal
 
     import shapely  # if geopandas is present, shapely is expected to be present
     from shapely.geometry import Point
 
+    from geopandas.testing import assert_geodataframe_equal
+    from pandas.testing import (
+        assert_index_equal,
+        assert_series_equal,
+    )
+
 except ImportError:
     pass
 
@@ -82,8 +103,20 @@ def spatialite_available(path):
         return False
 
 
-@pytest.mark.parametrize("encoding", ["utf-8", "cp1252", None])
-def test_read_csv_encoding(tmp_path, encoding):
+@pytest.mark.parametrize(
+    "encoding, arrow",
+    [
+        ("utf-8", False),
+        pytest.param("utf-8", True, marks=requires_pyarrow_api),
+        ("cp1252", False),
+        (None, False),
+    ],
+)
+def test_read_csv_encoding(tmp_path, encoding, arrow):
+    """ "Test reading CSV files with different encodings.
+
+    Arrow only supports utf-8 encoding.
+    """
     # Write csv test file. Depending on the os this will be written in a different
     # encoding: for linux and macos this is utf-8, for windows it is cp1252.
     csv_path = tmp_path / "test.csv"
@@ -94,7 +127,7 @@ def test_read_csv_encoding(tmp_path, encoding):
     # Read csv. The data should be read with the same default encoding as the csv file
     # was written in, but should have been converted to utf-8 in the dataframe returned.
     # Hence, the asserts below, with strings in utf-8, be OK.
-    df = read_dataframe(csv_path, encoding=encoding)
+    df = read_dataframe(csv_path, encoding=encoding, use_arrow=arrow)
 
     assert len(df) == 1
     assert df.columns.tolist() == ["näme", "city"]
@@ -106,25 +139,36 @@ def test_read_csv_encoding(tmp_path, encoding):
     locale.getpreferredencoding().upper() == "UTF-8",
     reason="test requires non-UTF-8 default platform",
 )
-def test_read_csv_platform_encoding(tmp_path):
-    """verify that read defaults to platform encoding; only works on Windows (CP1252)"""
+def test_read_csv_platform_encoding(tmp_path, use_arrow):
+    """Verify that read defaults to platform encoding; only works on Windows (CP1252).
+
+    When use_arrow=True, reading an non-UTF8 fails.
+    """
     csv_path = tmp_path / "test.csv"
     with open(csv_path, "w", encoding=locale.getpreferredencoding()) as csv:
         csv.write("näme,city\n")
         csv.write("Wilhelm Röntgen,Zürich\n")
 
-    df = read_dataframe(csv_path)
+    if use_arrow:
+        with pytest.raises(
+            DataSourceError,
+            match="; please use_arrow=False",
+        ):
+            df = read_dataframe(csv_path, use_arrow=use_arrow)
+    else:
+        df = read_dataframe(csv_path, use_arrow=use_arrow)
 
-    assert len(df) == 1
-    assert df.columns.tolist() == ["näme", "city"]
-    assert df.city.tolist() == ["Zürich"]
-    assert df.näme.tolist() == ["Wilhelm Röntgen"]
+        assert len(df) == 1
+        assert df.columns.tolist() == ["näme", "city"]
+        assert df.city.tolist() == ["Zürich"]
+        assert df.näme.tolist() == ["Wilhelm Röntgen"]
 
 
 def test_read_dataframe(naturalearth_lowres_all_ext):
     df = read_dataframe(naturalearth_lowres_all_ext)
 
-    assert df.crs == "EPSG:4326"
+    if HAS_PYPROJ:
+        assert df.crs == "EPSG:4326"
     assert len(df) == 177
     assert df.columns.tolist() == [
         "pop_est",
@@ -142,14 +186,13 @@ def test_read_dataframe_vsi(naturalearth_lowres_vsi, use_arrow):
 
 
 @pytest.mark.parametrize(
-    "columns, fid_as_index, exp_len", [(None, False, 2), ([], True, 2), ([], False, 0)]
+    "columns, fid_as_index, exp_len", [(None, False, 3), ([], True, 3), ([], False, 0)]
 )
 def test_read_layer_without_geometry(
-    test_fgdb_vsi, columns, fid_as_index, use_arrow, exp_len
+    no_geometry_file, columns, fid_as_index, use_arrow, exp_len
 ):
     result = read_dataframe(
-        test_fgdb_vsi,
-        layer="basetable",
+        no_geometry_file,
         columns=columns,
         fid_as_index=fid_as_index,
         use_arrow=use_arrow,
@@ -195,38 +238,85 @@ def test_read_no_geometry_no_columns_no_fids(naturalearth_lowres, use_arrow):
         )
 
 
-def test_read_force_2d(test_fgdb_vsi, use_arrow):
-    with pytest.warns(
-        UserWarning, match=r"Measured \(M\) geometry types are not supported"
-    ):
-        df = read_dataframe(test_fgdb_vsi, layer="test_lines", max_features=1)
-        assert df.iloc[0].geometry.has_z
+def test_read_force_2d(tmp_path, use_arrow):
+    filename = tmp_path / "test.gpkg"
+
+    # create a GPKG with 3D point values
+    expected = gp.GeoDataFrame(
+        geometry=[Point(0, 0, 0), Point(1, 1, 0)], crs="EPSG:4326"
+    )
+    write_dataframe(expected, filename)
+
+    df = read_dataframe(filename)
+    assert df.iloc[0].geometry.has_z
+
+    df = read_dataframe(
+        filename,
+        force_2d=True,
+        max_features=1,
+        use_arrow=use_arrow,
+    )
+    assert not df.iloc[0].geometry.has_z
+
+
+@pytest.mark.skipif(
+    not GDAL_GE_352,
+    reason="gdal >= 3.5.2 needed to use OGR_GEOJSON_MAX_OBJ_SIZE with a float value",
+)
+def test_read_geojson_error(naturalearth_lowres_geojson, use_arrow):
+    try:
+        set_gdal_config_options({"OGR_GEOJSON_MAX_OBJ_SIZE": 0.01})
+        with pytest.raises(
+            DataSourceError,
+            match="Failed to read GeoJSON data; .* GeoJSON object too complex",
+        ):
+            read_dataframe(naturalearth_lowres_geojson, use_arrow=use_arrow)
+    finally:
+        set_gdal_config_options({"OGR_GEOJSON_MAX_OBJ_SIZE": None})
 
-        df = read_dataframe(
-            test_fgdb_vsi,
-            layer="test_lines",
-            force_2d=True,
-            max_features=1,
-            use_arrow=use_arrow,
-        )
-        assert not df.iloc[0].geometry.has_z
 
+def test_read_layer(tmp_path, use_arrow):
+    filename = tmp_path / "test.gpkg"
 
-@pytest.mark.filterwarnings("ignore: Measured")
-@pytest.mark.filterwarnings("ignore: More than one layer found in")
-def test_read_layer(test_fgdb_vsi, use_arrow):
-    layers = list_layers(test_fgdb_vsi)
-    kwargs = {"use_arrow": use_arrow, "read_geometry": False, "max_features": 1}
+    # create a multilayer GPKG
+    expected1 = gp.GeoDataFrame(geometry=[Point(0, 0)], crs="EPSG:4326")
+    if use_arrow:
+        # TODO this needs to be fixed on the geopandas side (to ensure the
+        # GeoDataFrame() constructor does this), when use_arrow we already
+        # get columns Index with string dtype
+        expected1.columns = expected1.columns.astype("str")
+    write_dataframe(
+        expected1,
+        filename,
+        layer="layer1",
+    )
 
-    # The first layer is read by default (NOTE: first layer has no features)
-    df = read_dataframe(test_fgdb_vsi, **kwargs)
-    df2 = read_dataframe(test_fgdb_vsi, layer=layers[0][0], **kwargs)
-    assert_frame_equal(df, df2)
+    expected2 = gp.GeoDataFrame(geometry=[Point(1, 1)], crs="EPSG:4326")
+    if use_arrow:
+        expected2.columns = expected2.columns.astype("str")
+    write_dataframe(expected2, filename, layer="layer2", append=True)
 
-    # Reading a specific layer should return that layer.
+    assert np.array_equal(
+        list_layers(filename), [["layer1", "Point"], ["layer2", "Point"]]
+    )
+
+    kwargs = {"use_arrow": use_arrow, "max_features": 1}
+
+    # The first layer is read by default, which will warn when there are multiple
+    # layers
+    with pytest.warns(UserWarning, match="More than one layer found"):
+        df = read_dataframe(filename, **kwargs)
+
+    assert_geodataframe_equal(df, expected1)
+
+    # Reading a specific layer by name should return that layer.
     # Detected here by a known column.
-    df = read_dataframe(test_fgdb_vsi, layer="test_lines", **kwargs)
-    assert "RIVER_MILE" in df.columns
+    df = read_dataframe(filename, layer="layer2", **kwargs)
+    assert_geodataframe_equal(df, expected2)
+
+    # Reading a specific layer by index should return that layer
+    df = read_dataframe(filename, layer=1, **kwargs)
+    assert_geodataframe_equal(df, expected2)
 
 
 def test_read_layer_invalid(naturalearth_lowres_all_ext, use_arrow):
@@ -234,22 +324,19 @@ def test_read_layer_invalid(naturalearth_lowres_all_ext, use_arrow):
         read_dataframe(naturalearth_lowres_all_ext, layer="wrong", use_arrow=use_arrow)
 
 
-@pytest.mark.filterwarnings("ignore: Measured")
-def test_read_datetime(test_fgdb_vsi, use_arrow):
-    df = read_dataframe(
-        test_fgdb_vsi, layer="test_lines", use_arrow=use_arrow, max_features=1
-    )
+def test_read_datetime(datetime_file, use_arrow):
+    df = read_dataframe(datetime_file, use_arrow=use_arrow)
     if PANDAS_GE_20:
         # starting with pandas 2.0, it preserves the passed datetime resolution
-        assert df.SURVEY_DAT.dtype.name == "datetime64[ms]"
+        assert df.col.dtype.name == "datetime64[ms]"
     else:
-        assert df.SURVEY_DAT.dtype.name == "datetime64[ns]"
+        assert df.col.dtype.name == "datetime64[ns]"
 
 
 @pytest.mark.filterwarnings("ignore: Non-conformant content for record 1 in column ")
 @pytest.mark.requires_arrow_write_api
-def test_read_datetime_tz(test_datetime_tz, tmp_path, use_arrow):
-    df = read_dataframe(test_datetime_tz)
+def test_read_datetime_tz(datetime_tz_file, tmp_path, use_arrow):
+    df = read_dataframe(datetime_tz_file)
     # Make the index non-consecutive to test this case as well. Added for issue
     # https://github.com/geopandas/pyogrio/issues/324
     df = df.set_index(np.array([0, 2]))
@@ -319,14 +406,17 @@ def test_read_write_datetime_tz_with_nulls(tmp_path, use_arrow):
     assert_geodataframe_equal(df, result)
 
 
-def test_read_null_values(test_fgdb_vsi, use_arrow):
-    df = read_dataframe(
-        test_fgdb_vsi, layer="basetable_2", use_arrow=use_arrow, read_geometry=False
-    )
+def test_read_null_values(tmp_path, use_arrow):
+    filename = tmp_path / "test_null_values_no_geometry.gpkg"
+
+    # create a GPKG with no geometries and only null values
+    expected = pd.DataFrame({"col": [None, None]})
+    write_dataframe(expected, filename)
+
+    df = read_dataframe(filename, use_arrow=use_arrow, read_geometry=False)
 
     # make sure that Null values are preserved
-    assert df.SEGMENT_NAME.isnull().max()
-    assert df.loc[df.SEGMENT_NAME.isnull()].SEGMENT_NAME.iloc[0] is None
+    assert df["col"].isna().all()
 
 
 def test_read_fid_as_index(naturalearth_lowres_all_ext, use_arrow):
@@ -344,12 +434,9 @@ def test_read_fid_as_index(naturalearth_lowres_all_ext, use_arrow):
         fid_as_index=True,
         **kwargs,
     )
-    if naturalearth_lowres_all_ext.suffix in [".gpkg"]:
-        # File format where fid starts at 1
-        assert_index_equal(df.index, pd.Index([3, 4], name="fid"))
-    else:
-        # File format where fid starts at 0
-        assert_index_equal(df.index, pd.Index([2, 3], name="fid"))
+    fids_expected = pd.Index([2, 3], name="fid")
+    fids_expected += START_FID[naturalearth_lowres_all_ext.suffix]
+    assert_index_equal(df.index, fids_expected)
 
 
 def test_read_fid_as_index_only(naturalearth_lowres, use_arrow):
@@ -406,10 +493,17 @@ def test_read_where_invalid(request, naturalearth_lowres_all_ext, use_arrow):
     if use_arrow and naturalearth_lowres_all_ext.suffix == ".gpkg":
         # https://github.com/OSGeo/gdal/issues/8492
         request.node.add_marker(pytest.mark.xfail(reason="GDAL doesn't error for GPGK"))
-    with pytest.raises(ValueError, match="Invalid SQL"):
-        read_dataframe(
-            naturalearth_lowres_all_ext, use_arrow=use_arrow, where="invalid"
-        )
+
+    if naturalearth_lowres_all_ext.suffix == ".gpkg" and __gdal_version__ >= (3, 11, 0):
+        with pytest.raises(DataLayerError, match="no such column"):
+            read_dataframe(
+                naturalearth_lowres_all_ext, use_arrow=use_arrow, where="invalid"
+            )
+    else:
+        with pytest.raises(ValueError, match="Invalid SQL"):
+            read_dataframe(
+                naturalearth_lowres_all_ext, use_arrow=use_arrow, where="invalid"
+            )
 
 
 def test_read_where_ignored_field(naturalearth_lowres, use_arrow):
@@ -605,17 +699,22 @@ def test_read_fids_arrow_warning_old_gdal(naturalearth_lowres_all_ext):
         assert len(df) == 1
 
 
-def test_read_fids_force_2d(test_fgdb_vsi):
-    with pytest.warns(
-        UserWarning, match=r"Measured \(M\) geometry types are not supported"
-    ):
-        df = read_dataframe(test_fgdb_vsi, layer="test_lines", fids=[22])
-        assert len(df) == 1
-        assert df.iloc[0].geometry.has_z
+def test_read_fids_force_2d(tmp_path):
+    filename = tmp_path / "test.gpkg"
 
-        df = read_dataframe(test_fgdb_vsi, layer="test_lines", force_2d=True, fids=[22])
-        assert len(df) == 1
-        assert not df.iloc[0].geometry.has_z
+    # create a GPKG with 3D point values
+    expected = gp.GeoDataFrame(
+        geometry=[Point(0, 0, 0), Point(1, 1, 0)], crs="EPSG:4326"
+    )
+    write_dataframe(expected, filename)
+
+    df = read_dataframe(filename, fids=[1])
+    assert_geodataframe_equal(df, expected.iloc[:1])
+
+    df = read_dataframe(filename, force_2d=True, fids=[1])
+    assert np.array_equal(
+        df.geometry.values, shapely.force_2d(expected.iloc[:1].geometry.values)
+    )
 
 
 @pytest.mark.parametrize("skip_features", [10, 200])
@@ -638,6 +737,13 @@ def test_read_skip_features(naturalearth_lowres_all_ext, use_arrow, skip_feature
     # In .geojsonl the vertices are reordered, so normalize
     is_jsons = ext == ".geojsonl"
 
+    if skip_features == 200 and not use_arrow:
+        # result is an empty dataframe, so no proper dtype inference happens
+        # for the numpy object dtype arrays
+        df[["continent", "name", "iso_a3"]] = df[
+            ["continent", "name", "iso_a3"]
+        ].astype("str")
+
     assert_geodataframe_equal(
         df,
         expected,
@@ -769,7 +875,7 @@ def test_read_sql_invalid(naturalearth_lowres_all_ext, use_arrow):
             )
 
     with pytest.raises(
-        ValueError, match="'sql' paramater cannot be combined with 'layer'"
+        ValueError, match="'sql' parameter cannot be combined with 'layer'"
     ):
         read_dataframe(
             naturalearth_lowres_all_ext,
@@ -906,9 +1012,20 @@ def test_read_sql_dialect_sqlite_gpkg(naturalearth_lowres, use_arrow):
     assert df.iloc[0].geometry.area > area_canada
 
 
-@pytest.mark.parametrize("encoding", ["utf-8", "cp1252", None])
-def test_write_csv_encoding(tmp_path, encoding):
-    """Test if write_dataframe uses the default encoding correctly."""
+@pytest.mark.parametrize(
+    "encoding, arrow",
+    [
+        ("utf-8", False),
+        pytest.param("utf-8", True, marks=requires_arrow_write_api),
+        ("cp1252", False),
+        (None, False),
+    ],
+)
+def test_write_csv_encoding(tmp_path, encoding, arrow):
+    """Test if write_dataframe uses the default encoding correctly.
+
+    Arrow only supports utf-8 encoding.
+    """
     # Write csv test file. Depending on the os this will be written in a different
     # encoding: for linux and macos this is utf-8, for windows it is cp1252.
     csv_path = tmp_path / "test.csv"
@@ -921,12 +1038,12 @@ def test_write_csv_encoding(tmp_path, encoding):
     # same encoding as above.
     df = pd.DataFrame({"näme": ["Wilhelm Röntgen"], "city": ["Zürich"]})
     csv_pyogrio_path = tmp_path / "test_pyogrio.csv"
-    write_dataframe(df, csv_pyogrio_path, encoding=encoding)
+    write_dataframe(df, csv_pyogrio_path, encoding=encoding, use_arrow=arrow)
 
     # Check if the text files written both ways can be read again and give same result.
-    with open(csv_path, "r", encoding=encoding) as csv:
+    with open(csv_path, encoding=encoding) as csv:
         csv_str = csv.read()
-    with open(csv_pyogrio_path, "r", encoding=encoding) as csv_pyogrio:
+    with open(csv_pyogrio_path, encoding=encoding) as csv_pyogrio:
         csv_pyogrio_str = csv_pyogrio.read()
     assert csv_str == csv_pyogrio_str
 
@@ -939,6 +1056,48 @@ def test_write_csv_encoding(tmp_path, encoding):
     assert csv_bytes == csv_pyogrio_bytes
 
 
+@pytest.mark.parametrize(
+    "ext, fid_column, fid_param_value",
+    [
+        (".gpkg", "fid", None),
+        (".gpkg", "FID", None),
+        (".sqlite", "ogc_fid", None),
+        (".gpkg", "fid_custom", "fid_custom"),
+        (".gpkg", "FID_custom", "fid_custom"),
+        (".sqlite", "ogc_fid_custom", "ogc_fid_custom"),
+    ],
+)
+@pytest.mark.requires_arrow_write_api
+def test_write_custom_fids(tmp_path, ext, fid_column, fid_param_value, use_arrow):
+    """Test to specify FIDs to save when writing to a file.
+
+    Saving custom FIDs is only supported for formats that actually store the FID, like
+    e.g. GPKG and SQLite. The fid_column name check is case-insensitive.
+
+    Typically, GDAL supports using a custom FID column for these file formats via a
+    `FID` layer creation option, which is also tested here. If `fid_param_value` is
+    specified (not None), an `fid` parameter is passed to `write_dataframe`, causing
+    GDAL to use the column name specified for the FID.
+    """
+    input_gdf = gp.GeoDataFrame(
+        {fid_column: [5]}, geometry=[shapely.Point(0, 0)], crs="epsg:4326"
+    )
+    kwargs = {}
+    if fid_param_value is not None:
+        kwargs["fid"] = fid_param_value
+    path = tmp_path / f"test{ext}"
+
+    write_dataframe(input_gdf, path, use_arrow=use_arrow, **kwargs)
+
+    assert path.exists()
+    output_gdf = read_dataframe(path, fid_as_index=True, use_arrow=use_arrow)
+    output_gdf = output_gdf.reset_index()
+
+    # pyogrio always sets "fid" as index name with `fid_as_index`
+    expected_gdf = input_gdf.rename(columns={fid_column: "fid"})
+    assert_geodataframe_equal(output_gdf, expected_gdf)
+
+
 @pytest.mark.parametrize("ext", ALL_EXTS)
 @pytest.mark.requires_arrow_write_api
 def test_write_dataframe(tmp_path, naturalearth_lowres, ext, use_arrow):
@@ -960,7 +1119,7 @@ def test_write_dataframe(tmp_path, naturalearth_lowres, ext, use_arrow):
     if DRIVERS[ext] in DRIVERS_NO_MIXED_SINGLE_MULTI:
         assert list(geometry_types) == ["MultiPolygon"]
     else:
-        assert set(geometry_types) == set(["MultiPolygon", "Polygon"])
+        assert set(geometry_types) == {"MultiPolygon", "Polygon"}
 
     # Coordinates are not precisely equal when written to JSON
     # dtypes do not necessarily round-trip precisely through JSON
@@ -1050,18 +1209,79 @@ def test_write_dataframe_index(tmp_path, naturalearth_lowres, use_arrow):
 
 
 @pytest.mark.parametrize("ext", [ext for ext in ALL_EXTS if ext not in ".geojsonl"])
+@pytest.mark.parametrize(
+    "columns, dtype",
+    [
+        ([], None),
+        (["col_int"], np.int64),
+        (["col_float"], np.float64),
+        (["col_object"], object),
+    ],
+)
 @pytest.mark.requires_arrow_write_api
-def test_write_empty_dataframe(tmp_path, ext, use_arrow):
-    expected = gp.GeoDataFrame(geometry=[], crs=4326)
+def test_write_empty_dataframe(tmp_path, ext, columns, dtype, use_arrow):
+    """Test writing dataframe with no rows.
 
+    With use_arrow, object type columns with no rows are converted to null type columns
+    by pyarrow, but null columns are not supported by GDAL. Added to test fix for #513.
+    """
+    expected = gp.GeoDataFrame(geometry=[], columns=columns, dtype=dtype, crs=4326)
     filename = tmp_path / f"test{ext}"
     write_dataframe(expected, filename, use_arrow=use_arrow)
 
     assert filename.exists()
+    df = read_dataframe(filename, use_arrow=use_arrow)
+
+    # Check result
+    # For older pandas versions, the index is created as Object dtype but read as
+    # RangeIndex, so don't check the index dtype in that case.
+    check_index_type = True if PANDAS_GE_20 else False
+    # with pandas 3+ and reading through arrow, we preserve the string dtype
+    # (no proper dtype inference happens for the empty numpy object dtype arrays)
+    if use_arrow and dtype is object:
+        expected["col_object"] = expected["col_object"].astype("str")
+    assert_geodataframe_equal(df, expected, check_index_type=check_index_type)
+
+
+def test_write_empty_geometry(tmp_path):
+    expected = gp.GeoDataFrame({"x": [0]}, geometry=from_wkt(["POINT EMPTY"]), crs=4326)
+    filename = tmp_path / "test.gpkg"
+
+    # Check that no warning is raised with GeoSeries.notna()
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        if not HAS_PYPROJ:
+            warnings.filterwarnings("ignore", message="'crs' was not provided.")
+        write_dataframe(expected, filename)
+    assert filename.exists()
+
+    # Xref GH-436: round-tripping possible with GPKG but not others
     df = read_dataframe(filename)
     assert_geodataframe_equal(df, expected)
 
 
+@pytest.mark.requires_arrow_write_api
+def test_write_None_string_column(tmp_path, use_arrow):
+    """Test pandas object columns with all None values.
+
+    With use_arrow, such columns are converted to null type columns by pyarrow, but null
+    columns are not supported by GDAL. Added to test fix for #513.
+    """
+    gdf = gp.GeoDataFrame({"object_col": [None]}, geometry=[Point(0, 0)], crs=4326)
+    filename = tmp_path / "test.gpkg"
+
+    write_dataframe(gdf, filename, use_arrow=use_arrow)
+    assert filename.exists()
+
+    result_gdf = read_dataframe(filename, use_arrow=use_arrow)
+    if PANDAS_GE_30 and use_arrow:
+        assert result_gdf.object_col.dtype == "str"
+        gdf["object_col"] = gdf["object_col"].astype("str")
+    else:
+        assert result_gdf.object_col.dtype == object
+    assert_geodataframe_equal(result_gdf, gdf)
+
+
 @pytest.mark.parametrize("ext", [".geojsonl", ".geojsons"])
 @pytest.mark.requires_arrow_write_api
 def test_write_read_empty_dataframe_unsupported(tmp_path, ext, use_arrow):
@@ -1161,7 +1381,7 @@ def test_write_dataframe_gdal_options(
         df,
         outfilename2,
         use_arrow=use_arrow,
-        layer_options=dict(spatial_index=spatial_index),
+        layer_options={"spatial_index": spatial_index},
     )
     assert outfilename2.exists() is True
     index_filename2 = tmp_path / "test2.qix"
@@ -1207,7 +1427,7 @@ def test_write_dataframe_gdal_options_dataset(tmp_path, naturalearth_lowres, use
         df,
         test_no_contents_filename2,
         use_arrow=use_arrow,
-        dataset_options=dict(add_gpkg_ogr_contents=False),
+        dataset_options={"add_gpkg_ogr_contents": False},
     )
     assert "gpkg_ogr_contents" not in _get_gpkg_table_names(test_no_contents_filename2)
 
@@ -1320,7 +1540,8 @@ def test_write_dataframe_promote_to_multi_layer_geom_type(
             ".shp",
             None,
             "Point",
-            "Could not add feature to layer at index|Error while writing batch to OGR layer",
+            "Could not add feature to layer at index|Error while writing batch to OGR "
+            "layer",
         ),
     ],
 )
@@ -1443,6 +1664,7 @@ def test_write_dataframe_infer_geometry_with_nulls(tmp_path, geoms, ext, use_arr
     "ignore: You will likely lose important projection information"
 )
 @pytest.mark.requires_arrow_write_api
+@requires_pyproj
 def test_custom_crs_io(tmp_path, naturalearth_lowres_all_ext, use_arrow):
     df = read_dataframe(naturalearth_lowres_all_ext)
     # project Belgium to a custom Albers Equal Area projection
@@ -1465,6 +1687,30 @@ def test_custom_crs_io(tmp_path, naturalearth_lowres_all_ext, use_arrow):
     assert df.crs.equals(expected.crs)
 
 
+@pytest.mark.parametrize("ext", [".gpkg.zip", ".shp.zip", ".shz"])
+@pytest.mark.requires_arrow_write_api
+def test_write_read_zipped_ext(tmp_path, naturalearth_lowres, ext, use_arrow):
+    """Run a basic read and write test on some extra (zipped) extensions."""
+    if ext == ".gpkg.zip" and not GDAL_GE_37:
+        pytest.skip(".gpkg.zip support requires GDAL >= 3.7")
+
+    input_gdf = read_dataframe(naturalearth_lowres)
+    output_path = tmp_path / f"test{ext}"
+
+    write_dataframe(input_gdf, output_path, use_arrow=use_arrow)
+
+    assert output_path.exists()
+    result_gdf = read_dataframe(output_path)
+
+    geometry_types = result_gdf.geometry.type.unique()
+    if DRIVERS[ext] in DRIVERS_NO_MIXED_SINGLE_MULTI:
+        assert list(geometry_types) == ["MultiPolygon"]
+    else:
+        assert set(geometry_types) == {"MultiPolygon", "Polygon"}
+
+    assert_geodataframe_equal(result_gdf, input_gdf, check_index_type=False)
+
+
 def test_write_read_mixed_column_values(tmp_path):
     # use_arrow=True is tested separately below
     mixed_values = ["test", 1.0, 1, datetime.now(), None, np.nan]
@@ -1476,11 +1722,13 @@ def test_write_read_mixed_column_values(tmp_path):
     write_dataframe(test_gdf, output_path)
     output_gdf = read_dataframe(output_path)
     assert len(test_gdf) == len(output_gdf)
-    for idx, value in enumerate(mixed_values):
-        if value in (None, np.nan):
-            assert output_gdf["mixed"][idx] is None
-        else:
-            assert output_gdf["mixed"][idx] == str(value)
+    # mixed values as object dtype are currently written as strings
+    # (but preserving nulls)
+    expected = pd.Series(
+        [str(value) if value not in (None, np.nan) else None for value in mixed_values],
+        name="mixed",
+    )
+    assert_series_equal(output_gdf["mixed"], expected)
 
 
 @requires_arrow_write_api
@@ -1513,8 +1761,24 @@ def test_write_read_null(tmp_path, use_arrow):
     assert pd.isna(result_gdf["float64"][1])
     assert pd.isna(result_gdf["float64"][2])
     assert result_gdf["object_str"][0] == "test"
-    assert result_gdf["object_str"][1] is None
-    assert result_gdf["object_str"][2] is None
+    assert pd.isna(result_gdf["object_str"][1])
+    assert pd.isna(result_gdf["object_str"][2])
+
+
+@pytest.mark.requires_arrow_write_api
+def test_write_read_vsimem(naturalearth_lowres_vsi, use_arrow):
+    path, _ = naturalearth_lowres_vsi
+    mem_path = f"/vsimem/{path.name}"
+
+    input = read_dataframe(path, use_arrow=use_arrow)
+    assert len(input) == 177
+
+    try:
+        write_dataframe(input, mem_path, use_arrow=use_arrow)
+        result = read_dataframe(mem_path, use_arrow=use_arrow)
+        assert len(result) == 177
+    finally:
+        vsi_unlink(mem_path)
 
 
 @pytest.mark.parametrize(
@@ -1529,7 +1793,7 @@ def test_write_read_null(tmp_path, use_arrow):
             ["2.5D MultiLineString", "MultiLineString Z"],
         ),
         (
-            "MultiPolygon Z (((0 0 0, 0 1 0, 1 1 0, 0 0 0)), ((1 1 1, 1 2 1, 2 2 1, 1 1 1)))",
+            "MultiPolygon Z (((0 0 0, 0 1 0, 1 1 0, 0 0 0)), ((1 1 1, 1 2 1, 2 2 1, 1 1 1)))",  # noqa: E501
             ["2.5D MultiPolygon", "MultiPolygon Z"],
         ),
         (
@@ -1572,7 +1836,7 @@ def test_write_geometry_z_types(tmp_path, wkt, geom_types, use_arrow):
             "MultiPolygon Z",
             False,
             [
-                "MultiPolygon Z (((0 0 0, 0 1 0, 1 1 0, 0 0 0)), ((1 1 1, 1 2 1, 2 2 1, 1 1 1)))"
+                "MultiPolygon Z (((0 0 0, 0 1 0, 1 1 0, 0 0 0)), ((1 1 1, 1 2 1, 2 2 1, 1 1 1)))"  # noqa: E501
             ],
         ),
         (
@@ -1642,54 +1906,84 @@ def test_write_geometry_z_types_auto(
 
 
 @pytest.mark.parametrize(
-    "on_invalid, message",
+    "on_invalid, message, expected_wkt",
     [
         (
             "warn",
             "Invalid WKB: geometry is returned as None. IllegalArgumentException: "
-            "Invalid number of points in LinearRing found 2 - must be 0 or >=",
+            "Points of LinearRing do not form a closed linestring",
+            None,
         ),
-        ("raise", "Invalid number of points in LinearRing found 2 - must be 0 or >="),
-        ("ignore", None),
+        ("raise", "Points of LinearRing do not form a closed linestring", None),
+        ("ignore", None, None),
+        ("fix", None, "POLYGON ((0 0, 0 1, 0 0))"),
     ],
 )
-def test_read_invalid_shp(data_dir, use_arrow, on_invalid, message):
+@pytest.mark.filterwarnings("ignore:Non closed ring detected:RuntimeWarning")
+def test_read_invalid_poly_ring(tmp_path, use_arrow, on_invalid, message, expected_wkt):
+    if on_invalid == "fix" and not SHAPELY_GE_21:
+        pytest.skip("on_invalid=fix not available for Shapely < 2.1")
+
     if on_invalid == "raise":
         handler = pytest.raises(shapely.errors.GEOSException, match=message)
     elif on_invalid == "warn":
         handler = pytest.warns(match=message)
-    elif on_invalid == "ignore":
+    elif on_invalid in ("fix", "ignore"):
         handler = contextlib.nullcontext()
     else:
         raise ValueError(f"unknown value for on_invalid: {on_invalid}")
 
+    # create a GeoJSON file with an invalid exterior ring
+    invalid_geojson = """{
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "properties": {},
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [ [ [0, 0], [0, 1] ] ]
+                }
+            }
+        ]
+    }"""
+
+    filename = tmp_path / "test.geojson"
+    with open(filename, "w") as f:
+        _ = f.write(invalid_geojson)
+
     with handler:
         df = read_dataframe(
-            data_dir / "poly_not_enough_points.shp.zip",
+            filename,
             use_arrow=use_arrow,
             on_invalid=on_invalid,
         )
-        df.geometry.isnull().all()
+        if expected_wkt is None:
+            assert df.geometry.iloc[0] is None
+        else:
+            assert df.geometry.iloc[0].wkt == expected_wkt
 
 
-def test_read_multisurface(data_dir, use_arrow):
+def test_read_multisurface(multisurface_file, use_arrow):
     if use_arrow:
+        # TODO: revisit once https://github.com/geopandas/pyogrio/issues/478
+        # is resolved.
+        pytest.skip("Shapely + GEOS 3.13 crashes in from_wkb for this case")
+
         with pytest.raises(shapely.errors.GEOSException):
             # TODO(Arrow)
             # shapely fails parsing the WKB
-            read_dataframe(data_dir / "test_multisurface.gpkg", use_arrow=True)
+            read_dataframe(multisurface_file, use_arrow=True)
     else:
-        df = read_dataframe(data_dir / "test_multisurface.gpkg")
+        df = read_dataframe(multisurface_file)
 
         # MultiSurface should be converted to MultiPolygon
         assert df.geometry.type.tolist() == ["MultiPolygon"]
 
 
-def test_read_dataset_kwargs(data_dir, use_arrow):
-    filename = data_dir / "test_nested.geojson"
-
+def test_read_dataset_kwargs(nested_geojson_file, use_arrow):
     # by default, nested data are not flattened
-    df = read_dataframe(filename, use_arrow=use_arrow)
+    df = read_dataframe(nested_geojson_file, use_arrow=use_arrow)
 
     expected = gp.GeoDataFrame(
         {
@@ -1699,10 +1993,16 @@ def test_read_dataset_kwargs(data_dir, use_arrow):
         geometry=[shapely.Point(0, 0)],
         crs="EPSG:4326",
     )
+    if GDAL_GE_311 and use_arrow:
+        # GDAL 3.11 started to use json extension type, which is not yet handled
+        # correctly in the arrow->pandas conversion (using object instead of str dtype)
+        expected["intermediate_level"] = expected["intermediate_level"].astype(object)
 
     assert_geodataframe_equal(df, expected)
 
-    df = read_dataframe(filename, use_arrow=use_arrow, FLATTEN_NESTED_ATTRIBUTES="YES")
+    df = read_dataframe(
+        nested_geojson_file, use_arrow=use_arrow, FLATTEN_NESTED_ATTRIBUTES="YES"
+    )
 
     expected = gp.GeoDataFrame(
         {
@@ -1742,7 +2042,7 @@ def test_write_nullable_dtypes(tmp_path, use_arrow):
     expected["col2"] = expected["col2"].astype("float64")
     expected["col3"] = expected["col3"].astype("float32")
     expected["col4"] = expected["col4"].astype("float64")
-    expected["col5"] = expected["col5"].astype(object)
+    expected["col5"] = expected["col5"].astype("str")
     expected.loc[1, "col5"] = None  # pandas converts to pd.NA on line above
     assert_geodataframe_equal(output_gdf, expected)
 
@@ -1904,6 +2204,9 @@ def test_write_memory(naturalearth_lowres, driver):
         check_dtype=not is_json,
     )
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 def test_write_memory_driver_required(naturalearth_lowres):
     df = read_dataframe(naturalearth_lowres)
@@ -1916,6 +2219,9 @@ def test_write_memory_driver_required(naturalearth_lowres):
     ):
         write_dataframe(df.head(1), buffer, driver=None, layer="test")
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 @pytest.mark.parametrize("driver", ["ESRI Shapefile", "OpenFileGDB"])
 def test_write_memory_unsupported_driver(naturalearth_lowres, driver):
@@ -1931,6 +2237,9 @@ def test_write_memory_unsupported_driver(naturalearth_lowres, driver):
     ):
         write_dataframe(df, buffer, driver=driver, layer="test")
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 @pytest.mark.parametrize("driver", ["GeoJSON", "GPKG"])
 def test_write_memory_append_unsupported(naturalearth_lowres, driver):
@@ -1943,6 +2252,9 @@ def test_write_memory_append_unsupported(naturalearth_lowres, driver):
     ):
         write_dataframe(df.head(1), buffer, driver=driver, layer="test", append=True)
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 def test_write_memory_existing_unsupported(naturalearth_lowres):
     df = read_dataframe(naturalearth_lowres)
@@ -1954,6 +2266,33 @@ def test_write_memory_existing_unsupported(naturalearth_lowres):
     ):
         write_dataframe(df.head(1), buffer, driver="GeoJSON", layer="test")
 
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
+
+def test_write_open_file_handle(tmp_path, naturalearth_lowres):
+    """Verify that writing to an open file handle is not currently supported"""
+
+    df = read_dataframe(naturalearth_lowres)
+
+    # verify it fails for regular file handle
+    with pytest.raises(
+        NotImplementedError, match="writing to an open file handle is not yet supported"
+    ):
+        with open(tmp_path / "test.geojson", "wb") as f:
+            write_dataframe(df.head(1), f)
+
+    # verify it fails for ZipFile
+    with pytest.raises(
+        NotImplementedError, match="writing to an open file handle is not yet supported"
+    ):
+        with ZipFile(tmp_path / "test.geojson.zip", "w") as z:
+            with z.open("test.geojson", "w") as f:
+                write_dataframe(df.head(1), f)
+
+    # Check temp file was cleaned up. Filter, as gdal keeps cache files in /vsimem/.
+    assert vsi_listtree("/vsimem/", pattern="pyogrio_*") == []
+
 
 @pytest.mark.parametrize("ext", ["gpkg", "geojson"])
 def test_non_utf8_encoding_io(tmp_path, ext, encoded_text):
@@ -2026,7 +2365,10 @@ def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text, use_arrow):
 
     if use_arrow:
         # pyarrow cannot decode column name with incorrect encoding
-        with pytest.raises(UnicodeDecodeError):
+        with pytest.raises(
+            DataSourceError,
+            match="The file being read is not encoded in UTF-8; please use_arrow=False",
+        ):
             read_dataframe(output_path, use_arrow=True)
     else:
         bad = read_dataframe(output_path, use_arrow=False)
@@ -2045,7 +2387,8 @@ def test_non_utf8_encoding_io_shapefile(tmp_path, encoded_text, use_arrow):
 
 
 def test_encoding_read_option_collision_shapefile(naturalearth_lowres, use_arrow):
-    """Providing both encoding parameter and ENCODING open option (even if blank) is not allowed"""
+    """Providing both encoding parameter and ENCODING open option
+    (even if blank) is not allowed."""
 
     with pytest.raises(
         ValueError, match='cannot provide both encoding parameter and "ENCODING" option'
@@ -2056,7 +2399,8 @@ def test_encoding_read_option_collision_shapefile(naturalearth_lowres, use_arrow
 
 
 def test_encoding_write_layer_option_collision_shapefile(tmp_path, encoded_text):
-    """Providing both encoding parameter and ENCODING layer creation option (even if blank) is not allowed"""
+    """Providing both encoding parameter and ENCODING layer creation option
+    (even if blank) is not allowed."""
     encoding, text = encoded_text
 
     output_path = tmp_path / "test.shp"
@@ -2064,7 +2408,10 @@ def test_encoding_write_layer_option_collision_shapefile(tmp_path, encoded_text)
 
     with pytest.raises(
         ValueError,
-        match='cannot provide both encoding parameter and "ENCODING" layer creation option',
+        match=(
+            'cannot provide both encoding parameter and "ENCODING" layer creation '
+            "option"
+        ),
     ):
         write_dataframe(
             df, output_path, encoding=encoding, layer_options={"ENCODING": ""}
@@ -2102,7 +2449,8 @@ def test_non_utf8_encoding_shapefile_sql(tmp_path, use_arrow):
 
 @pytest.mark.requires_arrow_write_api
 def test_write_kml_file_coordinate_order(tmp_path, use_arrow):
-    # confirm KML coordinates are written in lon, lat order even if CRS axis specifies otherwise
+    # confirm KML coordinates are written in lon, lat order even if CRS axis
+    # specifies otherwise
     points = [Point(10, 20), Point(30, 40), Point(50, 60)]
     gdf = gp.GeoDataFrame(geometry=points, crs="EPSG:4326")
     output_path = tmp_path / "test.kml"
@@ -2117,7 +2465,7 @@ def test_write_kml_file_coordinate_order(tmp_path, use_arrow):
     if "LIBKML" in list_drivers():
         # test appending to the existing file only if LIBKML is available
         # as it appears to fall back on LIBKML driver when appending.
-        points_append = [Point(70, 80), Point(90, 100), Point(110, 120)]
+        points_append = [Point(7, 8), Point(9, 10), Point(11, 12)]
         gdf_append = gp.GeoDataFrame(geometry=points_append, crs="EPSG:4326")
 
         write_dataframe(