pyogrio 0.7.2__cp311-cp311-manylinux_2_28_aarch64.whl → 0.9.0__cp311-cp311-manylinux_2_28_aarch64.whl

pyogrio/_io.pyx

@@ -10,24 +10,27 @@ import locale
 import logging
 import math
 import os
+import sys
 import warnings
 
 from libc.stdint cimport uint8_t, uintptr_t
 from libc.stdlib cimport malloc, free
 from libc.string cimport strlen
 from libc.math cimport isnan
+from cpython.pycapsule cimport PyCapsule_GetPointer
 
 cimport cython
+from cpython.pycapsule cimport PyCapsule_New, PyCapsule_GetPointer
+
 import numpy as np
-cimport numpy as np
 
 from pyogrio._ogr cimport *
 from pyogrio._err cimport *
+from pyogrio._vsi cimport *
 from pyogrio._err import CPLE_BaseError, CPLE_NotSupportedError, NullPointerError
 from pyogrio._geometry cimport get_geometry_type, get_geometry_type_code
 from pyogrio.errors import CRSError, DataSourceError, DataLayerError, GeometryError, FieldError, FeatureError
 
-
 log = logging.getLogger(__name__)
 
 
@@ -135,7 +138,52 @@ cdef char** dict_to_options(object values):
     return options
 
 
+cdef const char* override_threadlocal_config_option(str key, str value):
+    """Set the CPLSetThreadLocalConfigOption for key=value
+
+    Parameters
+    ----------
+    key : str
+    value : str
+
+    Returns
+    -------
+    const char*
+        value previously set for key, so that it can be later restored.  Caller
+        is responsible for freeing this via CPLFree() if not NULL.
+    """
+
+    key_b = key.encode("UTF-8")
+    cdef const char* key_c = key_b
+
+    value_b = value.encode("UTF-8")
+    cdef const char* value_c = value_b
+
+
+    cdef const char *prev_value = CPLGetThreadLocalConfigOption(key_c, NULL)
+    if prev_value != NULL:
+        # strings returned from config options may be replaced via
+        # CPLSetConfigOption() below; GDAL instructs us to save a copy
+        # in a new string
+        prev_value = CPLStrdup(prev_value)
+
+    CPLSetThreadLocalConfigOption(key_c, value_c)
+
+    return prev_value
+
+
 cdef void* ogr_open(const char* path_c, int mode, char** options) except NULL:
+    """Open an existing OGR data source
+
+    Parameters
+    ----------
+    path_c : char *
+        input path, including an in-memory path (/vsimem/...)
+    mode : int
+        set to 1 to allow updating data source
+    options : char **, optional
+        dataset open options
+    """
     cdef void* ogr_dataset = NULL
 
     # Force linear approximations in all cases
@@ -163,7 +211,7 @@ cdef void* ogr_open(const char* path_c, int mode, char** options) except NULL:
         ) from None
 
     except CPLE_BaseError as exc:
-        if str(exc).endswith("not recognized as a supported file format."):
+        if str(exc).endswith("a supported file format."):
             raise DataSourceError(
                 f"{str(exc)} It might help to specify the correct driver explicitly by "
                 "prefixing the file path with '<DRIVER>:', e.g. 'CSV:path'."
@@ -171,6 +219,25 @@ cdef void* ogr_open(const char* path_c, int mode, char** options) except NULL:
         raise DataSourceError(str(exc)) from None
 
 
+cdef ogr_close(GDALDatasetH ogr_dataset):
+    """Close the dataset and raise exception if that fails.
+    NOTE: some drivers only raise errors on write when calling GDALClose()
+    """
+    if ogr_dataset != NULL:
+        IF CTE_GDAL_VERSION >= (3, 7, 0):
+            if GDALClose(ogr_dataset) != CE_None:
+                return exc_check()
+
+            return
+
+        ELSE:
+            GDALClose(ogr_dataset)
+
+            # GDAL will set an error if there was an error writing the data source
+            # on close
+            return exc_check()
+
+
 cdef OGRLayerH get_ogr_layer(GDALDatasetH ogr_dataset, layer) except NULL:
     """Open OGR layer by index or name.
 
@@ -462,9 +529,11 @@ cdef get_metadata(GDALMajorObjectH obj):
 
 
 cdef detect_encoding(OGRDataSourceH ogr_dataset, OGRLayerH ogr_layer):
-    """Attempt to detect the encoding of the layer.
-    If it supports UTF-8, use that.
-    If it is a shapefile, it must otherwise be ISO-8859-1.
+    """Attempt to detect the encoding to use to read/write string values.
+
+    If the layer/dataset supports reading/writing data in UTF-8, returns UTF-8.
+    If UTF-8 is not supported and ESRI Shapefile, returns ISO-8859-1
+    Otherwise the system locale preferred encoding is returned.
 
     Parameters
     ----------
@@ -477,18 +546,53 @@ cdef detect_encoding(OGRDataSourceH ogr_dataset, OGRLayerH ogr_layer):
     """
 
     if OGR_L_TestCapability(ogr_layer, OLCStringsAsUTF8):
-        return 'UTF-8'
+        # OGR_L_TestCapability returns True for OLCStringsAsUTF8 if GDAL hides encoding
+        # complexities for this layer/driver type. In this case all string attribute
+        # values have to be supplied in UTF-8 and values will be returned in UTF-8.
+        # The encoding used to read/write under the hood depends on the driver used.
+        # For layers/drivers where False is returned, the string values are written and
+        # read without recoding. Hence, it is up to you to supply the data in the
+        # appropriate encoding. More info:
+        # https://gdal.org/development/rfc/rfc23_ogr_unicode.html#oftstring-oftstringlist-fields
+        # NOTE: for shapefiles, this always returns False for the layer returned
+        # when executing SQL, even when it supports UTF-8 (patched below);
+        # this may be fixed by https://github.com/OSGeo/gdal/pull/9649 (GDAL >=3.9.0?)
+        return "UTF-8"
 
     driver = get_driver(ogr_dataset)
-    if driver == 'ESRI Shapefile':
-        return 'ISO-8859-1'
+    if driver == "ESRI Shapefile":
+        # OGR_L_TestCapability returns True for OLCStringsAsUTF8 (above) for
+        # shapefiles when a .cpg file is present with a valid encoding, or GDAL
+        # auto-detects the encoding from the code page of the .dbf file, or
+        # SHAPE_ENCODING config option is set, or ENCODING layer creation option
+        # is specified (shapefiles only).  Otherwise, we can only assume that
+        # shapefiles are in their default encoding of ISO-8859-1 (which may be
+        # incorrect and must be overridden by user-provided encoding)
+
+        # Always use the first layer to test capabilities until detection for
+        # SQL results from shapefiles are fixed (above)
+        # This block should only be used for unfixed versions of GDAL (<3.9.0?)
+        if OGR_L_TestCapability(GDALDatasetGetLayer(ogr_dataset, 0), OLCStringsAsUTF8):
+            return "UTF-8"
+
+        return "ISO-8859-1"
 
     if driver == "OSM":
         # always set OSM data to UTF-8
         # per https://help.openstreetmap.org/questions/2172/what-encoding-does-openstreetmap-use
         return "UTF-8"
 
-    return None
+    if driver in ("XLSX", "ODS"):
+        # TestCapability for OLCStringsAsUTF8 for XLSX and ODS was False for new files
+        # being created for GDAL < 3.8.5. Once these versions of GDAL are no longer
+        # supported, this can be removed.
+        return "UTF-8"
+
+    if driver == "GeoJSONSeq":
+        # In old gdal versions, OLCStringsAsUTF8 wasn't advertised yet.
+        return "UTF-8"
+
+    return locale.getpreferredencoding()
 
 
 cdef get_fields(OGRLayerH ogr_layer, str encoding, use_arrow=False):
@@ -608,7 +712,7 @@ cdef apply_bbox_filter(OGRLayerH ogr_layer, bbox):
     Parameters
     ----------
     ogr_layer : pointer to open OGR layer
-    bbox: list or tuple of xmin, ymin, xmax, ymax
+    bbox : list or tuple of xmin, ymin, xmax, ymax
 
     Raises
     ------
@@ -629,7 +733,7 @@ cdef apply_geometry_filter(OGRLayerH ogr_layer, wkb):
     Parameters
     ----------
     ogr_layer : pointer to open OGR layer
-    wkb: WKB encoding of geometry
+    wkb : WKB encoding of geometry
     """
 
     cdef OGRGeometryH ogr_geometry = NULL
@@ -783,7 +887,7 @@ cdef process_fields(
             data[i] = bin_value[:ret_length]
 
         elif field_type == OFTDateTime or field_type == OFTDate:
-            
+
             if datetime_as_string:
                 # defer datetime parsing to user/ pandas layer
                 # Update to OGR_F_GetFieldAsISO8601DateTime when GDAL 3.7+ only
@@ -851,7 +955,7 @@ cdef get_features(
 
     field_data = [
         np.empty(shape=(num_features, ),
-        dtype = ("object" if datetime_as_string and 
+        dtype = ("object" if datetime_as_string and
                     fields[field_index,3].startswith("datetime") else fields[field_index,3])
         ) for field_index in range(n_fields)
     ]
@@ -950,8 +1054,8 @@ cdef get_features_by_fid(
     field_ogr_types = fields[:,1]
     field_data = [
         np.empty(shape=(count, ),
-        dtype=("object" if datetime_as_string and fields[field_index,3].startswith("datetime") 
-            else fields[field_index,3])) 
+        dtype=("object" if datetime_as_string and fields[field_index,3].startswith("datetime")
+            else fields[field_index,3]))
         for field_index in range(n_fields)
     ]
 
@@ -1060,7 +1164,7 @@ cdef get_bounds(
 
 
 def ogr_read(
-    str path,
+    object path_or_buffer,
     object dataset_kwargs,
     object layer=None,
     object encoding=None,
@@ -1080,6 +1184,7 @@ def ogr_read(
     ):
 
     cdef int err = 0
+    cdef bint is_vsimem = isinstance(path_or_buffer, bytes)
     cdef const char *path_c = NULL
     cdef char **dataset_options = NULL
     cdef const char *where_c = NULL
@@ -1089,9 +1194,8 @@ def ogr_read(
     cdef OGRLayerH ogr_layer = NULL
     cdef int feature_count = 0
     cdef double xmin, ymin, xmax, ymax
-
-    path_b = path.encode('utf-8')
-    path_c = path_b
+    cdef const char *prev_shape_encoding = NULL
+    cdef bint override_shape_encoding = False
 
     if fids is not None:
         if where is not None or bbox is not None or mask is not None or sql is not None or skip_features or max_features:
@@ -1120,13 +1224,23 @@ def ogr_read(
         raise ValueError("'max_features' must be >= 0")
 
     try:
+        path = read_buffer_to_vsimem(path_or_buffer) if is_vsimem else path_or_buffer
+
+        if encoding:
+            # for shapefiles, SHAPE_ENCODING must be set before opening the file
+            # to prevent automatic decoding to UTF-8 by GDAL, so we save previous
+            # SHAPE_ENCODING so that it can be restored later
+            # (we do this for all data sources where encoding is set because
+            # we don't know the driver until after it is opened, which is too late)
+            override_shape_encoding = True
+            prev_shape_encoding = override_threadlocal_config_option("SHAPE_ENCODING", encoding)
+
         dataset_options = dict_to_options(dataset_kwargs)
-        ogr_dataset = ogr_open(path_c, 0, dataset_options)
+        ogr_dataset = ogr_open(path.encode('UTF-8'), 0, dataset_options)
 
         if sql is None:
-            # layer defaults to index 0
             if layer is None:
-                layer = 0
+                layer = get_default_layer(ogr_dataset)
             ogr_layer = get_ogr_layer(ogr_dataset, layer)
         else:
             ogr_layer = execute_sql(ogr_dataset, sql, sql_dialect)
@@ -1135,23 +1249,31 @@ def ogr_read(
 
         # Encoding is derived from the user, from the dataset capabilities / type,
         # or from the system locale
-        encoding = (
-            encoding
-            or detect_encoding(ogr_dataset, ogr_layer)
-            or locale.getpreferredencoding()
-        )
+        if encoding:
+            if get_driver(ogr_dataset) == "ESRI Shapefile":
+                # NOTE: SHAPE_ENCODING is a configuration option whereas ENCODING is the dataset open option
+                if "ENCODING" in dataset_kwargs:
+                    raise ValueError('cannot provide both encoding parameter and "ENCODING" option; use encoding parameter to specify correct encoding for data source')
+
+                # Because SHAPE_ENCODING is set above, GDAL will automatically
+                # decode shapefiles to UTF-8; ignore any encoding set by user
+                encoding = "UTF-8"
+
+        else:
+            encoding = detect_encoding(ogr_dataset, ogr_layer)
 
         fields = get_fields(ogr_layer, encoding)
 
         ignored_fields = []
         if columns is not None:
+            # identify ignored fields first
+            ignored_fields = list(set(fields[:,2]) - set(columns))
+
             # Fields are matched exactly by name, duplicates are dropped.
             # Find index of each field into fields
             idx = np.intersect1d(fields[:,2], columns, return_indices=True)[1]
             fields = fields[idx, :]
 
-            ignored_fields = list(set(fields[:,2]) - set(columns))
-
         if not read_geometry:
             ignored_fields.append("OGR_GEOMETRY")
 
@@ -1232,6 +1354,17 @@ def ogr_read(
             GDALClose(ogr_dataset)
             ogr_dataset = NULL
 
+        # reset SHAPE_ENCODING config parameter if temporarily set above
+        if override_shape_encoding:
+            CPLSetThreadLocalConfigOption("SHAPE_ENCODING", prev_shape_encoding)
+
+            if prev_shape_encoding != NULL:
+                CPLFree(<void*>prev_shape_encoding)
+                prev_shape_encoding = NULL
+
+        if is_vsimem:
+            delete_vsimem_file(path)
+
     return (
         meta,
         fid_data,
@@ -1239,9 +1372,38 @@ def ogr_read(
         field_data
     )
 
+
+cdef void pycapsule_array_stream_deleter(object stream_capsule) noexcept:
+    cdef ArrowArrayStream* stream = <ArrowArrayStream*>PyCapsule_GetPointer(
+        stream_capsule, 'arrow_array_stream'
+    )
+    # Do not invoke the deleter on a used/moved capsule
+    if stream.release != NULL:
+        stream.release(stream)
+
+    free(stream)
+
+
+cdef object alloc_c_stream(ArrowArrayStream** c_stream):
+    c_stream[0] = <ArrowArrayStream*> malloc(sizeof(ArrowArrayStream))
+    # Ensure the capsule destructor doesn't call a random release pointer
+    c_stream[0].release = NULL
+    return PyCapsule_New(c_stream[0], 'arrow_array_stream', &pycapsule_array_stream_deleter)
+
+
+class _ArrowStream:
+    def __init__(self, capsule):
+        self._capsule = capsule
+
+    def __arrow_c_stream__(self, requested_schema=None):
+        if requested_schema is not None:
+            raise NotImplementedError("requested_schema is not supported")
+        return self._capsule
+
+
 @contextlib.contextmanager
 def ogr_open_arrow(
-    str path,
+    object path_or_buffer,
     dataset_kwargs,
     object layer=None,
     object encoding=None,
@@ -1257,31 +1419,38 @@ def ogr_open_arrow(
     str sql=None,
     str sql_dialect=None,
     int return_fids=False,
-    int batch_size=0):
+    int batch_size=0,
+    use_pyarrow=False,
+):
 
     cdef int err = 0
+    cdef bint is_vsimem = isinstance(path_or_buffer, bytes)
     cdef const char *path_c = NULL
     cdef char **dataset_options = NULL
     cdef const char *where_c = NULL
     cdef OGRDataSourceH ogr_dataset = NULL
     cdef OGRLayerH ogr_layer = NULL
+    cdef void *ogr_driver = NULL
     cdef char **fields_c = NULL
     cdef const char *field_c = NULL
     cdef char **options = NULL
-    cdef ArrowArrayStream stream
+    cdef const char *prev_shape_encoding = NULL
+    cdef bint override_shape_encoding = False
+    cdef ArrowArrayStream* stream
     cdef ArrowSchema schema
 
     IF CTE_GDAL_VERSION < (3, 6, 0):
         raise RuntimeError("Need GDAL>=3.6 for Arrow support")
 
-    path_b = path.encode('utf-8')
-    path_c = path_b
-
     if force_2d:
         raise ValueError("forcing 2D is not supported for Arrow")
 
     if fids is not None:
-        raise ValueError("reading by FID is not supported for Arrow")
+        if where is not None or bbox is not None or mask is not None or sql is not None or skip_features or max_features:
+            raise ValueError(
+                "cannot set both 'fids' and any of 'where', 'bbox', 'mask', "
+                "'sql', 'skip_features', or 'max_features'"
+            )
 
     IF CTE_GDAL_VERSION < (3, 8, 0):
         if skip_features:
@@ -1311,13 +1480,18 @@ def ogr_open_arrow(
 
     reader = None
     try:
+        path = read_buffer_to_vsimem(path_or_buffer) if is_vsimem else path_or_buffer
+
+        if encoding:
+            override_shape_encoding = True
+            prev_shape_encoding = override_threadlocal_config_option("SHAPE_ENCODING", encoding)
+
         dataset_options = dict_to_options(dataset_kwargs)
-        ogr_dataset = ogr_open(path_c, 0, dataset_options)
+        ogr_dataset = ogr_open(path.encode('UTF-8'), 0, dataset_options)
 
         if sql is None:
-            # layer defaults to index 0
             if layer is None:
-                layer = 0
+                layer = get_default_layer(ogr_dataset)
             ogr_layer = get_ogr_layer(ogr_dataset, layer)
         else:
             ogr_layer = execute_sql(ogr_dataset, sql, sql_dialect)
@@ -1326,11 +1500,18 @@ def ogr_open_arrow(
 
         # Encoding is derived from the user, from the dataset capabilities / type,
         # or from the system locale
-        encoding = (
-            encoding
-            or detect_encoding(ogr_dataset, ogr_layer)
-            or locale.getpreferredencoding()
-        )
+        if encoding:
+            if get_driver(ogr_dataset) == "ESRI Shapefile":
+                if "ENCODING" in dataset_kwargs:
+                    raise ValueError('cannot provide both encoding parameter and "ENCODING" option; use encoding parameter to specify correct encoding for data source')
+
+                encoding = "UTF-8"
+
+            elif encoding.replace('-','').upper() != 'UTF8':
+                raise ValueError("non-UTF-8 encoding is not supported for Arrow; use the non-Arrow interface instead")
+
+        else:
+            encoding = detect_encoding(ogr_dataset, ogr_layer)
 
         fields = get_fields(ogr_layer, encoding, use_arrow=True)
 
@@ -1341,19 +1522,64 @@ def ogr_open_arrow(
         if not read_geometry:
             ignored_fields.append("OGR_GEOMETRY")
 
+        # raise error if schema has bool values for FGB / GPKG and GDAL <3.8.3
+        # due to https://github.com/OSGeo/gdal/issues/8998
+        IF CTE_GDAL_VERSION < (3, 8, 3):
+
+            driver = get_driver(ogr_dataset)
+            if driver in {'FlatGeobuf', 'GPKG'}:
+                ignored = set(ignored_fields)
+                for f in fields:
+                    if f[2] not in ignored and f[3] == 'bool':
+                        raise RuntimeError(
+                            "GDAL < 3.8.3 does not correctly read boolean data values using the "
+                            "Arrow API.  Do not use read_arrow() / use_arrow=True for this dataset."
+                        )
+
         geometry_type = get_geometry_type(ogr_layer)
 
         geometry_name = get_string(OGR_L_GetGeometryColumn(ogr_layer))
 
         fid_column = get_string(OGR_L_GetFIDColumn(ogr_layer))
+        fid_column_where = fid_column
         # OGR_L_GetFIDColumn returns the column name if it is a custom column,
-        # or "" if not. For arrow, the default column name is "OGC_FID".
+        # or "" if not. For arrow, the default column name used to return the FID data
+        # read is "OGC_FID". When accessing the underlying datasource like when using a
+        # where clause, the default column name is "FID".
         if fid_column == "":
             fid_column = "OGC_FID"
+            fid_column_where = "FID"
+
+        # Use fids list to create a where clause, as arrow doesn't support direct fid
+        # filtering.
+        if fids is not None:
+            IF CTE_GDAL_VERSION < (3, 8, 0):
+                driver = get_driver(ogr_dataset)
+                if driver not in {"GPKG", "GeoJSON"}:
+                    warnings.warn(
+                        "Using 'fids' and 'use_arrow=True' with GDAL < 3.8 can be slow "
+                        "for some drivers. Upgrading GDAL or using 'use_arrow=False' "
+                        "can avoid this.",
+                        stacklevel=2,
+                    )
+
+            fids_str = ",".join([str(fid) for fid in fids])
+            where = f"{fid_column_where} IN ({fids_str})"
 
         # Apply the attribute filter
         if where is not None and where != "":
-            apply_where_filter(ogr_layer, where)
+            try:
+                apply_where_filter(ogr_layer, where)
+            except ValueError as ex:
+                if fids is not None and str(ex).startswith("Invalid SQL query"):
+                    # If fids is not None, the where being applied is the one formatted
+                    # above.
+                    raise ValueError(
+                        f"error applying filter for {len(fids)} fids; max. number for "
+                        f"drivers with default SQL dialect 'OGRSQL' is 4997"
+                    ) from ex
+
+                raise
 
         # Apply the spatial filter
         if bbox is not None:
@@ -1381,22 +1607,34 @@ def ogr_open_arrow(
                 str(batch_size).encode('UTF-8')
             )
 
+        # Default to geoarrow metadata encoding
+        IF CTE_GDAL_VERSION >= (3, 8, 0):
+            options = CSLSetNameValue(
+                options,
+                "GEOMETRY_METADATA_ENCODING",
+                "GEOARROW".encode('UTF-8')
+            )
+
         # make sure layer is read from beginning
         OGR_L_ResetReading(ogr_layer)
 
-        if not OGR_L_GetArrowStream(ogr_layer, &stream, options):
-            raise RuntimeError("Failed to open ArrowArrayStream from Layer")
+        # allocate the stream struct and wrap in capsule to ensure clean-up on error
+        capsule = alloc_c_stream(&stream)
 
-        stream_ptr = <uintptr_t> &stream
+        if not OGR_L_GetArrowStream(ogr_layer, stream, options):
+            raise RuntimeError("Failed to open ArrowArrayStream from Layer")
 
         if skip_features:
             # only supported for GDAL >= 3.8.0; have to do this after getting
             # the Arrow stream
             OGR_L_SetNextByIndex(ogr_layer, skip_features)
 
-        # stream has to be consumed before the Dataset is closed
-        import pyarrow as pa
-        reader = pa.RecordBatchStreamReader._import_from_c(stream_ptr)
+        if use_pyarrow:
+            import pyarrow as pa
+
+            reader = pa.RecordBatchStreamReader._import_from_c(<uintptr_t> stream)
+        else:
+            reader = _ArrowStream(capsule)
 
         meta = {
             'crs': crs,
@@ -1407,13 +1645,16 @@ def ogr_open_arrow(
             'fid_column': fid_column,
         }
 
+        # stream has to be consumed before the Dataset is closed
         yield meta, reader
 
     finally:
-        if reader is not None:
+        if use_pyarrow and reader is not None:
             # Mark reader as closed to prevent reading batches
             reader.close()
 
+        # `stream` will be freed through `capsule` destructor
+
         CSLDestroy(options)
         if fields_c != NULL:
             CSLDestroy(fields_c)
@@ -1430,8 +1671,20 @@ def ogr_open_arrow(
             GDALClose(ogr_dataset)
             ogr_dataset = NULL
 
+        # reset SHAPE_ENCODING config parameter if temporarily set above
+        if override_shape_encoding:
+            CPLSetThreadLocalConfigOption("SHAPE_ENCODING", prev_shape_encoding)
+
+            if prev_shape_encoding != NULL:
+                CPLFree(<void*>prev_shape_encoding)
+                prev_shape_encoding = NULL
+
+        if is_vsimem:
+            delete_vsimem_file(path)
+
+
 def ogr_read_bounds(
-    str path,
+    object path_or_buffer,
     object layer=None,
     object encoding=None,
     int read_geometry=True,
@@ -1444,6 +1697,7 @@ def ogr_read_bounds(
     object mask=None):
 
     cdef int err = 0
+    cdef bint is_vsimem = isinstance(path_or_buffer, bytes)
     cdef const char *path_c = NULL
     cdef const char *where_c = NULL
     cdef OGRDataSourceH ogr_dataset = NULL
@@ -1460,77 +1714,91 @@ def ogr_read_bounds(
     if max_features < 0:
         raise ValueError("'max_features' must be >= 0")
 
-    path_b = path.encode('utf-8')
-    path_c = path_b
+    try:
+        path = read_buffer_to_vsimem(path_or_buffer) if is_vsimem else path_or_buffer
+        ogr_dataset = ogr_open(path.encode('UTF-8'), 0, NULL)
 
-    # layer defaults to index 0
-    if layer is None:
-        layer = 0
+        if layer is None:
+            layer = get_default_layer(ogr_dataset)
 
-    ogr_dataset = ogr_open(path_c, 0, NULL)
-    ogr_layer = get_ogr_layer(ogr_dataset, layer)
+        ogr_layer = get_ogr_layer(ogr_dataset, layer)
 
-    # Apply the attribute filter
-    if where is not None and where != "":
-        apply_where_filter(ogr_layer, where)
+        # Apply the attribute filter
+        if where is not None and where != "":
+            apply_where_filter(ogr_layer, where)
+
+        # Apply the spatial filter
+        if bbox is not None:
+            apply_bbox_filter(ogr_layer, bbox)
+
+        elif mask is not None:
+            apply_geometry_filter(ogr_layer, mask)
 
-    # Apply the spatial filter
-    if bbox is not None:
-        apply_bbox_filter(ogr_layer, bbox)
+        # Limit feature range to available range
+        skip_features, num_features = validate_feature_range(ogr_layer, skip_features, max_features)
 
-    elif mask is not None:
-        apply_geometry_filter(ogr_layer, mask)
+        bounds = get_bounds(ogr_layer, skip_features, num_features)
 
-    # Limit feature range to available range
-    skip_features, num_features = validate_feature_range(ogr_layer, skip_features, max_features)
+    finally:
+        if ogr_dataset != NULL:
+            GDALClose(ogr_dataset)
+            ogr_dataset = NULL
 
-    return get_bounds(ogr_layer, skip_features, num_features)
+        if is_vsimem:
+            delete_vsimem_file(path)
+
+    return bounds
 
 
 def ogr_read_info(
-    str path,
+    object path_or_buffer,
     dataset_kwargs,
     object layer=None,
     object encoding=None,
     int force_feature_count=False,
     int force_total_bounds=False):
 
+    cdef bint is_vsimem = isinstance(path_or_buffer, bytes)
     cdef const char *path_c = NULL
     cdef char **dataset_options = NULL
     cdef OGRDataSourceH ogr_dataset = NULL
     cdef OGRLayerH ogr_layer = NULL
+    cdef const char *prev_shape_encoding = NULL
+    cdef bint override_shape_encoding = False
 
-    path_b = path.encode('utf-8')
-    path_c = path_b
+    try:
+        path = read_buffer_to_vsimem(path_or_buffer) if is_vsimem else path_or_buffer
 
-    # layer defaults to index 0
-    if layer is None:
-        layer = 0
+        if encoding:
+            override_shape_encoding = True
+            prev_shape_encoding = override_threadlocal_config_option("SHAPE_ENCODING", encoding)
 
-    try:
         dataset_options = dict_to_options(dataset_kwargs)
-        ogr_dataset = ogr_open(path_c, 0, dataset_options)
+        ogr_dataset = ogr_open(path.encode('UTF-8'), 0, dataset_options)
+
+        if layer is None:
+            layer = get_default_layer(ogr_dataset)
         ogr_layer = get_ogr_layer(ogr_dataset, layer)
 
-        # Encoding is derived from the user, from the dataset capabilities / type,
-        # or from the system locale
-        encoding = (
-            encoding
-            or detect_encoding(ogr_dataset, ogr_layer)
-            or locale.getpreferredencoding()
-        )
+        if encoding and get_driver(ogr_dataset) == "ESRI Shapefile":
+            encoding = "UTF-8"
+        else:
+            encoding = encoding or detect_encoding(ogr_dataset, ogr_layer)
 
         fields = get_fields(ogr_layer, encoding)
 
         meta = {
-            'crs': get_crs(ogr_layer),
-            'encoding': encoding,
-            'fields': fields[:,2], # return only names
-            'dtypes': fields[:,3],
-            'geometry_type': get_geometry_type(ogr_layer),
-            'features': get_feature_count(ogr_layer, force_feature_count),
-            'total_bounds': get_total_bounds(ogr_layer, force_total_bounds),
-            'driver': get_driver(ogr_dataset),
+            "layer_name": get_string(OGR_L_GetName(ogr_layer)),
+            "crs": get_crs(ogr_layer),
+            "encoding": encoding,
+            "fields": fields[:,2], # return only names
+            "dtypes": fields[:,3],
+            "fid_column": get_string(OGR_L_GetFIDColumn(ogr_layer)),
+            "geometry_name": get_string(OGR_L_GetGeometryColumn(ogr_layer)),
+            "geometry_type": get_geometry_type(ogr_layer),
+            "features": get_feature_count(ogr_layer, force_feature_count),
+            "total_bounds": get_total_bounds(ogr_layer, force_total_bounds),
+            "driver": get_driver(ogr_dataset),
             "capabilities": {
                 "random_read": OGR_L_TestCapability(ogr_layer, OLCRandomRead) == 1,
                 "fast_set_next_by_index": OGR_L_TestCapability(ogr_layer, OLCFastSetNextByIndex) == 1,
@@ -1538,8 +1806,8 @@ def ogr_read_info(
                 "fast_feature_count": OGR_L_TestCapability(ogr_layer, OLCFastFeatureCount) == 1,
                 "fast_total_bounds": OGR_L_TestCapability(ogr_layer, OLCFastGetExtent) == 1,
             },
-            'layer_metadata': get_metadata(ogr_layer),
-            'dataset_metadata': get_metadata(ogr_dataset),
+            "layer_metadata": get_metadata(ogr_layer),
+            "dataset_metadata": get_metadata(ogr_dataset),
         }
 
     finally:
@@ -1551,19 +1819,88 @@ def ogr_read_info(
             GDALClose(ogr_dataset)
             ogr_dataset = NULL
 
+        # reset SHAPE_ENCODING config parameter if temporarily set above
+        if override_shape_encoding:
+            CPLSetThreadLocalConfigOption("SHAPE_ENCODING", prev_shape_encoding)
+
+            if prev_shape_encoding != NULL:
+                CPLFree(<void*>prev_shape_encoding)
+
+        if is_vsimem:
+            delete_vsimem_file(path)
+
     return meta
 
 
-def ogr_list_layers(str path):
+def ogr_list_layers(object path_or_buffer):
+    cdef bint is_vsimem = isinstance(path_or_buffer, bytes)
     cdef const char *path_c = NULL
-    cdef const char *ogr_name = NULL
     cdef OGRDataSourceH ogr_dataset = NULL
-    cdef OGRLayerH ogr_layer = NULL
 
-    path_b = path.encode('utf-8')
-    path_c = path_b
+    try:
+        path = read_buffer_to_vsimem(path_or_buffer) if is_vsimem else path_or_buffer
+        ogr_dataset = ogr_open(path.encode('UTF-8'), 0, NULL)
+        layers = get_layer_names(ogr_dataset)
+
+    finally:
+        if ogr_dataset != NULL:
+            GDALClose(ogr_dataset)
+            ogr_dataset = NULL
+
+        if is_vsimem:
+            delete_vsimem_file(path)
+
+    return layers
+
+
+cdef str get_default_layer(OGRDataSourceH ogr_dataset):
+    """ Get the layer in the dataset that is read by default.
+
+    The caller is responsible for closing the dataset.
+
+    Parameters
+    ----------
+    ogr_dataset : pointer to open OGR dataset
+
+    Returns
+    -------
+    str
+        the name of the default layer to be read.
+
+    """
+    layers = get_layer_names(ogr_dataset)
+    first_layer_name = layers[0][0]
+
+    if len(layers) > 1:
+        dataset_name = os.path.basename(get_string(OGR_DS_GetName(ogr_dataset)))
+
+        other_layer_names = ', '.join([f"'{l}'" for l in layers[1:, 0]])
+        warnings.warn(
+            f"More than one layer found in '{dataset_name}': '{first_layer_name}' "
+            f"(default), {other_layer_names}. Specify layer parameter to avoid this "
+            "warning.",
+            stacklevel=2,
+        )
 
-    ogr_dataset = ogr_open(path_c, 0, NULL)
+    return first_layer_name
+
+
+cdef get_layer_names(OGRDataSourceH ogr_dataset):
+    """ Get the layers in the dataset.
+
+    The caller is responsible for closing the dataset.
+
+    Parameters
+    ----------
+    ogr_dataset : pointer to open OGR dataset
+
+    Returns
+    -------
+    ndarray(n)
+        array of layer names
+
+    """
+    cdef OGRLayerH ogr_layer = NULL
 
     layer_count = GDALDatasetGetLayerCount(ogr_dataset)
 
@@ -1575,10 +1912,6 @@ def ogr_list_layers(str path):
         data_view[i, 0] = get_string(OGR_L_GetName(ogr_layer))
         data_view[i, 1] = get_geometry_type(ogr_layer)
 
-    if ogr_dataset != NULL:
-        GDALClose(ogr_dataset)
-    ogr_dataset = NULL
-
     return data
 
 
@@ -1659,10 +1992,10 @@ cdef infer_field_types(list dtypes):
             field_types_view[i, 0] = OFTString
             # Convert to unicode string then take itemsize
             # TODO: better implementation of this
-            # width = values.astype(np.unicode_).dtype.itemsize // 4
+            # width = values.astype(np.str_).dtype.itemsize // 4
             # DO WE NEED WIDTH HERE?
 
-        elif dtype.type is np.unicode_ or dtype.type is np.string_:
+        elif dtype.type is np.str_ or dtype.type is np.bytes_:
             field_types_view[i, 0] = OFTString
             field_types_view[i, 2] = int(dtype.itemsize // 4)
 
@@ -1679,14 +2012,49 @@ cdef infer_field_types(list dtypes):
     return field_types
 
 
-# TODO: set geometry and field data as memory views?
-def ogr_write(
-    str path, str layer, str driver, geometry, fields, field_data, field_mask,
-    str crs, str geometry_type, str encoding, object dataset_kwargs,
-    object layer_kwargs, bint promote_to_multi=False, bint nan_as_null=True,
-    bint append=False, dataset_metadata=None, layer_metadata=None,
-    gdal_tz_offsets=None
+cdef create_ogr_dataset_layer(
+    str path,
+    bint is_vsi,
+    str layer,
+    str driver,
+    str crs,
+    str geometry_type,
+    str encoding,
+    object dataset_kwargs,
+    object layer_kwargs,
+    bint append,
+    dataset_metadata,
+    layer_metadata,
+    OGRDataSourceH* ogr_dataset_out,
+    OGRLayerH* ogr_layer_out,
 ):
+    """
+    Construct the OGRDataSource and OGRLayer objects based on input
+    path and layer.
+
+    If the file already exists, will open the existing dataset and overwrite
+    or append the layer (depending on `append`), otherwise will create a new
+    dataset.
+
+    Fills in the `ogr_dataset_out` and `ogr_layer_out` pointers passed as
+    parameter with initialized objects (or raise error is it fails to do so).
+    It is the responsibility of the caller to clean up those objects after use.
+    Returns whether a new layer was created or not (when the layer was created,
+    the caller still needs to set up the layer definition, i.e. create the
+    fields).
+
+    Parameters
+    ----------
+    encoding : str
+        Only used if `driver` is "ESRI Shapefile". If not None, it overrules the default
+        shapefile encoding, which is "UTF-8" in pyogrio.
+
+    Returns
+    -------
+    bool :
+        Whether a new layer was created, or False if we are appending to an
+        existing layer.
+    """
     cdef const char *path_c = NULL
     cdef const char *layer_c = NULL
     cdef const char *driver_c = NULL
@@ -1697,55 +2065,9 @@ def ogr_write(
     cdef const char *ogr_name = NULL
     cdef OGRDataSourceH ogr_dataset = NULL
     cdef OGRLayerH ogr_layer = NULL
-    cdef OGRFeatureH ogr_feature = NULL
-    cdef OGRGeometryH ogr_geometry = NULL
-    cdef OGRGeometryH ogr_geometry_multi = NULL
-    cdef OGRFeatureDefnH ogr_featuredef = NULL
-    cdef OGRFieldDefnH ogr_fielddef = NULL
-    cdef unsigned char *wkb_buffer = NULL
     cdef OGRSpatialReferenceH ogr_crs = NULL
-    cdef int layer_idx = -1
-    cdef int supports_transactions = 0
     cdef OGRwkbGeometryType geometry_code
-    cdef int err = 0
-    cdef int i = 0
-    cdef int num_records = -1
-    cdef int num_field_data = len(field_data) if field_data is not None else 0
-    cdef int num_fields = len(fields) if fields is not None else 0
-
-    if num_fields != num_field_data:
-        raise ValueError("field_data array needs to be same length as fields array")
-
-    if num_fields == 0 and geometry is None:
-        raise ValueError("You must provide at least a geometry column or a field")
-
-    if num_fields > 0:
-        num_records = len(field_data[0])
-        for i in range(1, len(field_data)):
-            if len(field_data[i]) != num_records:
-                raise ValueError("field_data arrays must be same length")
-
-    if geometry is None:
-        # If no geometry data, we ignore the geometry_type and don't create a geometry
-        # column
-        geometry_type = None
-    else:
-        if num_fields > 0:
-            if len(geometry) != num_records:
-                raise ValueError(
-                    "field_data arrays must be same length as geometry array"
-                )
-        else:
-            num_records = len(geometry)
-
-    if field_mask is not None:
-        if len(field_data) != len(field_mask):
-            raise ValueError("field_data and field_mask must be same length")
-        for i in range(0, len(field_mask)):
-            if field_mask[i] is not None and len(field_mask[i]) != num_records:
-                raise ValueError("field_mask arrays must be same length as geometry array")
-    else:
-        field_mask = [None] * num_fields
+    cdef int layer_idx = -1
 
     path_b = path.encode('UTF-8')
     path_c = path_b
@@ -1753,22 +2075,22 @@ def ogr_write(
     driver_b = driver.encode('UTF-8')
     driver_c = driver_b
 
+    # in-memory dataset is always created from scratch
+    path_exists = os.path.exists(path) if not is_vsi else False
+
     if not layer:
         layer = os.path.splitext(os.path.split(path)[1])[0]
 
-    if gdal_tz_offsets is None:
-        gdal_tz_offsets = {}
-
-
     # if shapefile, GeoJSON, or FlatGeobuf, always delete first
     # for other types, check if we can create layers
     # GPKG might be the only multi-layer writeable type.  TODO: check this
-    if driver in ('ESRI Shapefile', 'GeoJSON', 'GeoJSONSeq', 'FlatGeobuf') and os.path.exists(path):
+    if driver in ('ESRI Shapefile', 'GeoJSON', 'GeoJSONSeq', 'FlatGeobuf') and path_exists:
         if not append:
             os.unlink(path)
+            path_exists = False
 
     layer_exists = False
-    if os.path.exists(path):
+    if path_exists:
         try:
             ogr_dataset = ogr_open(path_c, 1, NULL)
 
@@ -1790,7 +2112,11 @@ def ogr_write(
                 raise exc
 
             # otherwise create from scratch
-            os.unlink(path)
+            if is_vsi:
+                VSIUnlink(path_c)
+            else:
+                os.unlink(path)
+
             ogr_dataset = NULL
 
     # either it didn't exist or could not open it in write mode
@@ -1808,25 +2134,20 @@ def ogr_write(
         if crs is not None:
             try:
                 ogr_crs = create_crs(crs)
+                # force geographic CRS to use lon, lat order and ignore axis order specified by CRS, in order
+                # to correctly write KML and GeoJSON coordinates in correct order
+                OSRSetAxisMappingStrategy(ogr_crs, OAMS_TRADITIONAL_GIS_ORDER)
+
 
             except Exception as exc:
-                OGRReleaseDataSource(ogr_dataset)
-                ogr_dataset = NULL
                 if dataset_options != NULL:
                     CSLDestroy(dataset_options)
                     dataset_options = NULL
-                raise exc
 
-        # Setup layer creation options
-        if not encoding:
-            encoding = locale.getpreferredencoding()
+                GDALClose(ogr_dataset)
+                ogr_dataset = NULL
 
-        if driver == 'ESRI Shapefile':
-            # Fiona only sets encoding for shapefiles; other drivers do not support
-            # encoding as an option.
-            encoding_b = encoding.upper().encode('UTF-8')
-            encoding_c = encoding_b
-            layer_options = CSLSetNameValue(layer_options, "ENCODING", encoding_c)
+                raise exc
 
         # Setup other layer creation options
         for k, v in layer_kwargs.items():
@@ -1834,6 +2155,21 @@ def ogr_write(
             v = v.encode('UTF-8')
             layer_options = CSLAddNameValue(layer_options, <const char *>k, <const char *>v)
 
+        if driver == 'ESRI Shapefile':
+            # ENCODING option must be set for shapefiles to properly write *.cpg
+            # file containing the encoding; this is not a supported option for
+            # other drivers.  This is done after setting general options above
+            # to override ENCODING if passed by the user as a layer option.
+            if encoding and "ENCODING" in layer_kwargs:
+                raise ValueError('cannot provide both encoding parameter and "ENCODING" layer creation option; use the encoding parameter')
+
+            # always write to UTF-8 if encoding is not set
+            encoding = encoding or "UTF-8"
+            encoding_b = encoding.upper().encode('UTF-8')
+            encoding_c = encoding_b
+            layer_options = CSLSetNameValue(layer_options, "ENCODING", encoding_c)
+
+
         ### Get geometry type
         # TODO: this is brittle for 3D / ZM / M types
         # TODO: fail on M / ZM types
@@ -1856,7 +2192,7 @@ def ogr_write(
         set_metadata(ogr_layer, layer_metadata)
 
     except Exception as exc:
-        OGRReleaseDataSource(ogr_dataset)
+        GDALClose(ogr_dataset)
         ogr_dataset = NULL
         raise DataLayerError(str(exc))
 
@@ -1873,60 +2209,152 @@ def ogr_write(
             CSLDestroy(layer_options)
             layer_options = NULL
 
-    ### Create the fields
-    field_types = None
+    ogr_dataset_out[0] = ogr_dataset
+    ogr_layer_out[0] = ogr_layer
+
+    return create_layer
+
+
+# TODO: set geometry and field data as memory views?
+def ogr_write(
+    object path_or_fp,
+    str layer,
+    str driver,
+    geometry,
+    fields,
+    field_data,
+    field_mask,
+    str crs,
+    str geometry_type,
+    str encoding,
+    object dataset_kwargs,
+    object layer_kwargs,
+    bint promote_to_multi=False,
+    bint nan_as_null=True,
+    bint append=False,
+    dataset_metadata=None,
+    layer_metadata=None,
+    gdal_tz_offsets=None
+):
+    cdef OGRDataSourceH ogr_dataset = NULL
+    cdef OGRLayerH ogr_layer = NULL
+    cdef OGRFeatureH ogr_feature = NULL
+    cdef OGRGeometryH ogr_geometry = NULL
+    cdef OGRGeometryH ogr_geometry_multi = NULL
+    cdef OGRFeatureDefnH ogr_featuredef = NULL
+    cdef OGRFieldDefnH ogr_fielddef = NULL
+    cdef unsigned char *wkb_buffer = NULL
+    cdef int supports_transactions = 0
+    cdef int err = 0
+    cdef int i = 0
+    cdef int num_records = -1
+    cdef int num_field_data = len(field_data) if field_data is not None else 0
+    cdef int num_fields = len(fields) if fields is not None else 0
+    cdef bint is_vsi = False
+
+    if num_fields != num_field_data:
+        raise ValueError("field_data array needs to be same length as fields array")
+
+    if num_fields == 0 and geometry is None:
+        raise ValueError("You must provide at least a geometry column or a field")
+
     if num_fields > 0:
-        field_types = infer_field_types([field.dtype for field in field_data])
+        num_records = len(field_data[0])
+        for i in range(1, len(field_data)):
+            if len(field_data[i]) != num_records:
+                raise ValueError("field_data arrays must be same length")
 
-    ### Create the fields
-    if create_layer:
-        for i in range(num_fields):
-            field_type, field_subtype, width, precision = field_types[i]
+    if geometry is None:
+        # If no geometry data, we ignore the geometry_type and don't create a geometry
+        # column
+        geometry_type = None
+    else:
+        if num_fields > 0:
+            if len(geometry) != num_records:
+                raise ValueError(
+                    "field_data arrays must be same length as geometry array"
+                )
+        else:
+            num_records = len(geometry)
 
-            name_b = fields[i].encode(encoding)
-            try:
-                ogr_fielddef = exc_wrap_pointer(OGR_Fld_Create(name_b, field_type))
+    if field_mask is not None:
+        if len(field_data) != len(field_mask):
+            raise ValueError("field_data and field_mask must be same length")
+        for i in range(0, len(field_mask)):
+            if field_mask[i] is not None and len(field_mask[i]) != num_records:
+                raise ValueError("field_mask arrays must be same length as geometry array")
+    else:
+        field_mask = [None] * num_fields
 
-                # subtypes, see: https://gdal.org/development/rfc/rfc50_ogr_field_subtype.html
-                if field_subtype != OFSTNone:
-                    OGR_Fld_SetSubType(ogr_fielddef, field_subtype)
+    if gdal_tz_offsets is None:
+        gdal_tz_offsets = {}
 
-                if width:
-                    OGR_Fld_SetWidth(ogr_fielddef, width)
+    try:
+        # Setup in-memory handler if needed
+        path = get_ogr_vsimem_write_path(path_or_fp, driver)
+        is_vsi = path.startswith('/vsimem/')
+
+        # Setup dataset and layer
+        layer_created = create_ogr_dataset_layer(
+            path, is_vsi, layer, driver, crs, geometry_type, encoding,
+            dataset_kwargs, layer_kwargs, append,
+            dataset_metadata, layer_metadata,
+            &ogr_dataset, &ogr_layer,
+        )
 
-                # TODO: set precision
+        if driver == 'ESRI Shapefile':
+            # force encoding for remaining operations to be in UTF-8 (even if user
+            # provides an encoding) because GDAL will automatically convert those to
+            # the target encoding because ENCODING is set as a layer creation option
+            encoding = "UTF-8"
 
-            except:
-                if ogr_fielddef != NULL:
-                    OGR_Fld_Destroy(ogr_fielddef)
-                    ogr_fielddef = NULL
+        else:
+            # Now the dataset and layer have been created, we can properly determine the
+            # encoding. It is derived from the user, from the dataset capabilities / type,
+            # or from the system locale
+            encoding = encoding or detect_encoding(ogr_dataset, ogr_layer)
 
-                OGRReleaseDataSource(ogr_dataset)
-                ogr_dataset = NULL
-                raise FieldError(f"Error creating field '{fields[i]}' from field_data") from None
+        ### Create the fields
+        field_types = None
+        if num_fields > 0:
+            field_types = infer_field_types([field.dtype for field in field_data])
 
-            try:
-                exc_wrap_int(OGR_L_CreateField(ogr_layer, ogr_fielddef, 1))
+        if layer_created:
+            for i in range(num_fields):
+                field_type, field_subtype, width, precision = field_types[i]
 
-            except:
-                OGRReleaseDataSource(ogr_dataset)
-                ogr_dataset = NULL
-                raise FieldError(f"Error adding field '{fields[i]}' to layer") from None
+                name_b = fields[i].encode(encoding)
+                try:
+                    ogr_fielddef = exc_wrap_pointer(OGR_Fld_Create(name_b, field_type))
 
-            finally:
-                if ogr_fielddef != NULL:
-                    OGR_Fld_Destroy(ogr_fielddef)
+                    # subtypes, see: https://gdal.org/development/rfc/rfc50_ogr_field_subtype.html
+                    if field_subtype != OFSTNone:
+                        OGR_Fld_SetSubType(ogr_fielddef, field_subtype)
 
+                    if width:
+                        OGR_Fld_SetWidth(ogr_fielddef, width)
 
-    ### Create the features
-    ogr_featuredef = OGR_L_GetLayerDefn(ogr_layer)
+                    # TODO: set precision
 
-    supports_transactions = OGR_L_TestCapability(ogr_layer, OLCTransactions)
-    if supports_transactions:
-        start_transaction(ogr_dataset, 0)
+                    exc_wrap_int(OGR_L_CreateField(ogr_layer, ogr_fielddef, 1))
 
-    for i in range(num_records):
-        try:
+                except:
+                    raise FieldError(f"Error adding field '{fields[i]}' to layer") from None
+
+                finally:
+                    if ogr_fielddef != NULL:
+                        OGR_Fld_Destroy(ogr_fielddef)
+                        ogr_fielddef = NULL
+
+
+        ### Create the features
+        ogr_featuredef = OGR_L_GetLayerDefn(ogr_layer)
+
+        supports_transactions = OGR_L_TestCapability(ogr_layer, OLCTransactions)
+        if supports_transactions:
+            start_transaction(ogr_dataset, 0)
+
+        for i in range(num_records):
             # create the feature
             ogr_feature = OGR_F_Create(ogr_featuredef)
             if ogr_feature == NULL:
@@ -1947,9 +2375,6 @@ def ogr_write(
                 wkb_buffer = wkb
                 err = OGR_G_ImportFromWkb(ogr_geometry, wkb_buffer, len(wkb))
                 if err:
-                    if ogr_geometry != NULL:
-                        OGR_G_DestroyGeometry(ogr_geometry)
-                        ogr_geometry = NULL
                     raise GeometryError(f"Could not create geometry from WKB at index {i}") from None
 
                 # Convert to multi type
@@ -1964,6 +2389,7 @@ def ogr_write(
                 # Set the geometry on the feature
                 # this assumes ownership of the geometry and it's cleanup
                 err = OGR_F_SetGeometryDirectly(ogr_feature, ogr_geometry)
+                ogr_geometry = NULL  # to prevent cleanup after this point
                 if err:
                     raise GeometryError(f"Could not set geometry for feature at index {i}") from None
 
@@ -1977,7 +2403,6 @@ def ogr_write(
                     OGR_F_SetFieldNull(ogr_feature, field_idx)
 
                 elif field_type == OFTString:
-                    # TODO: encode string using approach from _get_internal_encoding which checks layer capabilities
                     if (
                         field_value is None
                         or (isinstance(field_value, float) and isnan(field_value))
@@ -1989,7 +2414,7 @@ def ogr_write(
                             field_value = str(field_value)
 
                         try:
-                            value_b = field_value.encode("UTF-8")
+                            value_b = field_value.encode(encoding)
                             OGR_F_SetFieldString(ogr_feature, field_idx, value_b)
 
                         except AttributeError:
@@ -2056,25 +2481,262 @@ def ogr_write(
             # Add feature to the layer
             try:
                 exc_wrap_int(OGR_L_CreateFeature(ogr_layer, ogr_feature))
+
             except CPLE_BaseError as exc:
                 raise FeatureError(f"Could not add feature to layer at index {i}: {exc}") from None
 
-        finally:
-            if ogr_feature != NULL:
-                OGR_F_Destroy(ogr_feature)
-                ogr_feature = NULL
+            OGR_F_Destroy(ogr_feature)
+            ogr_feature = NULL
 
-    if supports_transactions:
-        commit_transaction(ogr_dataset)
 
-    log.info(f"Created {num_records:,} records" )
+        if supports_transactions:
+            commit_transaction(ogr_dataset)
 
-    ### Final cleanup
-    if ogr_dataset != NULL:
-        GDALClose(ogr_dataset)
+        log.info(f"Created {num_records:,} records" )
+
+        # close dataset to force driver to flush data
+        exc = ogr_close(ogr_dataset)
+        ogr_dataset = NULL
+        if exc:
+            raise DataSourceError(f"Failed to write features to dataset {path}; {exc}")
+
+        # copy in-memory file back to path_or_fp object
+        if is_vsi:
+            read_vsimem_to_buffer(path, path_or_fp)
+
+    finally:
+        ### Final cleanup
+        # make sure that all objects allocated above are released if exceptions
+        # are raised, and the dataset is closed
+        if ogr_fielddef != NULL:
+            OGR_Fld_Destroy(ogr_fielddef)
+            ogr_fielddef = NULL
+
+        if ogr_feature != NULL:
+            OGR_F_Destroy(ogr_feature)
+            ogr_feature = NULL
+
+        if ogr_geometry != NULL:
+            OGR_G_DestroyGeometry(ogr_geometry)
+            ogr_geometry = NULL
+
+        if ogr_dataset != NULL:
+            ogr_close(ogr_dataset)
+
+        if is_vsi:
+            delete_vsimem_file(path)
+
+
+def ogr_write_arrow(
+    object path_or_fp,
+    str layer,
+    str driver,
+    object arrow_obj,
+    str crs,
+    str geometry_type,
+    str geometry_name,
+    str encoding,
+    object dataset_kwargs,
+    object layer_kwargs,
+    bint append=False,
+    dataset_metadata=None,
+    layer_metadata=None,
+):
+    IF CTE_GDAL_VERSION < (3, 8, 0):
+        raise RuntimeError("Need GDAL>=3.8 for Arrow write support")
+
+    cdef OGRDataSourceH ogr_dataset = NULL
+    cdef OGRLayerH ogr_layer = NULL
+    cdef char **options = NULL
+    cdef bint is_vsi = False
+    cdef ArrowArrayStream* stream = NULL
+    cdef ArrowSchema schema
+    cdef ArrowArray array
+
+    schema.release = NULL
+    array.release = NULL
+
+    try:
+        path = get_ogr_vsimem_write_path(path_or_fp, driver)
+        is_vsi = path.startswith('/vsimem/')
+
+        layer_created = create_ogr_dataset_layer(
+            path, is_vsi, layer, driver, crs, geometry_type, encoding,
+            dataset_kwargs, layer_kwargs, append,
+            dataset_metadata, layer_metadata,
+            &ogr_dataset, &ogr_layer,
+        )
+
+        # only shapefile supports non-UTF encoding because ENCODING option is set
+        # during dataset creation and GDAL auto-translates from UTF-8 values to that
+        # encoding
+        if encoding and encoding.replace('-','').upper() != 'UTF8' and driver != 'ESRI Shapefile':
+            raise ValueError("non-UTF-8 encoding is not supported for Arrow; use the non-Arrow interface instead")
+
+        if geometry_name:
+            opts = {"GEOMETRY_NAME": geometry_name}
+        else:
+            opts = {}
 
-        # GDAL will set an error if there was an error writing the data source
-        # on close
-        exc = exc_check()
+        options = dict_to_options(opts)
+
+        stream_capsule = arrow_obj.__arrow_c_stream__()
+        stream = <ArrowArrayStream*>PyCapsule_GetPointer(
+            stream_capsule, "arrow_array_stream"
+        )
+
+        if stream == NULL:
+            raise RuntimeError("Could not extract valid Arrow array stream.")
+
+        if stream.release == NULL:
+            raise RuntimeError("Arrow array stream was already released.")
+
+        if stream.get_schema(stream, &schema) != 0:
+            raise RuntimeError("Could not get Arrow schema from stream.")
+
+        if layer_created:
+            create_fields_from_arrow_schema(ogr_layer, &schema, options, geometry_name)
+
+        while True:
+            if stream.get_next(stream, &array) != 0:
+                raise RuntimeError("Error while accessing batch from stream.")
+
+            # We've reached the end of the stream
+            if array.release == NULL:
+                break
+
+            if not OGR_L_WriteArrowBatch(ogr_layer, &schema, &array, options):
+                exc = exc_check()
+                gdal_msg = f": {str(exc)}" if exc else "."
+                raise DataLayerError(
+                    f"Error while writing batch to OGR layer{gdal_msg}"
+                )
+
+            if array.release != NULL:
+                array.release(&array)
+
+        # close dataset to force driver to flush data
+        exc = ogr_close(ogr_dataset)
+        ogr_dataset = NULL
         if exc:
-            raise DataSourceError(f"Failed to write features to dataset {path}; {exc}")
+            raise DataSourceError(f"Failed to write features to dataset {path}; {exc}")
+
+        # copy in-memory file back to path_or_fp object
+        if is_vsi:
+            read_vsimem_to_buffer(path, path_or_fp)
+
+    finally:
+        if stream != NULL and stream.release != NULL:
+            stream.release(stream)
+
+        if schema.release != NULL:
+            schema.release(&schema)
+
+        if array.release != NULL:
+            array.release(&array)
+
+        if options != NULL:
+            CSLDestroy(options)
+            options = NULL
+
+        if ogr_dataset != NULL:
+            ogr_close(ogr_dataset)
+
+        if is_vsi:
+            delete_vsimem_file(path)
+
+
+cdef get_arrow_extension_metadata(const ArrowSchema* schema):
+    """
+    Parse the metadata of the ArrowSchema and extract extension type
+    metadata (extension name and metadata).
+
+    For the exact layout of the bytes, see
+    https://arrow.apache.org/docs/dev/format/CDataInterface.html#c.ArrowSchema.metadata
+    """
+    cdef const char *metadata = schema.metadata
+
+    extension_name = None
+    extension_metadata = None
+
+    if metadata == NULL:
+        return extension_name, extension_metadata
+
+    # the number of metadata key/value pairs is stored
+    # as an int32 value in the first 4 bytes
+    n = int.from_bytes(metadata[:4], byteorder=sys.byteorder)
+    pos = 4
+
+    for i in range(n):
+        # for each metadata key/value pair, the first 4 bytes is the byte length
+        # of the key as an int32, then follows the key (not null-terminated),
+        # and then the same for the value length and bytes
+        key_length = int.from_bytes(
+            metadata[pos:pos+4], byteorder=sys.byteorder, signed=True
+        )
+        pos += 4
+        key = metadata[pos:pos+key_length]
+        pos += key_length
+        value_length = int.from_bytes(
+            metadata[pos:pos+4], byteorder=sys.byteorder, signed=True
+        )
+        pos += 4
+        value = metadata[pos:pos+value_length]
+        pos += value_length
+
+        if key == b"ARROW:extension:name":
+            extension_name = value
+        elif key == b"ARROW:extension:metadata":
+            extension_metadata = value
+
+        if extension_name is not None and extension_metadata is not None:
+            break
+
+    return extension_name, extension_metadata
+
+
+cdef is_arrow_geometry_field(const ArrowSchema* schema):
+    name, _ = get_arrow_extension_metadata(schema)
+    if name is not None:
+        if name == b"geoarrow.wkb" or name == b"ogc.wkb":
+            return True
+
+        # raise an error for other geoarrow types
+        if name.startswith(b"geoarrow."):
+            raise NotImplementedError(
+                f"Writing a geometry column of type {name.decode()} is not yet "
+                "supported. Only WKB is currently supported ('geoarrow.wkb' or "
+                "'ogc.wkb' types)."
+            )
+
+    return False
+
+
+cdef create_fields_from_arrow_schema(
+    OGRLayerH destLayer, const ArrowSchema* schema, char** options, str geometry_name
+):
+    """Create output fields using CreateFieldFromArrowSchema()"""
+
+    IF CTE_GDAL_VERSION < (3, 8, 0):
+        raise RuntimeError("Need GDAL>=3.8 for Arrow write support")
+
+    # The schema object is a struct type where each child is a column.
+    cdef ArrowSchema* child
+    for i in range(schema.n_children):
+        child = schema.children[i]
+
+        if child == NULL:
+            raise RuntimeError("Received invalid Arrow schema (null child)")
+
+        # Don't create property for geometry column
+        if get_string(child.name) == geometry_name or is_arrow_geometry_field(child):
+            continue
+
+        if not OGR_L_CreateFieldFromArrowSchema(destLayer, child, options):
+            exc = exc_check()
+            gdal_msg = f" ({str(exc)})" if exc else ""
+            raise FieldError(
+                f"Error while creating field from Arrow for field {i} with name "
+                f"'{get_string(child.name)}' and type {get_string(child.format)}"
+                f"{gdal_msg}."
+            )