pyogrio 0.7.1__cp38-cp38-manylinux_2_28_aarch64.whl → 0.8.0__cp38-cp38-manylinux_2_28_aarch64.whl

pyogrio/_ogr.pxd

@@ -12,6 +12,9 @@ cdef extern from "cpl_conv.h":
     const char* CPLFindFile(const char *pszClass, const char *filename)
     const char* CPLGetConfigOption(const char* key, const char* value)
     void        CPLSetConfigOption(const char* key, const char* value)
+    const char* CPLGetThreadLocalConfigOption(const char* key, const char* value)
+    void        CPLSetThreadLocalConfigOption(const char* key, const char* value)
+    char*       CPLStrdup(const char* string)
 
 
 cdef extern from "cpl_error.h" nogil:
@@ -42,13 +45,23 @@ cdef extern from "cpl_string.h":
 
 
 cdef extern from "cpl_vsi.h" nogil:
-
+    int VSI_STAT_EXISTS_FLAG
+    ctypedef int vsi_l_offset
     ctypedef FILE VSILFILE
+    ctypedef struct VSIStatBufL:
+        long st_size
+        long st_mode
+        int st_mtime
+
+    int         VSIFCloseL(VSILFILE *fp)
+    int         VSIFFlushL(VSILFILE *fp)
+    int         VSIUnlink(const char *path)
 
-    VSILFILE *VSIFileFromMemBuffer(const char *path, void *data,
-                                   int data_len, int take_ownership)
-    int VSIFCloseL(VSILFILE *fp)
-    int VSIUnlink(const char *path)
+    VSILFILE        *VSIFileFromMemBuffer(const char *path, void *data, vsi_l_offset data_len, int take_ownership)
+    unsigned char   *VSIGetMemFileBuffer(const char *path, vsi_l_offset *data_len, int take_ownership)
+
+    int     VSIMkdir(const char *path, long mode)
+    int     VSIRmdirRecursive(const char *pszDirname)
 
 
 cdef extern from "ogr_core.h":
@@ -190,12 +203,36 @@ cdef extern from "ogr_srs_api.h":
     void                    OSRRelease(OGRSpatialReferenceH srs)
 
 
-cdef extern from "arrow_bridge.h":
+cdef extern from "arrow_bridge.h" nogil:
+    struct ArrowArray:
+        int64_t length
+        int64_t null_count
+        int64_t offset
+        int64_t n_buffers
+        int64_t n_children
+        const void** buffers
+        ArrowArray** children
+        ArrowArray* dictionary
+        void (*release)(ArrowArray*) noexcept nogil
+        void* private_data
+
     struct ArrowSchema:
+        const char* format
+        const char* name
+        const char* metadata
+        int64_t flags
         int64_t n_children
+        ArrowSchema** children
+        ArrowSchema* dictionary
+        void (*release)(ArrowSchema*) noexcept nogil
+        void* private_data
 
     struct ArrowArrayStream:
-        int (*get_schema)(ArrowArrayStream* stream, ArrowSchema* out)
+        int (*get_schema)(ArrowArrayStream*, ArrowSchema* out) noexcept
+        int (*get_next)(ArrowArrayStream*, ArrowArray* out)
+        const char* (*get_last_error)(ArrowArrayStream*)
+        void (*release)(ArrowArrayStream*) noexcept
+        void* private_data
 
 
 cdef extern from "ogr_api.h":
@@ -205,6 +242,8 @@ cdef extern from "ogr_api.h":
     OGRDataSourceH  OGR_Dr_Open(OGRSFDriverH driver, const char *path, int bupdate)
     const char*     OGR_Dr_GetName(OGRSFDriverH driver)
 
+    const char*     OGR_DS_GetName(OGRDataSourceH)
+
     OGRFeatureH     OGR_F_Create(OGRFeatureDefnH featuredefn)
     void            OGR_F_Destroy(OGRFeatureH feature)
 
@@ -298,8 +337,6 @@ cdef extern from "ogr_api.h":
     void            OGRSetNonLinearGeometriesEnabledFlag(int bFlag)
     int             OGRGetNonLinearGeometriesEnabledFlag()
 
-    int             OGRReleaseDataSource(OGRDataSourceH ds)
-
     const char*     OLCStringsAsUTF8
     const char*     OLCRandomRead
     const char*     OLCFastSetNextByIndex
@@ -312,8 +349,14 @@ cdef extern from "ogr_api.h":
 IF CTE_GDAL_VERSION >= (3, 6, 0):
 
     cdef extern from "ogr_api.h":
-        int8_t OGR_L_GetArrowStream(OGRLayerH hLayer, ArrowArrayStream *out_stream, char** papszOptions)
+        bint OGR_L_GetArrowStream(OGRLayerH hLayer, ArrowArrayStream *out_stream, char** papszOptions)
+
 
+IF CTE_GDAL_VERSION >= (3, 8, 0):
+
+    cdef extern from "ogr_api.h":
+        bint OGR_L_CreateFieldFromArrowSchema(OGRLayerH hLayer, ArrowSchema *schema, char **papszOptions)
+        bint OGR_L_WriteArrowBatch(OGRLayerH hLayer, ArrowSchema *schema, ArrowArray *array, char **papszOptions)
 
 cdef extern from "gdal.h":
     ctypedef enum GDALDataType:
@@ -366,7 +409,6 @@ cdef extern from "gdal.h":
                                const char *const *papszOpenOptions,
                                const char *const *papszSiblingFiles)
 
-    void            GDALClose(GDALDatasetH ds)
     int             GDALDatasetGetLayerCount(GDALDatasetH ds)
     OGRLayerH       GDALDatasetGetLayer(GDALDatasetH ds, int iLayer)
     OGRLayerH       GDALDatasetGetLayerByName(GDALDatasetH ds, char * pszName)
@@ -385,4 +427,15 @@ cdef extern from "gdal.h":
     const char*     GDALVersionInfo(const char *pszRequest)
 
 
-cdef get_string(const char *c_str, str encoding=*)
+# GDALClose returns error code for >= 3.7.0
+IF CTE_GDAL_VERSION >= (3, 7, 0):
+
+    cdef extern from "ogr_api.h":
+        int GDALClose(GDALDatasetH ds)
+ELSE:
+
+    cdef extern from "ogr_api.h":
+        void GDALClose(GDALDatasetH ds)
+
+
+cdef get_string(const char *c_str, str encoding=*)

pyogrio/_ogr.pyx

@@ -108,6 +108,14 @@ def ogr_driver_supports_write(driver):
     return False
 
 
+def ogr_driver_supports_vsi(driver):
+    # check metadata for driver to see if it supports write
+    if _get_driver_metadata_item(driver, "DCAP_VIRTUALIO") == 'YES':
+        return True
+
+    return False
+
+
 def ogr_list_drivers():
     cdef OGRSFDriverH driver = NULL
     cdef int i
@@ -129,30 +137,6 @@ def ogr_list_drivers():
     return drivers
 
 
-def buffer_to_virtual_file(bytesbuf, ext=''):
-    """Maps a bytes buffer to a virtual file.
-    `ext` is empty or begins with a period and contains at most one period.
-
-    This (and remove_virtual_file) is originally copied from the Fiona project
-    (https://github.com/Toblerity/Fiona/blob/c388e9adcf9d33e3bb04bf92b2ff210bbce452d9/fiona/ogrext.pyx#L1863-L1879)
-    """
-
-    vsi_filename = f"/vsimem/{uuid4().hex + ext}"
-
-    vsi_handle = VSIFileFromMemBuffer(vsi_filename.encode("UTF-8"), <unsigned char *>bytesbuf, len(bytesbuf), 0)
-
-    if vsi_handle == NULL:
-        raise OSError('failed to map buffer to file')
-    if VSIFCloseL(vsi_handle) != 0:
-        raise OSError('failed to close mapped file handle')
-
-    return vsi_filename
-
-
-def remove_virtual_file(vsi_filename):
-    return VSIUnlink(vsi_filename.encode("UTF-8"))
-
-
 cdef void set_proj_search_path(str path):
     """Set PROJ library data file search path for use in GDAL."""
     cdef char **paths = NULL

pyogrio/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-26T15:56:41-0700",
+ "date": "2024-05-06T14:34:51-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "97d9dee48584208ccbb479b87d58ec1eb92b3f9c",
- "version": "0.7.1"
+ "full-revisionid": "46c35a7e98d85923cfefe73dae7285404e72d9a6",
+ "version": "0.8.0"
 }
 '''  # END VERSION_JSON
 

pyogrio/_vsi.pxd

@@ -0,0 +1,4 @@
+cdef str get_ogr_vsimem_write_path(object path_or_fp, str driver)
+cdef str read_buffer_to_vsimem(bytes bytes_buffer)
+cdef read_vsimem_to_buffer(str path, object out_buffer)
+cdef delete_vsimem_file(str path)

pyogrio/_vsi.pyx

@@ -0,0 +1,140 @@
+from io import BytesIO
+from uuid import uuid4
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memcpy
+
+from pyogrio._ogr cimport *
+from pyogrio._ogr import _get_driver_metadata_item
+
+
+cdef str get_ogr_vsimem_write_path(object path_or_fp, str driver):
+    """ Return the original path or a /vsimem/ path
+
+    If passed a io.BytesIO object, this will return a /vsimem/ path that can be
+    used to create a new in-memory file with an extension inferred from the driver
+    if possible.  Path will be contained in an in-memory directory to contain
+    sibling files (though drivers that create sibling files are not supported for
+    in-memory files).
+
+    Caller is responsible for deleting the directory via delete_vsimem_file()
+
+    Parameters
+    ----------
+    path_or_fp : str or io.BytesIO object
+    driver : str
+    """
+
+    if not isinstance(path_or_fp, BytesIO):
+        return path_or_fp
+
+    # Create in-memory directory to contain auxiliary files
+    memfilename = uuid4().hex
+    VSIMkdir(f"/vsimem/{memfilename}".encode("UTF-8"), 0666)
+
+    # file extension is required for some drivers, set it based on driver metadata
+    ext = ""
+    recommended_ext = _get_driver_metadata_item(driver, "DMD_EXTENSIONS")
+    if recommended_ext is not None:
+        ext = "." + recommended_ext.split(" ")[0]
+
+    path = f"/vsimem/{memfilename}/{memfilename}{ext}"
+
+    # check for existing bytes
+    if path_or_fp.getbuffer().nbytes > 0:
+        raise NotImplementedError("writing to existing in-memory object is not supported")
+
+    return path
+
+
+cdef str read_buffer_to_vsimem(bytes bytes_buffer):
+    """ Wrap the bytes (zero-copy) into an in-memory dataset
+
+    If the first 4 bytes indicate the bytes are a zip file, the returned path
+    will be prefixed with /vsizip/ and suffixed with .zip to enable proper
+    reading by GDAL.
+
+    Caller is responsible for deleting the in-memory file via delete_vsimem_file().
+
+    Parameters
+    ----------
+    bytes_buffer : bytes
+    """
+    cdef int num_bytes = len(bytes_buffer)
+
+    is_zipped = len(bytes_buffer) > 4 and bytes_buffer[:4].startswith(b"PK\x03\x04")
+    ext = ".zip" if is_zipped else ""
+
+    path = f"/vsimem/{uuid4().hex}{ext}"
+
+    # Create an in-memory object that references bytes_buffer
+    # NOTE: GDAL does not copy the contents of bytes_buffer; it must remain
+    # in scope through the duration of using this file
+    vsi_handle = VSIFileFromMemBuffer(path.encode("UTF-8"), <unsigned char *>bytes_buffer, num_bytes, 0)
+
+    if vsi_handle == NULL:
+        raise OSError("failed to read buffer into in-memory file")
+
+    if VSIFCloseL(vsi_handle) != 0:
+        raise OSError("failed to close in-memory file")
+
+    if is_zipped:
+        path = f"/vsizip/{path}"
+
+    return path
+
+
+cdef read_vsimem_to_buffer(str path, object out_buffer):
+    """Copy bytes from in-memory file to buffer
+
+    This will automatically unlink the in-memory file pointed to by path; caller
+    is still responsible for calling delete_vsimem_file() to cleanup any other
+    files contained in the in-memory directory.
+
+    Parameters:
+    -----------
+    path : str
+        path to in-memory file
+    buffer : BytesIO object
+    """
+
+    cdef unsigned char *vsi_buffer = NULL
+    cdef vsi_l_offset vsi_buffer_size = 0
+
+    try:
+        # Take ownership of the buffer to avoid a copy; GDAL will automatically
+        # unlink the memory file
+        vsi_buffer = VSIGetMemFileBuffer(path.encode("UTF-8"), &vsi_buffer_size, 1)
+        if vsi_buffer == NULL:
+            raise RuntimeError("could not read bytes from in-memory file")
+
+        # write bytes to buffer
+        out_buffer.write(<bytes>vsi_buffer[:vsi_buffer_size])
+        # rewind to beginning to allow caller to read
+        out_buffer.seek(0)
+
+    finally:
+        if vsi_buffer != NULL:
+            CPLFree(vsi_buffer)
+
+
+cdef delete_vsimem_file(str path):
+    """ Recursively delete in-memory path or directory containing path
+
+    This is used for final cleanup of an in-memory dataset, which may have been
+    created within a directory to contain sibling files.
+
+    Additional VSI handlers may be chained to the left of /vsimem/ in path and
+    will be ignored.
+
+    Parameters:
+    -----------
+    path : str
+        path to in-memory file
+    """
+
+    if "/vsimem/" not in path:
+        return
+
+    root = "/vsimem/" + path.split("/vsimem/")[1].split("/")[0]
+    VSIRmdirRecursive(root.encode("UTF-8"))

pyogrio/core.py

@@ -1,5 +1,9 @@
 from pyogrio._env import GDALEnv
-from pyogrio.util import get_vsi_path, _preprocess_options_key_value, _mask_to_wkb
+from pyogrio.util import (
+    get_vsi_path_or_buffer,
+    _preprocess_options_key_value,
+    _mask_to_wkb,
+)
 
 
 with GDALEnv():
@@ -13,7 +17,6 @@ with GDALEnv():
         get_gdal_data_path as _get_gdal_data_path,
         init_gdal_data as _init_gdal_data,
         init_proj_data as _init_proj_data,
-        remove_virtual_file,
         _register_drivers,
         _get_drivers_for_path,
     )
@@ -102,7 +105,7 @@ def list_layers(path_or_buffer, /):
 
     Parameters
     ----------
-    path : str or pathlib.Path
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
 
     Returns
     -------
@@ -110,14 +113,8 @@ def list_layers(path_or_buffer, /):
         array of pairs of [<layer name>, <layer geometry type>]
         Note: geometry is `None` for nonspatial layers.
     """
-    path, buffer = get_vsi_path(path_or_buffer)
 
-    try:
-        result = ogr_list_layers(path)
-    finally:
-        if buffer is not None:
-            remove_virtual_file(path)
-    return result
+    return ogr_list_layers(get_vsi_path_or_buffer(path_or_buffer))
 
 
 def read_bounds(
@@ -138,8 +135,7 @@ def read_bounds(
 
     Parameters
     ----------
-    path : pathlib.Path or str
-        data source path
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
     layer : int or str, optional (default: first layer)
         If an integer is provided, it corresponds to the index of the layer
         with the data source.  If a string is provided, it must match the name
@@ -177,22 +173,16 @@ def read_bounds(
         bounds are ndarray of shape(4, n) containing ``xmin``, ``ymin``, ``xmax``,
         ``ymax``
     """
-    path, buffer = get_vsi_path(path_or_buffer)
-
-    try:
-        result = ogr_read_bounds(
-            path,
-            layer=layer,
-            skip_features=skip_features,
-            max_features=max_features or 0,
-            where=where,
-            bbox=bbox,
-            mask=_mask_to_wkb(mask),
-        )
-    finally:
-        if buffer is not None:
-            remove_virtual_file(path)
-    return result
+
+    return ogr_read_bounds(
+        get_vsi_path_or_buffer(path_or_buffer),
+        layer=layer,
+        skip_features=skip_features,
+        max_features=max_features or 0,
+        where=where,
+        bbox=bbox,
+        mask=_mask_to_wkb(mask),
+    )
 
 
 def read_info(
@@ -217,9 +207,21 @@ def read_info(
     driver or if the data source is nonspatial. You can force it to be calculated using
     the ``force_total_bounds`` parameter.
 
+    ``fid_column`` is the name of the FID field in the data source, if the FID is
+    physically stored (e.g. in GPKG). If the FID is just a sequence, ``fid_column``
+    will be "" (e.g. ESRI Shapefile).
+
+    ``geometry_name`` is the name of the field where the main geometry is stored in the
+    data data source, if the field name can by customized (e.g. in GPKG). If no custom
+    name is supported, ``geometry_name`` will be "" (e.g. ESRI Shapefile).
+
+    ``encoding`` will be ``UTF-8`` if either the native encoding is likely to be
+    ``UTF-8`` or GDAL can automatically convert from the detected native encoding
+    to ``UTF-8``.
+
     Parameters
     ----------
-    path : str or pathlib.Path
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
     layer : [type], optional
         Name or index of layer in data source.  Reads the first layer by default.
     encoding : [type], optional (default: None)
@@ -240,11 +242,14 @@ def read_info(
         A dictionary with the following keys::
 
             {
+                "layer_name": "<layer name>",
                 "crs": "<crs>",
                 "fields": <ndarray of field names>,
                 "dtypes": <ndarray of field dtypes>,
                 "encoding": "<encoding>",
-                "geometry": "<geometry type>",
+                "fid_column": "<fid column name or "">",
+                "geometry_name": "<geometry column name or "">",
+                "geometry_type": "<geometry type>",
                 "features": <feature count or -1>,
                 "total_bounds": <tuple with total bounds or None>,
                 "driver": "<driver>",
@@ -253,23 +258,17 @@ def read_info(
                 "layer_metadata": "<dict of layer metadata or None>"
             }
     """
-    path, buffer = get_vsi_path(path_or_buffer)
 
     dataset_kwargs = _preprocess_options_key_value(kwargs) if kwargs else {}
 
-    try:
-        result = ogr_read_info(
-            path,
-            layer=layer,
-            encoding=encoding,
-            force_feature_count=force_feature_count,
-            force_total_bounds=force_total_bounds,
-            dataset_kwargs=dataset_kwargs,
-        )
-    finally:
-        if buffer is not None:
-            remove_virtual_file(path)
-    return result
+    return ogr_read_info(
+        get_vsi_path_or_buffer(path_or_buffer),
+        layer=layer,
+        encoding=encoding,
+        force_feature_count=force_feature_count,
+        force_total_bounds=force_total_bounds,
+        dataset_kwargs=dataset_kwargs,
+    )
 
 
 def set_gdal_config_options(options):

pyogrio/gdal_data/GDAL-targets-release.cmake

@@ -8,12 +8,12 @@ set(CMAKE_IMPORT_FILE_VERSION 1)
 # Import target "GDAL::GDAL" for configuration "Release"
 set_property(TARGET GDAL::GDAL APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE)
 set_target_properties(GDAL::GDAL PROPERTIES
-  IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libgdal.so.33.3.7.2"
-  IMPORTED_SONAME_RELEASE "libgdal.so.33"
+  IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/lib/libgdal.so.34.3.8.5"
+  IMPORTED_SONAME_RELEASE "libgdal.so.34"
   )
 
 list(APPEND _cmake_import_check_targets GDAL::GDAL )
-list(APPEND _cmake_import_check_files_for_GDAL::GDAL "${_IMPORT_PREFIX}/lib/libgdal.so.33.3.7.2" )
+list(APPEND _cmake_import_check_files_for_GDAL::GDAL "${_IMPORT_PREFIX}/lib/libgdal.so.34.3.8.5" )
 
 # Commands beyond this point should not need to know the version.
 set(CMAKE_IMPORT_FILE_VERSION)

pyogrio/gdal_data/GDAL-targets.cmake

@@ -7,7 +7,7 @@ if(CMAKE_VERSION VERSION_LESS "2.8.3")
    message(FATAL_ERROR "CMake >= 2.8.3 required")
 endif()
 cmake_policy(PUSH)
-cmake_policy(VERSION 2.8.3...3.25)
+cmake_policy(VERSION 2.8.3...3.27)
 #----------------------------------------------------------------
 # Generated CMake target import file.
 #----------------------------------------------------------------
@@ -74,9 +74,12 @@ set(_IMPORT_PREFIX)
 
 # Loop over all imported files and verify that they actually exist
 foreach(_cmake_target IN LISTS _cmake_import_check_targets)
-  foreach(_cmake_file IN LISTS "_cmake_import_check_files_for_${_cmake_target}")
-    if(NOT EXISTS "${_cmake_file}")
-      message(FATAL_ERROR "The imported target \"${_cmake_target}\" references the file
+  if(CMAKE_VERSION VERSION_LESS "3.28"
+      OR NOT DEFINED _cmake_import_check_xcframework_for_${_cmake_target}
+      OR NOT IS_DIRECTORY "${_cmake_import_check_xcframework_for_${_cmake_target}}")
+    foreach(_cmake_file IN LISTS "_cmake_import_check_files_for_${_cmake_target}")
+      if(NOT EXISTS "${_cmake_file}")
+        message(FATAL_ERROR "The imported target \"${_cmake_target}\" references the file
    \"${_cmake_file}\"
 but this file does not exist.  Possible reasons include:
 * The file was deleted, renamed, or moved to another location.
@@ -85,8 +88,9 @@ but this file does not exist.  Possible reasons include:
    \"${CMAKE_CURRENT_LIST_FILE}\"
 but not all the files it references.
 ")
-    endif()
-  endforeach()
+      endif()
+    endforeach()
+  endif()
   unset(_cmake_file)
   unset("_cmake_import_check_files_for_${_cmake_target}")
 endforeach()

pyogrio/gdal_data/GDALConfigVersion.cmake

@@ -10,13 +10,13 @@
 # The variable CVF_VERSION must be set before calling configure_file().
 
 
-set(PACKAGE_VERSION "3.7.2")
+set(PACKAGE_VERSION "3.8.5")
 
 if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION)
   set(PACKAGE_VERSION_COMPATIBLE FALSE)
 else()
 
-  if("3.7.2" MATCHES "^([0-9]+)\\.([0-9]+)")
+  if("3.8.5" MATCHES "^([0-9]+)\\.([0-9]+)")
     set(CVF_VERSION_MAJOR "${CMAKE_MATCH_1}")
     set(CVF_VERSION_MINOR "${CMAKE_MATCH_2}")
 
@@ -27,7 +27,7 @@ else()
       string(REGEX REPLACE "^0+" "" CVF_VERSION_MINOR "${CVF_VERSION_MINOR}")
     endif()
   else()
-    set(CVF_VERSION_MAJOR "3.7.2")
+    set(CVF_VERSION_MAJOR "3.8.5")
     set(CVF_VERSION_MINOR "")
   endif()
 

pyogrio/gdal_data/gdalinfo_output.schema.json

@@ -176,6 +176,7 @@
           }
         },
         "size": {
+          "$comment": "note that the order of items in side is width,height",
           "$ref": "#/definitions/arrayOfTwoIntegers"
         },
         "coordinateSystem": {
@@ -306,6 +307,7 @@
         },
 
         "proj:shape": {
+          "$comment": "note that the order of items in proj:shape is height,width starting with GDAL 3.8.5 (previous versions ordered it wrongly as width,height)",
           "title": "Shape",
           "type": "array",
           "minItems": 2,