pyogrio 0.7.2__cp38-cp38-win_amd64.whl → 0.9.0__cp38-cp38-win_amd64.whl

pyogrio/_vsi.pxd

@@ -0,0 +1,4 @@
+cdef str get_ogr_vsimem_write_path(object path_or_fp, str driver)
+cdef str read_buffer_to_vsimem(bytes bytes_buffer)
+cdef read_vsimem_to_buffer(str path, object out_buffer)
+cdef delete_vsimem_file(str path)

pyogrio/_vsi.pyx

@@ -0,0 +1,140 @@
+from io import BytesIO
+from uuid import uuid4
+
+from libc.stdlib cimport malloc, free
+from libc.string cimport memcpy
+
+from pyogrio._ogr cimport *
+from pyogrio._ogr import _get_driver_metadata_item
+
+
+cdef str get_ogr_vsimem_write_path(object path_or_fp, str driver):
+    """ Return the original path or a /vsimem/ path
+
+    If passed a io.BytesIO object, this will return a /vsimem/ path that can be
+    used to create a new in-memory file with an extension inferred from the driver
+    if possible.  Path will be contained in an in-memory directory to contain
+    sibling files (though drivers that create sibling files are not supported for
+    in-memory files).
+
+    Caller is responsible for deleting the directory via delete_vsimem_file()
+
+    Parameters
+    ----------
+    path_or_fp : str or io.BytesIO object
+    driver : str
+    """
+
+    if not isinstance(path_or_fp, BytesIO):
+        return path_or_fp
+
+    # Create in-memory directory to contain auxiliary files
+    memfilename = uuid4().hex
+    VSIMkdir(f"/vsimem/{memfilename}".encode("UTF-8"), 0666)
+
+    # file extension is required for some drivers, set it based on driver metadata
+    ext = ""
+    recommended_ext = _get_driver_metadata_item(driver, "DMD_EXTENSIONS")
+    if recommended_ext is not None:
+        ext = "." + recommended_ext.split(" ")[0]
+
+    path = f"/vsimem/{memfilename}/{memfilename}{ext}"
+
+    # check for existing bytes
+    if path_or_fp.getbuffer().nbytes > 0:
+        raise NotImplementedError("writing to existing in-memory object is not supported")
+
+    return path
+
+
+cdef str read_buffer_to_vsimem(bytes bytes_buffer):
+    """ Wrap the bytes (zero-copy) into an in-memory dataset
+
+    If the first 4 bytes indicate the bytes are a zip file, the returned path
+    will be prefixed with /vsizip/ and suffixed with .zip to enable proper
+    reading by GDAL.
+
+    Caller is responsible for deleting the in-memory file via delete_vsimem_file().
+
+    Parameters
+    ----------
+    bytes_buffer : bytes
+    """
+    cdef int num_bytes = len(bytes_buffer)
+
+    is_zipped = len(bytes_buffer) > 4 and bytes_buffer[:4].startswith(b"PK\x03\x04")
+    ext = ".zip" if is_zipped else ""
+
+    path = f"/vsimem/{uuid4().hex}{ext}"
+
+    # Create an in-memory object that references bytes_buffer
+    # NOTE: GDAL does not copy the contents of bytes_buffer; it must remain
+    # in scope through the duration of using this file
+    vsi_handle = VSIFileFromMemBuffer(path.encode("UTF-8"), <unsigned char *>bytes_buffer, num_bytes, 0)
+
+    if vsi_handle == NULL:
+        raise OSError("failed to read buffer into in-memory file")
+
+    if VSIFCloseL(vsi_handle) != 0:
+        raise OSError("failed to close in-memory file")
+
+    if is_zipped:
+        path = f"/vsizip/{path}"
+
+    return path
+
+
+cdef read_vsimem_to_buffer(str path, object out_buffer):
+    """Copy bytes from in-memory file to buffer
+
+    This will automatically unlink the in-memory file pointed to by path; caller
+    is still responsible for calling delete_vsimem_file() to cleanup any other
+    files contained in the in-memory directory.
+
+    Parameters:
+    -----------
+    path : str
+        path to in-memory file
+    buffer : BytesIO object
+    """
+
+    cdef unsigned char *vsi_buffer = NULL
+    cdef vsi_l_offset vsi_buffer_size = 0
+
+    try:
+        # Take ownership of the buffer to avoid a copy; GDAL will automatically
+        # unlink the memory file
+        vsi_buffer = VSIGetMemFileBuffer(path.encode("UTF-8"), &vsi_buffer_size, 1)
+        if vsi_buffer == NULL:
+            raise RuntimeError("could not read bytes from in-memory file")
+
+        # write bytes to buffer
+        out_buffer.write(<bytes>vsi_buffer[:vsi_buffer_size])
+        # rewind to beginning to allow caller to read
+        out_buffer.seek(0)
+
+    finally:
+        if vsi_buffer != NULL:
+            CPLFree(vsi_buffer)
+
+
+cdef delete_vsimem_file(str path):
+    """ Recursively delete in-memory path or directory containing path
+
+    This is used for final cleanup of an in-memory dataset, which may have been
+    created within a directory to contain sibling files.
+
+    Additional VSI handlers may be chained to the left of /vsimem/ in path and
+    will be ignored.
+
+    Parameters:
+    -----------
+    path : str
+        path to in-memory file
+    """
+
+    if "/vsimem/" not in path:
+        return
+
+    root = "/vsimem/" + path.split("/vsimem/")[1].split("/")[0]
+    VSIRmdirRecursive(root.encode("UTF-8"))

pyogrio/core.py

@@ -1,5 +1,9 @@
 from pyogrio._env import GDALEnv
-from pyogrio.util import get_vsi_path, _preprocess_options_key_value, _mask_to_wkb
+from pyogrio.util import (
+    get_vsi_path_or_buffer,
+    _preprocess_options_key_value,
+    _mask_to_wkb,
+)
 
 
 with GDALEnv():
@@ -13,7 +17,6 @@ with GDALEnv():
         get_gdal_data_path as _get_gdal_data_path,
         init_gdal_data as _init_gdal_data,
         init_proj_data as _init_proj_data,
-        remove_virtual_file,
         _register_drivers,
         _get_drivers_for_path,
     )
@@ -102,7 +105,7 @@ def list_layers(path_or_buffer, /):
 
     Parameters
     ----------
-    path : str or pathlib.Path
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
 
     Returns
     -------
@@ -110,14 +113,8 @@ def list_layers(path_or_buffer, /):
         array of pairs of [<layer name>, <layer geometry type>]
         Note: geometry is `None` for nonspatial layers.
     """
-    path, buffer = get_vsi_path(path_or_buffer)
 
-    try:
-        result = ogr_list_layers(path)
-    finally:
-        if buffer is not None:
-            remove_virtual_file(path)
-    return result
+    return ogr_list_layers(get_vsi_path_or_buffer(path_or_buffer))
 
 
 def read_bounds(
@@ -138,8 +135,7 @@ def read_bounds(
 
     Parameters
     ----------
-    path : pathlib.Path or str
-        data source path
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
     layer : int or str, optional (default: first layer)
         If an integer is provided, it corresponds to the index of the layer
         with the data source.  If a string is provided, it must match the name
@@ -177,22 +173,16 @@ def read_bounds(
         bounds are ndarray of shape(4, n) containing ``xmin``, ``ymin``, ``xmax``,
         ``ymax``
     """
-    path, buffer = get_vsi_path(path_or_buffer)
-
-    try:
-        result = ogr_read_bounds(
-            path,
-            layer=layer,
-            skip_features=skip_features,
-            max_features=max_features or 0,
-            where=where,
-            bbox=bbox,
-            mask=_mask_to_wkb(mask),
-        )
-    finally:
-        if buffer is not None:
-            remove_virtual_file(path)
-    return result
+
+    return ogr_read_bounds(
+        get_vsi_path_or_buffer(path_or_buffer),
+        layer=layer,
+        skip_features=skip_features,
+        max_features=max_features or 0,
+        where=where,
+        bbox=bbox,
+        mask=_mask_to_wkb(mask),
+    )
 
 
 def read_info(
@@ -217,9 +207,21 @@ def read_info(
     driver or if the data source is nonspatial. You can force it to be calculated using
     the ``force_total_bounds`` parameter.
 
+    ``fid_column`` is the name of the FID field in the data source, if the FID is
+    physically stored (e.g. in GPKG). If the FID is just a sequence, ``fid_column``
+    will be "" (e.g. ESRI Shapefile).
+
+    ``geometry_name`` is the name of the field where the main geometry is stored in the
+    data data source, if the field name can by customized (e.g. in GPKG). If no custom
+    name is supported, ``geometry_name`` will be "" (e.g. ESRI Shapefile).
+
+    ``encoding`` will be ``UTF-8`` if either the native encoding is likely to be
+    ``UTF-8`` or GDAL can automatically convert from the detected native encoding
+    to ``UTF-8``.
+
     Parameters
     ----------
-    path : str or pathlib.Path
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
     layer : [type], optional
         Name or index of layer in data source.  Reads the first layer by default.
     encoding : [type], optional (default: None)
@@ -240,11 +242,14 @@ def read_info(
         A dictionary with the following keys::
 
             {
+                "layer_name": "<layer name>",
                 "crs": "<crs>",
                 "fields": <ndarray of field names>,
                 "dtypes": <ndarray of field dtypes>,
                 "encoding": "<encoding>",
-                "geometry": "<geometry type>",
+                "fid_column": "<fid column name or "">",
+                "geometry_name": "<geometry column name or "">",
+                "geometry_type": "<geometry type>",
                 "features": <feature count or -1>,
                 "total_bounds": <tuple with total bounds or None>,
                 "driver": "<driver>",
@@ -253,23 +258,17 @@ def read_info(
                 "layer_metadata": "<dict of layer metadata or None>"
             }
     """
-    path, buffer = get_vsi_path(path_or_buffer)
 
     dataset_kwargs = _preprocess_options_key_value(kwargs) if kwargs else {}
 
-    try:
-        result = ogr_read_info(
-            path,
-            layer=layer,
-            encoding=encoding,
-            force_feature_count=force_feature_count,
-            force_total_bounds=force_total_bounds,
-            dataset_kwargs=dataset_kwargs,
-        )
-    finally:
-        if buffer is not None:
-            remove_virtual_file(path)
-    return result
+    return ogr_read_info(
+        get_vsi_path_or_buffer(path_or_buffer),
+        layer=layer,
+        encoding=encoding,
+        force_feature_count=force_feature_count,
+        force_total_bounds=force_total_bounds,
+        dataset_kwargs=dataset_kwargs,
+    )
 
 
 def set_gdal_config_options(options):

pyogrio/gdal_data/GDAL-targets-release.cmake

@@ -9,7 +9,7 @@ set(CMAKE_IMPORT_FILE_VERSION 1)
 set_property(TARGET GDAL::GDAL APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE)
 set_target_properties(GDAL::GDAL PROPERTIES
   IMPORTED_IMPLIB_RELEASE "${_IMPORT_PREFIX}/lib/gdal.lib"
-  IMPORTED_LINK_DEPENDENT_LIBRARIES_RELEASE "json-c::json-c;CURL::libcurl;GEOS::geos_c;PROJ::proj;Qhull::qhull_r;expat::expat;geotiff_library"
+  IMPORTED_LINK_DEPENDENT_LIBRARIES_RELEASE "json-c::json-c;GEOS::geos_c;PROJ::proj;Qhull::qhull_r;expat::expat;geotiff_library"
   IMPORTED_LOCATION_RELEASE "${_IMPORT_PREFIX}/bin/gdal.dll"
   )
 

pyogrio/gdal_data/GDAL-targets.cmake

@@ -7,7 +7,7 @@ if(CMAKE_VERSION VERSION_LESS "2.8.3")
    message(FATAL_ERROR "CMake >= 2.8.3 required")
 endif()
 cmake_policy(PUSH)
-cmake_policy(VERSION 2.8.3...3.25)
+cmake_policy(VERSION 2.8.3...3.27)
 #----------------------------------------------------------------
 # Generated CMake target import file.
 #----------------------------------------------------------------
@@ -74,9 +74,12 @@ set(_IMPORT_PREFIX)
 
 # Loop over all imported files and verify that they actually exist
 foreach(_cmake_target IN LISTS _cmake_import_check_targets)
-  foreach(_cmake_file IN LISTS "_cmake_import_check_files_for_${_cmake_target}")
-    if(NOT EXISTS "${_cmake_file}")
-      message(FATAL_ERROR "The imported target \"${_cmake_target}\" references the file
+  if(CMAKE_VERSION VERSION_LESS "3.28"
+      OR NOT DEFINED _cmake_import_check_xcframework_for_${_cmake_target}
+      OR NOT IS_DIRECTORY "${_cmake_import_check_xcframework_for_${_cmake_target}}")
+    foreach(_cmake_file IN LISTS "_cmake_import_check_files_for_${_cmake_target}")
+      if(NOT EXISTS "${_cmake_file}")
+        message(FATAL_ERROR "The imported target \"${_cmake_target}\" references the file
    \"${_cmake_file}\"
 but this file does not exist.  Possible reasons include:
 * The file was deleted, renamed, or moved to another location.
@@ -85,8 +88,9 @@ but this file does not exist.  Possible reasons include:
    \"${CMAKE_CURRENT_LIST_FILE}\"
 but not all the files it references.
 ")
-    endif()
-  endforeach()
+      endif()
+    endforeach()
+  endif()
   unset(_cmake_file)
   unset("_cmake_import_check_files_for_${_cmake_target}")
 endforeach()

pyogrio/gdal_data/GDALConfigVersion.cmake

@@ -10,13 +10,13 @@
 # The variable CVF_VERSION must be set before calling configure_file().
 
 
-set(PACKAGE_VERSION "3.7.2")
+set(PACKAGE_VERSION "3.8.5")
 
 if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION)
   set(PACKAGE_VERSION_COMPATIBLE FALSE)
 else()
 
-  if("3.7.2" MATCHES "^([0-9]+)\\.([0-9]+)")
+  if("3.8.5" MATCHES "^([0-9]+)\\.([0-9]+)")
     set(CVF_VERSION_MAJOR "${CMAKE_MATCH_1}")
     set(CVF_VERSION_MINOR "${CMAKE_MATCH_2}")
 
@@ -27,7 +27,7 @@ else()
       string(REGEX REPLACE "^0+" "" CVF_VERSION_MINOR "${CVF_VERSION_MINOR}")
     endif()
   else()
-    set(CVF_VERSION_MAJOR "3.7.2")
+    set(CVF_VERSION_MAJOR "3.8.5")
     set(CVF_VERSION_MINOR "")
   endif()
 

pyogrio/gdal_data/gdalinfo_output.schema.json

@@ -176,6 +176,7 @@
           }
         },
         "size": {
+          "$comment": "note that the order of items in side is width,height",
           "$ref": "#/definitions/arrayOfTwoIntegers"
         },
         "coordinateSystem": {
@@ -306,6 +307,7 @@
         },
 
         "proj:shape": {
+          "$comment": "note that the order of items in proj:shape is height,width starting with GDAL 3.8.5 (previous versions ordered it wrongly as width,height)",
           "title": "Shape",
           "type": "array",
           "minItems": 2,

pyogrio/gdal_data/gdalvrt.xsd

@@ -138,6 +138,7 @@
     <xs:complexType name="PanchroBandType">
         <xs:sequence>
             <xs:element name="SourceFilename" type="SourceFilenameType"/>
+            <xs:element name="OpenOptions" type="OpenOptionsType"/>
             <xs:element name="SourceBand" type="xs:string"/>  <!-- should be refined into xs:nonNegativeInteger or mask,xs:nonNegativeInteger -->
         </xs:sequence>
     </xs:complexType>
@@ -145,6 +146,7 @@
     <xs:complexType name="SpectralBandType">
         <xs:sequence>
             <xs:element name="SourceFilename" type="SourceFilenameType"/>
+            <xs:element name="OpenOptions" type="OpenOptionsType"/>
             <xs:element name="SourceBand" type="xs:string"/>  <!-- should be refined into xs:nonNegativeInteger or mask,xs:nonNegativeInteger -->
         </xs:sequence>
         <xs:attribute name="dstBand" type="xs:nonNegativeInteger"/>
@@ -188,6 +190,7 @@
                 <xs:element name="ComplexSource" type="ComplexSourceType"/>
                 <xs:element name="AveragedSource" type="SimpleSourceType"/>
                 <xs:element name="KernelFilteredSource" type="KernelFilteredSourceType"/>
+                <xs:element name="ArraySource" type="ArraySourceType"/>
 
                 <!-- for a VRTDerivedRasterBand -->
                 <xs:element name="PixelFunctionType" type="xs:string"/>
@@ -435,6 +438,31 @@
         </xs:restriction>
     </xs:simpleType>
 
+    <xs:complexType name="ArraySourceType">
+        <xs:sequence>
+            <xs:element ref="AbstractArray"/>
+            <xs:element name="SrcRect" type="RectType" minOccurs="0"/>
+            <xs:element name="DstRect" type="RectType" minOccurs="0"/>
+        </xs:sequence>
+    </xs:complexType>
+
+    <xs:element name="AbstractArray" type="AbstractArrayType" abstract="true"/>
+
+    <xs:complexType name="AbstractArrayType"/>
+
+    <xs:element name="SingleSourceArray" substitutionGroup="AbstractArray" type="SingleSourceArrayType"/>
+
+    <xs:complexType name="SingleSourceArrayType">
+        <xs:complexContent>
+            <xs:extension base="AbstractArrayType">
+                <xs:sequence>
+                    <xs:element name="SourceFilename" type="SourceFilenameType"/>
+                    <xs:element name="SourceArray" type="xs:string"/>
+                </xs:sequence>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
     <xs:complexType name="SourceFilenameType">
         <xs:simpleContent>
             <xs:extension base="xs:string">
@@ -606,4 +634,139 @@
         <xs:attribute name="ref" type="xs:string" use="required"/>
     </xs:complexType>
 
+    <xs:element name="DerivedArray" substitutionGroup="AbstractArray" type="DerivedArrayType"/>
+
+    <xs:complexType name="DerivedArrayType">
+        <xs:complexContent>
+            <xs:extension base="AbstractArrayType">
+                <xs:sequence>
+                    <xs:element ref="AbstractArray"/>
+                    <xs:element name="Step" type="StepType" minOccurs="0" maxOccurs="unbounded"/>
+                </xs:sequence>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
+    <xs:complexType name="StepType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+            <xs:element ref="AbstractStep"/>
+        </xs:sequence>
+    </xs:complexType>
+
+    <xs:element name="AbstractStep" type="AbstractStepType" abstract="true"/>
+
+    <xs:complexType name="AbstractStepType"/>
+
+    <xs:element name="View" substitutionGroup="AbstractStep" type="ViewType"/>
+
+    <xs:complexType name="ViewType">
+        <xs:complexContent>
+            <xs:extension base="AbstractStepType">
+                <xs:attribute name="expr" type="xs:string" use="required"/>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
+    <xs:element name="Transpose" substitutionGroup="AbstractStep" type="TransposeType"/>
+
+    <xs:complexType name="TransposeType">
+        <xs:complexContent>
+            <xs:extension base="AbstractStepType">
+                <xs:attribute name="newOrder" type="CommaSeparatedListOfIntegerType" use="required"/>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
+    <xs:simpleType name="CommaSeparatedListOfIntegerType">
+      <xs:restriction base="xs:string">
+        <xs:pattern value="(\d)+(,(\d)+).*"/>
+      </xs:restriction>
+    </xs:simpleType>
+
+    <xs:element name="Resample" substitutionGroup="AbstractStep" type="ResampleType"/>
+
+    <xs:complexType name="ResampleType">
+        <xs:complexContent>
+            <xs:extension base="AbstractStepType">
+                <xs:sequence>
+                    <xs:element name="Dimension" type="DimensionType" minOccurs="0" maxOccurs="unbounded"/>
+                    <xs:element name="ResampleAlg" type="ResampleAlgType" minOccurs="0"/>
+                    <xs:element name="SRS" type="SRSType" minOccurs="0"/>
+                    <xs:element name="Option" type="OptionType" minOccurs="0" maxOccurs="unbounded"/>
+                </xs:sequence>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
+    <xs:simpleType name="ResampleAlgType">
+        <xs:restriction base="xs:string">
+            <xs:enumeration value="NearestNeighbour"/>
+            <xs:enumeration value="Bilinear"/>
+            <xs:enumeration value="Cubic"/>
+            <xs:enumeration value="CubicSpline"/>
+            <xs:enumeration value="Lanczos"/>
+            <xs:enumeration value="Average"/>
+            <xs:enumeration value="RMS"/>
+            <xs:enumeration value="Mode"/>
+            <xs:enumeration value="Gauss"/>
+        </xs:restriction>
+    </xs:simpleType>
+
+    <xs:complexType name="OptionType">
+        <xs:simpleContent>
+            <xs:extension base="xs:string">
+                <xs:attribute name="name" type="xs:string"/>
+            </xs:extension>
+        </xs:simpleContent>
+    </xs:complexType>
+
+    <xs:element name="Grid" substitutionGroup="AbstractStep" type="GridType"/>
+
+    <xs:complexType name="GridType">
+        <xs:complexContent>
+            <xs:extension base="AbstractStepType">
+                <xs:sequence>
+                    <xs:element name="GridOptions" type="xs:string" minOccurs="1"/>
+                    <xs:element name="XArray" minOccurs="0">
+                        <xs:complexType>
+                            <xs:sequence>
+                                <xs:element ref="AbstractArray"/>
+                            </xs:sequence>
+                        </xs:complexType>
+                    </xs:element>
+                    <xs:element name="YArray" minOccurs="0">
+                        <xs:complexType>
+                            <xs:sequence>
+                                <xs:element ref="AbstractArray"/>
+                            </xs:sequence>
+                        </xs:complexType>
+                    </xs:element>
+                    <xs:element name="Option" type="OptionType" minOccurs="0" maxOccurs="unbounded"/>
+                </xs:sequence>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
+    <xs:element name="GetMask" substitutionGroup="AbstractStep" type="GetMaskType"/>
+
+    <xs:complexType name="GetMaskType">
+        <xs:complexContent>
+            <xs:extension base="AbstractStepType">
+                <xs:sequence>
+                    <xs:element name="Option" type="OptionType" minOccurs="0" maxOccurs="unbounded"/>
+                </xs:sequence>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
+    <xs:element name="GetUnscaled" substitutionGroup="AbstractStep" type="GetUnscaledType"/>
+
+    <xs:complexType name="GetUnscaledType">
+        <xs:complexContent>
+            <xs:extension base="AbstractStepType">
+                <xs:sequence/>
+            </xs:extension>
+        </xs:complexContent>
+    </xs:complexType>
+
 </xs:schema>

pyogrio/gdal_data/ogrinfo_output.schema.json

@@ -149,6 +149,10 @@
         },
         "comment": {
           "type": "string"
+        },
+        "timezone": {
+          "type": "string",
+          "pattern": "^(localtime|(mixed timezones)|UTC|((\\+|-)[0-9][0-9]:[0-9][0-9]))$"
         }
       },
       "required": [
@@ -209,7 +213,14 @@
           }
         },
         "coordinateSystem": {
-          "$ref": "#/definitions/coordinateSystem"
+          "oneOf": [
+            {
+              "type": "null"
+            },
+            {
+              "$ref": "#/definitions/coordinateSystem"
+            }
+          ]
         },
         "supportedSRSList": {
           "type": "array",