pyogrio 0.10.0__cp313-cp313-manylinux_2_28_aarch64.whl

pyogrio/tests/test_util.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from pyogrio import vsi_listtree, vsi_unlink
+from pyogrio.raw import read, write
+from pyogrio.util import vsimem_rmtree_toplevel
+
+import pytest
+
+
+def test_vsimem_rmtree_toplevel(naturalearth_lowres):
+    # Prepare test data in /vsimem/
+    meta, _, geometry, field_data = read(naturalearth_lowres)
+    meta["spatial_index"] = False
+    meta["geometry_type"] = "MultiPolygon"
+    test_dir_path = Path(f"/vsimem/test/{naturalearth_lowres.stem}.gpkg")
+    test_dir2_path = Path(f"/vsimem/test2/test2/{naturalearth_lowres.stem}.gpkg")
+
+    write(test_dir_path, geometry, field_data, **meta)
+    write(test_dir2_path, geometry, field_data, **meta)
+
+    # Check if everything was created properly with listtree
+    files = vsi_listtree("/vsimem/")
+    assert test_dir_path.as_posix() in files
+    assert test_dir2_path.as_posix() in files
+
+    # Test deleting parent dir of file in single directory
+    vsimem_rmtree_toplevel(test_dir_path)
+    files = vsi_listtree("/vsimem/")
+    assert test_dir_path.parent.as_posix() not in files
+    assert test_dir2_path.as_posix() in files
+
+    # Test deleting top-level dir of file in a subdirectory
+    vsimem_rmtree_toplevel(test_dir2_path)
+    assert test_dir2_path.as_posix() not in vsi_listtree("/vsimem/")
+
+
+def test_vsimem_rmtree_toplevel_error(naturalearth_lowres):
+    # Prepare test data in /vsimem
+    meta, _, geometry, field_data = read(naturalearth_lowres)
+    meta["spatial_index"] = False
+    meta["geometry_type"] = "MultiPolygon"
+    test_file_path = Path(f"/vsimem/pyogrio_test_{naturalearth_lowres.stem}.gpkg")
+
+    write(test_file_path, geometry, field_data, **meta)
+    assert test_file_path.as_posix() in vsi_listtree("/vsimem/")
+
+    # Deleting parent dir of non-existent file should raise an error.
+    with pytest.raises(FileNotFoundError, match="Path does not exist"):
+        vsimem_rmtree_toplevel("/vsimem/test/non-existent.gpkg")
+
+    # File should still be there
+    assert test_file_path.as_posix() in vsi_listtree("/vsimem/")
+
+    # Cleanup.
+    vsi_unlink(test_file_path)
+    assert test_file_path not in vsi_listtree("/vsimem/")

pyogrio/util.py

@@ -0,0 +1,247 @@
+"""Utility functions."""
+
+import re
+import sys
+from packaging.version import Version
+from pathlib import Path
+from typing import Union
+from urllib.parse import urlparse
+
+from pyogrio._vsi import vsimem_rmtree_toplevel as _vsimem_rmtree_toplevel
+
+
+def get_vsi_path_or_buffer(path_or_buffer):
+    """Get VSI-prefixed path or bytes buffer depending on type of path_or_buffer.
+
+    If path_or_buffer is a bytes object, it will be returned directly and will
+    be read into an in-memory dataset when passed to one of the Cython functions.
+
+    If path_or_buffer is a file-like object with a read method, bytes will be
+    read from the file-like object and returned.
+
+    Otherwise, it will be converted to a string, and parsed to prefix with
+    appropriate GDAL /vsi*/ prefixes.
+
+    Parameters
+    ----------
+    path_or_buffer : str, pathlib.Path, bytes, or file-like
+        A dataset path or URI, raw buffer, or file-like object with a read method.
+
+    Returns
+    -------
+    str or bytes
+
+    """
+    # treat Path objects here already to ignore their read method + to avoid backslashes
+    # on Windows.
+    if isinstance(path_or_buffer, Path):
+        return vsi_path(path_or_buffer)
+
+    if isinstance(path_or_buffer, bytes):
+        return path_or_buffer
+
+    if hasattr(path_or_buffer, "read"):
+        bytes_buffer = path_or_buffer.read()
+
+        # rewind buffer if possible so that subsequent operations do not need to rewind
+        if hasattr(path_or_buffer, "seekable") and path_or_buffer.seekable():
+            path_or_buffer.seek(0)
+
+        return bytes_buffer
+
+    return vsi_path(str(path_or_buffer))
+
+
+def vsi_path(path: Union[str, Path]) -> str:
+    """Ensure path is a local path or a GDAL-compatible VSI path."""
+    # Convert Path objects to string, but for VSI paths, keep posix style path.
+    if isinstance(path, Path):
+        if sys.platform == "win32" and path.as_posix().startswith("/vsi"):
+            path = path.as_posix()
+        else:
+            path = str(path)
+
+    # path is already in GDAL format
+    if path.startswith("/vsi"):
+        return path
+
+    # Windows drive letters (e.g. "C:\") confuse `urlparse` as they look like
+    # URL schemes
+    if sys.platform == "win32" and re.match("^[a-zA-Z]\\:", path):
+        if not path.split("!")[0].endswith(".zip"):
+            return path
+
+        # prefix then allow to proceed with remaining parsing
+        path = f"zip://{path}"
+
+    path, archive, scheme = _parse_uri(path)
+
+    if scheme or archive or path.endswith(".zip"):
+        return _construct_vsi_path(path, archive, scheme)
+
+    return path
+
+
+# Supported URI schemes and their mapping to GDAL's VSI suffix.
+SCHEMES = {
+    "file": "file",
+    "zip": "zip",
+    "tar": "tar",
+    "gzip": "gzip",
+    "http": "curl",
+    "https": "curl",
+    "ftp": "curl",
+    "s3": "s3",
+    "gs": "gs",
+    "az": "az",
+    "adls": "adls",
+    "adl": "adls",  # fsspec uses this
+    "hdfs": "hdfs",
+    "webhdfs": "webhdfs",
+    # GDAL additionally supports oss and swift for remote filesystems, but
+    # those are for now not added as supported URI
+}
+
+CURLSCHEMES = {k for k, v in SCHEMES.items() if v == "curl"}
+
+
+def _parse_uri(path: str):
+    """Parse a URI.
+
+    Returns a tuples of (path, archive, scheme)
+
+    path : str
+        Parsed path. Includes the hostname and query string in the case
+        of a URI.
+    archive : str
+        Parsed archive path.
+    scheme : str
+        URI scheme such as "https" or "zip+s3".
+    """
+    parts = urlparse(path, allow_fragments=False)
+
+    # if the scheme is not one of GDAL's supported schemes, return raw path
+    if parts.scheme and not all(p in SCHEMES for p in parts.scheme.split("+")):
+        return path, "", ""
+
+    # we have a URI
+    path = parts.path
+    scheme = parts.scheme or ""
+
+    if parts.query:
+        path += "?" + parts.query
+
+    if parts.scheme and parts.netloc:
+        path = parts.netloc + path
+
+    parts = path.split("!")
+    path = parts.pop() if parts else ""
+    archive = parts.pop() if parts else ""
+    return (path, archive, scheme)
+
+
+def _construct_vsi_path(path, archive, scheme) -> str:
+    """Convert a parsed path to a GDAL VSI path."""
+    prefix = ""
+    suffix = ""
+    schemes = scheme.split("+")
+
+    if "zip" not in schemes and (archive.endswith(".zip") or path.endswith(".zip")):
+        schemes.insert(0, "zip")
+
+    if schemes:
+        prefix = "/".join(f"vsi{SCHEMES[p]}" for p in schemes if p and p != "file")
+
+        if schemes[-1] in CURLSCHEMES:
+            suffix = f"{schemes[-1]}://"
+
+    if prefix:
+        if archive:
+            return "/{}/{}{}/{}".format(prefix, suffix, archive, path.lstrip("/"))
+        else:
+            return f"/{prefix}/{suffix}{path}"
+
+    return path
+
+
+def _preprocess_options_key_value(options):
+    """Preprocess options.
+
+    For example, `spatial_index=True` gets converted to `SPATIAL_INDEX="YES"`.
+    """
+    if not isinstance(options, dict):
+        raise TypeError(f"Expected options to be a dict, got {type(options)}")
+
+    result = {}
+    for k, v in options.items():
+        if v is None:
+            continue
+        k = k.upper()
+        if isinstance(v, bool):
+            v = "ON" if v else "OFF"
+        else:
+            v = str(v)
+        result[k] = v
+    return result
+
+
+def _mask_to_wkb(mask):
+    """Convert a Shapely mask geometry to WKB.
+
+    Parameters
+    ----------
+    mask : Shapely geometry
+        The geometry to convert to WKB.
+
+    Returns
+    -------
+    WKB bytes or None
+
+    Raises
+    ------
+    ValueError
+        raised if Shapely >= 2.0 is not available or mask is not a Shapely
+        Geometry object
+
+    """
+    if mask is None:
+        return mask
+
+    try:
+        import shapely
+
+        if Version(shapely.__version__) < Version("2.0.0"):
+            shapely = None
+    except ImportError:
+        shapely = None
+
+    if not shapely:
+        raise ValueError("'mask' parameter requires Shapely >= 2.0")
+
+    if not isinstance(mask, shapely.Geometry):
+        raise ValueError("'mask' parameter must be a Shapely geometry")
+
+    return shapely.to_wkb(mask)
+
+
+def vsimem_rmtree_toplevel(path: Union[str, Path]):
+    """Remove the parent directory of the file path recursively.
+
+    This is used for final cleanup of an in-memory dataset, which may have been
+    created within a directory to contain sibling files.
+
+    Additional VSI handlers may be chained to the left of /vsimem/ in path and
+    will be ignored.
+
+    Remark: function is defined here to be able to run tests on it.
+
+    Parameters
+    ----------
+    path : str or pathlib.Path
+        path to in-memory file
+
+    """
+    if isinstance(path, Path):
+        path = path.as_posix()
+
+    _vsimem_rmtree_toplevel(path)

pyogrio-0.10.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020-2024 Brendan C. Ward and pyogrio contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyogrio-0.10.0.dist-info/METADATA

@@ -0,0 +1,129 @@
+Metadata-Version: 2.1
+Name: pyogrio
+Version: 0.10.0
+Summary: Vectorized spatial vector file format I/O using GDAL/OGR
+Author: pyogrio contributors
+Author-email: "Brendan C. Ward" <bcward@astutespruce.com>
+Maintainer: pyogrio contributors
+License: MIT License
+        
+        Copyright (c) 2020-2024 Brendan C. Ward and pyogrio contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Home, https://pyogrio.readthedocs.io/
+Project-URL: Repository, https://github.com/geopandas/pyogrio
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: GIS
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: certifi
+Requires-Dist: numpy
+Requires-Dist: packaging
+Provides-Extra: benchmark
+Requires-Dist: pytest-benchmark ; extra == 'benchmark'
+Provides-Extra: dev
+Requires-Dist: cython ; extra == 'dev'
+Provides-Extra: geopandas
+Requires-Dist: geopandas ; extra == 'geopandas'
+Provides-Extra: test
+Requires-Dist: pytest ; extra == 'test'
+Requires-Dist: pytest-cov ; extra == 'test'
+
+# pyogrio - Vectorized spatial vector file format I/O using GDAL/OGR
+
+Pyogrio provides a
+[GeoPandas](https://github.com/geopandas/geopandas)-oriented API to OGR vector
+data sources, such as ESRI Shapefile, GeoPackage, and GeoJSON. Vector data sources
+have geometries, such as points, lines, or polygons, and associated records
+with potentially many columns worth of data.
+
+Pyogrio uses a vectorized approach for reading and writing GeoDataFrames to and
+from OGR vector data sources in order to give you faster interoperability. It
+uses pre-compiled bindings for GDAL/OGR so that the performance is primarily
+limited by the underlying I/O speed of data source drivers in GDAL/OGR rather
+than multiple steps of converting to and from Python data types within Python.
+
+We have seen \>5-10x speedups reading files and \>5-20x speedups writing files
+compared to using non-vectorized approaches (Fiona and current I/O support in
+GeoPandas).
+
+You can read these data sources into
+`GeoDataFrames`, read just the non-geometry columns into Pandas `DataFrames`,
+or even read non-spatial data sources that exist alongside vector data sources,
+such as tables in a ESRI File Geodatabase, or antiquated DBF files.
+
+Pyogrio also enables you to write `GeoDataFrames` to at least a few different
+OGR vector data source formats.
+
+Read the documentation for more information:
+[https://pyogrio.readthedocs.io](https://pyogrio.readthedocs.io/en/latest/).
+
+WARNING: Pyogrio is still at an early version and the API is subject to
+substantial change. Please see [CHANGES](CHANGES.md).
+
+## Requirements
+
+Supports Python 3.9 - 3.13 and GDAL 3.4.x - 3.9.x.
+
+Reading to GeoDataFrames requires `geopandas>=0.12` with `shapely>=2`.
+
+Additionally, installing `pyarrow` in combination with GDAL 3.6+ enables
+a further speed-up when specifying `use_arrow=True`.
+
+## Installation
+
+Pyogrio is currently available on
+[conda-forge](https://anaconda.org/conda-forge/pyogrio)
+and [PyPI](https://pypi.org/project/pyogrio/)
+for Linux, MacOS, and Windows.
+
+Please read the
+[installation documentation](https://pyogrio.readthedocs.io/en/latest/install.html)
+for more information.
+
+## Supported vector formats
+
+Pyogrio supports some of the most common vector data source formats (provided
+they are also supported by GDAL/OGR), including ESRI Shapefile, GeoPackage,
+GeoJSON, and FlatGeobuf.
+
+Please see the [list of supported formats](https://pyogrio.readthedocs.io/en/latest/supported_formats.html)
+for more information.
+
+## Getting started
+
+Please read the [introduction](https://pyogrio.readthedocs.io/en/latest/supported_formats.html)
+for more information and examples to get started using Pyogrio.
+
+You can also check out the the [API documentation](https://pyogrio.readthedocs.io/en/latest/api.html)
+for full details on using the API.
+
+## Credits
+
+This project is made possible by the tremendous efforts of the GDAL, Fiona, and
+Geopandas communities.
+
+-   Core I/O methods and supporting functions adapted from [Fiona](https://github.com/Toblerity/Fiona)
+-   Inspired by [Fiona PR](https://github.com/Toblerity/Fiona/pull/540/files)