flytekitplugins-geopandas 1.16.0__tar.gz

flytekitplugins_geopandas-1.16.0/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: flytekitplugins-geopandas
+Version: 1.16.0
+Summary: Geopandas plugin for flytekit
+Author: flyteorg
+Author-email: admin@flyte.org
+License: apache2
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: flytekit<2.0.0,>=1.3.0b2
+Requires-Dist: geopandas<2.0.0,>=1.0.0
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

flytekitplugins_geopandas-1.16.0/README.md

@@ -0,0 +1,10 @@
+# Flytekit GeoPandas Plugin
+[GeoPandas](https://geopandas.org/en/stable/) GeoPandas is an open source project to make working with geospatial data in python easier.
+
+This plugin supports `gpd.GeoDataFrame` as a data type with [StructuredDataset](https://docs.flyte.org/en/latest/user_guide/data_types_and_io/structureddataset.html).
+
+To install the plugin, run the following command:
+
+```bash
+pip install flytekitplugins-geopandas
+```

flytekitplugins_geopandas-1.16.0/flytekitplugins/geopandas/__init__.py

@@ -0,0 +1,14 @@
+"""
+.. currentmodule:: flytekitplugins.geopandas
+
+This package contains things that are useful when extending Flytekit.
+
+.. autosummary::
+   :template: custom.rst
+   :toctree: generated/
+
+   GeoPandasDecodingHandler
+   GeoPandasEncodingHandler
+"""
+
+from .gdf_transformers import GeoPandasDecodingHandler, GeoPandasEncodingHandler

flytekitplugins_geopandas-1.16.0/flytekitplugins/geopandas/gdf_transformers.py

@@ -0,0 +1,74 @@
+import typing
+from pathlib import Path
+
+from flytekit import FlyteContext, lazy_module
+from flytekit.models import literals
+from flytekit.models.literals import StructuredDatasetMetadata
+from flytekit.models.types import StructuredDatasetType
+from flytekit.types.structured.structured_dataset import (
+    PARQUET,
+    StructuredDataset,
+    StructuredDatasetDecoder,
+    StructuredDatasetEncoder,
+    StructuredDatasetTransformerEngine,
+)
+
+if typing.TYPE_CHECKING:
+    import pyarrow
+
+    import geopandas as gpd
+else:
+    gpd = lazy_module("geopandas")
+    pyarrow = lazy_module("pyarrow")
+
+
+class GeoPandasDataFrameRenderer:
+    """
+    The Geopandas DataFrame summary statistics are rendered as an HTML table.
+    """
+
+    def to_html(self, df: gpd.GeoDataFrame) -> str:
+        assert isinstance(df, gpd.GeoDataFrame)
+        return df.describe()._repr_html_()
+
+
+class GeoPandasEncodingHandler(StructuredDatasetEncoder):
+    def encode(
+        self,
+        ctx: FlyteContext,
+        structured_dataset: StructuredDataset,
+        structured_dataset_type: StructuredDatasetType,
+    ) -> literals.StructuredDataset:
+        uri = typing.cast(str, structured_dataset.uri) or ctx.file_access.join(
+            ctx.file_access.raw_output_prefix, ctx.file_access.get_random_string()
+        )
+        if not ctx.file_access.is_remote(uri):
+            Path(uri).mkdir(parents=True, exist_ok=True)
+        uri = str(Path(uri) / "data.parquet")
+        df = typing.cast(gpd.GeoDataFrame, structured_dataset.dataframe)
+        df.to_parquet(uri)
+        structured_dataset_type.format = PARQUET
+        return literals.StructuredDataset(uri=uri, metadata=StructuredDatasetMetadata(structured_dataset_type))
+
+
+class GeoPandasDecodingHandler(StructuredDatasetDecoder):
+    def decode(
+        self,
+        ctx: FlyteContext,
+        flyte_value: literals.StructuredDataset,
+        current_task_metadata: StructuredDatasetMetadata,
+    ) -> gpd.GeoDataFrame:
+        # a user may want to bring a non-parquet gdf, which uses a different
+        # opening method.
+        try:
+            return gpd.read_parquet(flyte_value.uri)
+        except pyarrow.lib.ArrowInvalid:
+            return gpd.read_file(flyte_value.uri)
+
+
+StructuredDatasetTransformerEngine.register_renderer(gpd.GeoDataFrame, GeoPandasDataFrameRenderer())
+# We register GeoPandas encoder to support parquet between and from tasks / workflows
+StructuredDatasetTransformerEngine.register(GeoPandasEncodingHandler(gpd.GeoDataFrame, None, PARQUET))
+# We register to any format for decoder in the event a user provides geopackage,
+# shape file, parquet, etc.
+StructuredDatasetTransformerEngine.register(GeoPandasDecodingHandler(gpd.GeoDataFrame, None, None))

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: flytekitplugins-geopandas
+Version: 1.16.0
+Summary: Geopandas plugin for flytekit
+Author: flyteorg
+Author-email: admin@flyte.org
+License: apache2
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Requires-Dist: flytekit<2.0.0,>=1.3.0b2
+Requires-Dist: geopandas<2.0.0,>=1.0.0
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+setup.py
+flytekitplugins/geopandas/__init__.py
+flytekitplugins/geopandas/gdf_transformers.py
+flytekitplugins_geopandas.egg-info/PKG-INFO
+flytekitplugins_geopandas.egg-info/SOURCES.txt
+flytekitplugins_geopandas.egg-info/dependency_links.txt
+flytekitplugins_geopandas.egg-info/entry_points.txt
+flytekitplugins_geopandas.egg-info/namespace_packages.txt
+flytekitplugins_geopandas.egg-info/requires.txt
+flytekitplugins_geopandas.egg-info/top_level.txt
+tests/test_geopandas_plugin.py

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[flytekit.plugins]
+geopandas = flytekitplugins.geopandas

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/namespace_packages.txt

@@ -0,0 +1 @@
+flytekitplugins

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/requires.txt

@@ -0,0 +1,3 @@
+flytekit<2.0.0,>=1.3.0b2
+geopandas<2.0.0,>=1.0.0
+pandas

flytekitplugins_geopandas-1.16.0/flytekitplugins_geopandas.egg-info/top_level.txt

@@ -0,0 +1 @@
+flytekitplugins

flytekitplugins_geopandas-1.16.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

flytekitplugins_geopandas-1.16.0/setup.py

@@ -0,0 +1,35 @@
+from setuptools import setup
+
+PLUGIN_NAME = "geopandas"
+
+microlib_name = f"flytekitplugins-{PLUGIN_NAME}"
+
+plugin_requires = ["flytekit>=1.3.0b2,<2.0.0", "geopandas>=1.0.0,<2.0.0", "pandas"]
+
+__version__ = "v1.16.0"
+
+setup(
+    name=microlib_name,
+    version=__version__,
+    author="flyteorg",
+    author_email="admin@flyte.org",
+    description="Geopandas plugin for flytekit",
+    namespace_packages=["flytekitplugins"],
+    packages=[f"flytekitplugins.{PLUGIN_NAME}"],
+    install_requires=plugin_requires,
+    license="apache2",
+    python_requires=">=3.9",
+    classifiers=[
+        "Intended Audience :: Science/Research",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "Topic :: Software Development",
+        "Topic :: Software Development :: Libraries",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+    ],
+    entry_points={"flytekit.plugins": [f"{PLUGIN_NAME}=flytekitplugins.{PLUGIN_NAME}"]},
+)

flytekitplugins_geopandas-1.16.0/tests/test_geopandas_plugin.py

@@ -0,0 +1,63 @@
+import geopandas as gpd
+from flytekitplugins.geopandas.gdf_transformers import GeoPandasDataFrameRenderer
+from pathlib import Path
+
+import pytest
+
+from flytekit import task
+from flytekit.types.structured.structured_dataset import StructuredDataset
+import numpy as np
+
+
+def test_geopandas_encodes_decodes():
+    @task
+    def _gdf_task(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
+        return gdf
+
+    gdf = gpd.GeoDataFrame(
+        {"geometry": gpd.points_from_xy([0, 1], [0, 1]), "other_column": [1, 2]},
+        crs="EPSG:4326",
+    )
+    rt_gdf = _gdf_task(gdf)
+    assert rt_gdf.equals(gdf)
+
+
+@pytest.mark.parametrize("file_name", ["output.geojson", "output.gpkg"])
+def test_geopandas_encodes_common_formats(tmp_path: Path, file_name: str):
+    @task
+    def _gdf_task(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
+        return gdf
+
+    gdf = gpd.GeoDataFrame(
+        {"other": np.array([1.0, 2.0])},
+        geometry=gpd.points_from_xy([0, 1], [0, 1]),
+        crs="EPSG:4326",
+    )
+    uri = str(tmp_path / file_name)
+    gdf.to_file(uri)
+    rt_gdf = _gdf_task(gdf=StructuredDataset(uri=uri))
+    assert rt_gdf.equals(gdf)
+
+
+def test_geopandas_encodes_shp_not_yet_supported(tmp_path: Path):
+    @task
+    def _gdf_task(gdf: gpd.GeoDataFrame) -> gpd.GeoDataFrame:
+        return gdf
+
+    gdf = gpd.GeoDataFrame(
+        {"geometry": gpd.points_from_xy([0, 1], [0, 1]), "other": [1, 2]},
+        crs="EPSG:4326",
+    )
+    uri = str(tmp_path / "output.shp")
+    gdf.to_file(uri)
+    with pytest.raises(ValueError, match=r"Set SHAPE_RESTORE_SHX config option to YES"):
+        rt_gdf = _gdf_task(gdf=StructuredDataset(uri=uri))
+
+
+def test_gdf_renderer():
+    gdf = gpd.GeoDataFrame(
+        {"geometry": gpd.points_from_xy([0, 1], [0, 1]), "other_column": [1, 2]},
+        crs="EPSG:4326",
+    )
+    described = gdf.describe()._repr_html_()
+    assert GeoPandasDataFrameRenderer().to_html(gdf) == described