eodash_catalog 0.3.1__tar.gz → 0.3.2__tar.gz

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.1
+current_version = 0.3.2
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/Dockerfile

@@ -29,4 +29,4 @@ RUN eodash_catalog --help
 
 CMD ["eodash_catalog"]
 
-LABEL version="0.3.1"
+LABEL version="0.3.2"

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eodash_catalog
-Version: 0.3.1
+Version: 0.3.2
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/src/eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.3.1"
+__version__ = "0.3.2"

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/src/eodash_catalog/endpoints.py

@@ -1,4 +1,5 @@
 import importlib
+import io
 import json
 import os
 import sys
@@ -8,6 +9,7 @@ from datetime import datetime, timedelta
 from itertools import groupby
 from operator import itemgetter
 
+import pyarrow.parquet as pq
 import requests
 from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries, TemporalExtent
 from pystac_client import Client
@@ -28,6 +30,7 @@ from eodash_catalog.thumbnails import generate_thumbnail
 from eodash_catalog.utils import (
     Options,
     create_geometry_from_bbox,
+    extract_extent_from_geoparquet,
     filter_time_entries,
     format_datetime_to_isostring_zulu,
     generate_veda_cog_link,
@@ -1218,6 +1221,33 @@ def handle_raw_source(
         # eodash v4 compatibility, adding last referenced style to collection
         if style_link:
             collection.add_link(style_link)
+    elif endpoint_config.get("ParquetSource"):
+        # if parquet source is provided, download it and create items from it
+        parquet_source = endpoint_config["ParquetSource"]
+        if parquet_source.startswith("http"):
+            # download parquet file
+            parquet_file = requests.get(parquet_source)
+            if parquet_file.status_code != 200:
+                LOGGER.error(f"Failed to download parquet file from {parquet_source}")
+                return collection
+            try:
+                table = pq.read_table(io.BytesIO(parquet_file.content))
+            except Exception as e:
+                LOGGER.error(f"Failed to read parquet file: {e}")
+                return collection
+            extents = extract_extent_from_geoparquet(table)
+            collection.extent.temporal = extents[0]
+            collection.extent.spatial = extents[1]
+            collection.add_asset(
+                "geoparquet",
+                Asset(
+                    href=parquet_source,
+                    media_type="application/vnd.apache.parquet",
+                    title="GeoParquet Items",
+                    roles=["collection-mirror"],
+                ),
+            )
+
     else:
         LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
 

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/src/eodash_catalog/utils.py

@@ -18,7 +18,7 @@ from dateutil import parser
 from owslib.wcs import WebCoverageService
 from owslib.wms import WebMapService
 from owslib.wmts import WebMapTileService
-from pystac import Asset, Catalog, Collection, Item, Link, RelType, SpatialExtent, TemporalExtent
+from pystac import Asset, Catalog, Collection, Item, RelType, SpatialExtent, TemporalExtent
 from pytz import timezone as pytztimezone
 from shapely import geometry as sgeom
 from shapely import wkb
@@ -448,6 +448,29 @@ def update_extents_from_collection_children(collection: Collection):
     collection.extent.temporal = TemporalExtent([time_extent])
 
 
+def extract_extent_from_geoparquet(table) -> tuple[TemporalExtent, SpatialExtent]:
+    """
+    Extract spatial and temporal extents from a GeoParquet file.
+    Args:
+        table (pyarrow.Table): The table containing the GeoParquet data.
+    Returns:
+        tuple: A tuple containing spatial and temporal extents.
+    """
+    # add extent information to the collection
+    min_datetime = pc.min(table["datetime"]).as_py()
+    max_datetime = pc.max(table["datetime"]).as_py()
+    if not min_datetime:
+        # cases when datetime was null
+        # fallback to start_datetime
+        min_datetime = pc.min(table["start_datetime"]).as_py()
+        max_datetime = pc.max(table["start_datetime"]).as_py()
+    temporal = TemporalExtent([min_datetime, max_datetime])
+    geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
+    bbox = sgeom.MultiPolygon(geoms).bounds
+    spatial = SpatialExtent([bbox])
+    return [temporal, spatial]
+
+
 def save_items(
     collection: Collection,
     items: list[Item],
@@ -497,25 +520,9 @@ def save_items(
         output_path = f"{buildcatpath}/{colpath}"
         os.makedirs(output_path, exist_ok=True)
         stacgp.arrow.to_parquet(table, f"{output_path}/items.parquet")
-        gp_link = Link(
-            rel="items",
-            target="./items.parquet",
-            media_type="application/vnd.apache.parquet",
-            title="GeoParquet Items",
-        )
-        collection.add_link(gp_link)
-        # add extent information to the collection
-        min_datetime = pc.min(table["datetime"]).as_py()
-        max_datetime = pc.max(table["datetime"]).as_py()
-        if not min_datetime:
-            # cases when datetime was null
-            # fallback to start_datetime
-            min_datetime = pc.min(table["start_datetime"]).as_py()
-            max_datetime = pc.max(table["start_datetime"]).as_py()
-        collection.extent.temporal = TemporalExtent([min_datetime, max_datetime])
-        geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
-        bbox = sgeom.MultiPolygon(geoms).bounds
-        collection.extent.spatial = SpatialExtent([bbox])
+        extents = extract_extent_from_geoparquet(table)
+        collection.extent.temporal = extents[0]
+        collection.extent.spatial = extents[1]
         # Make sure to also reference the geoparquet as asset
         collection.add_asset(
             "geoparquet",

{eodash_catalog-0.3.1 → eodash_catalog-0.3.2}/tests/test_geoparquet.py

@@ -45,11 +45,11 @@ def test_geoparquet_geojson_items(catalog_output_folder):
 
     with open(os.path.join(child_collection_path, "collection.json")) as fp:
         collection_json = json.load(fp)
-        items_links = [link for link in collection_json["links"] if link["rel"] == "items"]
-        assert len(items_links) == 1
-        gp_link = items_links[0]
-        assert gp_link["type"] == "application/vnd.apache.parquet"
-        items_path = os.path.join(child_collection_path, gp_link["href"].split("/")[-1])
+        # check if parquet source is present in assets
+        assert "geoparquet" in collection_json["assets"]
+        parquet_asset = collection_json["assets"]["geoparquet"]
+        assert parquet_asset["type"] == "application/vnd.apache.parquet"
+        items_path = os.path.join(child_collection_path, parquet_asset["href"].split("/")[-1])
         assert os.path.exists(items_path)
 
     with open(items_path, "rb") as fp: