eodash_catalog 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.3.1"
+__version__ = "0.3.3"

eodash_catalog/endpoints.py

@@ -1,4 +1,5 @@
 import importlib
+import io
 import json
 import os
 import sys
@@ -8,6 +9,7 @@ from datetime import datetime, timedelta
 from itertools import groupby
 from operator import itemgetter
 
+import pyarrow.parquet as pq
 import requests
 from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries, TemporalExtent
 from pystac_client import Client
@@ -28,6 +30,7 @@ from eodash_catalog.thumbnails import generate_thumbnail
 from eodash_catalog.utils import (
     Options,
     create_geometry_from_bbox,
+    extract_extent_from_geoparquet,
     filter_time_entries,
     format_datetime_to_isostring_zulu,
     generate_veda_cog_link,
@@ -606,19 +609,31 @@ def handle_GeoDB_endpoint(
             items = []
             for v in values:
                 # add items based on inputData fields for each time step available in values
-                first_match = next(
+                first_match: dict = next(
                     (item for item in input_data if item.get("Identifier") == v["input_data"]), None
                 )
                 time_object = datetime.fromisoformat(v["time"])
+                if endpoint_config.get("MapReplaceDates"):
+                    # get mapping of AOI_ID to list of dates
+                    available_dates_for_aoi_id = endpoint_config.get("MapReplaceDates").get(
+                        v["aoi_id"]
+                    )
+                    if available_dates_for_aoi_id:
+                        formatted_datetime = time_object.strftime("%Y-%m-%d")
+                        if formatted_datetime not in available_dates_for_aoi_id:
+                            # discard this date because not in available map dates
+                            continue
                 # extract wkt geometry from sub_aoi
                 if "sub_aoi" in v and v["sub_aoi"] != "/":
                     # create geometry from wkt
-                    geometry = mapping(wkt.loads(v["sub_aoi"]))
+                    shapely_geometry = wkt.loads(v["sub_aoi"])
+                    geometry = mapping(shapely_geometry)
                     # converting multipolygon to polygon to avoid shapely throwing an exception
                     # in collection extent from geoparquet table generation
                     # while trying to create a multipolygon extent of all multipolygons
                     if geometry["type"] == "MultiPolygon":
                         geometry = {"type": "Polygon", "coordinates": geometry["coordinates"][0]}
+                    bbox = shapely_geometry.bounds
                 else:
                     geometry = create_geometry_from_bbox(bbox)
                 item = Item(
@@ -637,7 +652,7 @@ def handle_GeoDB_endpoint(
                                 "wms:layers": [first_match["Layers"]],
                                 "role": ["data"],
                             }
-                            if url.startswith("https://services.sentinel-hub.com/ogc/wms/"):
+                            if "sentinel-hub.com" in url:
                                 instanceId = os.getenv("SH_INSTANCE_ID")
                                 if "InstanceId" in endpoint_config:
                                     instanceId = endpoint_config["InstanceId"]
@@ -652,7 +667,7 @@ def handle_GeoDB_endpoint(
                                     {"wms:dimensions": {"TIME": f"{start_date}/{end_date}"}}
                                 )
                                 # we add the instance id to the url
-                                url = f"https://services.sentinel-hub.com/ogc/wms/{instanceId}"
+                                url = f"{url}{instanceId}"
                             else:
                                 extra_fields.update({"wms:dimensions": {"TIME": v["time"]}})
                             link = Link(
@@ -664,6 +679,34 @@ def handle_GeoDB_endpoint(
                             )
                             item.add_link(link)
                             items.append(item)
+                        case "XYZ":
+                            # handler for NASA apis
+                            url = first_match["Url"]
+                            extra_fields = {}
+                            # replace time to a formatted version
+                            date_formatted = time_object.strftime(
+                                first_match.get("DateFormat", "%Y_%m_%d")
+                            )
+                            target_url = url.replace("{time}", date_formatted)
+                            if SiteMapping := first_match.get("SiteMapping"):
+                                # match with aoi_id
+                                site = SiteMapping.get(v["aoi_id"])
+                                # replace in URL
+                                if site:
+                                    target_url = target_url.replace("{site}", site)
+                                else:
+                                    LOGGER.info(
+                                        f"Warning: no match for SiteMapping in config for {site}"
+                                    )
+                            link = Link(
+                                rel="xyz",
+                                target=target_url,
+                                media_type="image/png",
+                                title=collection_config["Name"],
+                                extra_fields=extra_fields,
+                            )
+                            item.add_link(link)
+                            items.append(item)
             save_items(
                 locations_collection,
                 items,
@@ -687,6 +730,7 @@ def handle_GeoDB_endpoint(
         link.extra_fields["latlng"] = latlon
         link.extra_fields["country"] = country
         link.extra_fields["name"] = city
+        add_collection_information(catalog_config, locations_collection, collection_config)
 
     if "yAxis" not in collection_config:
         # fetch yAxis and store it to data, preventing need to save it per dataset in yml
@@ -778,19 +822,24 @@ def handle_WMS_endpoint(
     # Create an item per time to allow visualization in stac clients
     if len(datetimes) > 0:
         for dt in datetimes:
+            # case of wms interval coming from config
+            dt_item = dt[0] if isinstance(dt, list) else dt
             item = Item(
-                id=format_datetime_to_isostring_zulu(dt),
+                id=format_datetime_to_isostring_zulu(dt_item),
                 bbox=spatial_extent,
                 properties={},
                 geometry=create_geometry_from_bbox(spatial_extent),
-                datetime=dt,
+                datetime=dt_item,
                 stac_extensions=[
                     "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
                 ],
                 assets={"dummy_asset": Asset(href="")},
             )
             add_projection_info(endpoint_config, item)
-            add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
+            dt_visualization = dt if isinstance(dt, list) else [dt]
+            add_visualization_info(
+                item, collection_config, endpoint_config, datetimes=dt_visualization
+            )
             items.append(item)
     else:
         LOGGER.warn(f"NO datetimes returned for collection: {collection_config['Name']}!")
@@ -875,7 +924,6 @@ def add_visualization_info(
             start_isostring = format_datetime_to_isostring_zulu(dt)
             # SH WMS for public collections needs time interval, we use full day here
             end = dt + timedelta(days=1) - timedelta(milliseconds=1)
-            # we have start_datetime and end_datetime
             if len(datetimes) == 2:
                 end = datetimes[1]
             end_isostring = format_datetime_to_isostring_zulu(end)
@@ -913,7 +961,13 @@ def add_visualization_info(
                     )
                 dimensions[key] = value
         if datetimes is not None:
-            dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
+            if len(datetimes) > 1:
+                start = format_datetime_to_isostring_zulu(datetimes[0])
+                end = format_datetime_to_isostring_zulu(datetimes[1])
+                interval = f"{start}/{end}"
+                dimensions["TIME"] = interval
+            else:
+                dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
         if dimensions != {}:
             extra_fields["wms:dimensions"] = dimensions
         if endpoint_config.get("Styles"):
@@ -1218,6 +1272,33 @@ def handle_raw_source(
         # eodash v4 compatibility, adding last referenced style to collection
         if style_link:
             collection.add_link(style_link)
+    elif endpoint_config.get("ParquetSource"):
+        # if parquet source is provided, download it and create items from it
+        parquet_source = endpoint_config["ParquetSource"]
+        if parquet_source.startswith("http"):
+            # download parquet file
+            parquet_file = requests.get(parquet_source)
+            if parquet_file.status_code != 200:
+                LOGGER.error(f"Failed to download parquet file from {parquet_source}")
+                return collection
+            try:
+                table = pq.read_table(io.BytesIO(parquet_file.content))
+            except Exception as e:
+                LOGGER.error(f"Failed to read parquet file: {e}")
+                return collection
+            extents = extract_extent_from_geoparquet(table)
+            collection.extent.temporal = extents[0]
+            collection.extent.spatial = extents[1]
+            collection.add_asset(
+                "geoparquet",
+                Asset(
+                    href=parquet_source,
+                    media_type="application/vnd.apache.parquet",
+                    title="GeoParquet Items",
+                    roles=["collection-mirror"],
+                ),
+            )
+
     else:
         LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
 

eodash_catalog/stac_handling.py

@@ -18,6 +18,7 @@ from structlog import get_logger
 from eodash_catalog.utils import (
     generateDatetimesFromInterval,
     get_full_url,
+    make_intervals,
     parse_datestring_to_tz_aware_datetime,
     read_config_file,
 )
@@ -535,16 +536,22 @@ def add_extra_fields(
 def get_collection_datetimes_from_config(endpoint_config: dict) -> list[datetime]:
     times_datetimes: list[datetime] = []
     if endpoint_config:
+        interval_between_dates = endpoint_config.get("WMSIntervalsBetweenDates")
         if endpoint_config.get("Times"):
             times = list(endpoint_config.get("Times", []))
             times_datetimes = sorted(
                 [parse_datestring_to_tz_aware_datetime(time) for time in times]
             )
+            if interval_between_dates:
+                # convert to list of datetime_start and datetime_end
+                times_datetimes = make_intervals(times_datetimes)
         elif endpoint_config.get("DateTimeInterval"):
             start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00Z")
             end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00Z")
             timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
-            times_datetimes = generateDatetimesFromInterval(start, end, timedelta_config)
+            times_datetimes = generateDatetimesFromInterval(
+                start, end, timedelta_config, interval_between_dates
+            )
     return times_datetimes
 
 

eodash_catalog/utils.py

@@ -18,7 +18,7 @@ from dateutil import parser
 from owslib.wcs import WebCoverageService
 from owslib.wms import WebMapService
 from owslib.wmts import WebMapTileService
-from pystac import Asset, Catalog, Collection, Item, Link, RelType, SpatialExtent, TemporalExtent
+from pystac import Asset, Catalog, Collection, Item, RelType, SpatialExtent, TemporalExtent
 from pytz import timezone as pytztimezone
 from shapely import geometry as sgeom
 from shapely import wkb
@@ -214,7 +214,7 @@ def parse_duration(datestring):
 
 
 def generateDatetimesFromInterval(
-    start: str, end: str, timedelta_config: dict | None = None
+    start: str, end: str, timedelta_config: dict | None = None, interval_between_dates: bool = False
 ) -> list[datetime]:
     if timedelta_config is None:
         timedelta_config = {}
@@ -226,7 +226,10 @@ def generateDatetimesFromInterval(
     delta = timedelta(**timedelta_config)
     dates = []
     while start_dt <= end_dt:
-        dates.append(start_dt)
+        if interval_between_dates:
+            dates.append([start_dt, start_dt + delta - timedelta(seconds=1)])
+        else:
+            dates.append(start_dt)
         start_dt += delta
     return dates
 
@@ -444,8 +447,32 @@ def update_extents_from_collection_children(collection: Collection):
         ):
             individual_datetimes.extend(c_child.extent.temporal.intervals[0])  # type: ignore
     individual_datetimes = list(filter(lambda x: x is not None, individual_datetimes))
-    time_extent = [min(individual_datetimes), max(individual_datetimes)]
-    collection.extent.temporal = TemporalExtent([time_extent])
+    if individual_datetimes:
+        time_extent = [min(individual_datetimes), max(individual_datetimes)]
+        collection.extent.temporal = TemporalExtent([time_extent])
+
+
+def extract_extent_from_geoparquet(table) -> tuple[TemporalExtent, SpatialExtent]:
+    """
+    Extract spatial and temporal extents from a GeoParquet file.
+    Args:
+        table (pyarrow.Table): The table containing the GeoParquet data.
+    Returns:
+        tuple: A tuple containing spatial and temporal extents.
+    """
+    # add extent information to the collection
+    min_datetime = pc.min(table["datetime"]).as_py()
+    max_datetime = pc.max(table["datetime"]).as_py()
+    if not min_datetime:
+        # cases when datetime was null
+        # fallback to start_datetime
+        min_datetime = pc.min(table["start_datetime"]).as_py()
+        max_datetime = pc.max(table["start_datetime"]).as_py()
+    temporal = TemporalExtent([min_datetime, max_datetime])
+    geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
+    bbox = sgeom.MultiPolygon(geoms).bounds
+    spatial = SpatialExtent([bbox])
+    return [temporal, spatial]
 
 
 def save_items(
@@ -497,25 +524,9 @@ def save_items(
         output_path = f"{buildcatpath}/{colpath}"
         os.makedirs(output_path, exist_ok=True)
         stacgp.arrow.to_parquet(table, f"{output_path}/items.parquet")
-        gp_link = Link(
-            rel="items",
-            target="./items.parquet",
-            media_type="application/vnd.apache.parquet",
-            title="GeoParquet Items",
-        )
-        collection.add_link(gp_link)
-        # add extent information to the collection
-        min_datetime = pc.min(table["datetime"]).as_py()
-        max_datetime = pc.max(table["datetime"]).as_py()
-        if not min_datetime:
-            # cases when datetime was null
-            # fallback to start_datetime
-            min_datetime = pc.min(table["start_datetime"]).as_py()
-            max_datetime = pc.max(table["start_datetime"]).as_py()
-        collection.extent.temporal = TemporalExtent([min_datetime, max_datetime])
-        geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
-        bbox = sgeom.MultiPolygon(geoms).bounds
-        collection.extent.spatial = SpatialExtent([bbox])
+        extents = extract_extent_from_geoparquet(table)
+        collection.extent.temporal = extents[0]
+        collection.extent.spatial = extents[1]
         # Make sure to also reference the geoparquet as asset
         collection.add_asset(
             "geoparquet",
@@ -608,3 +619,26 @@ def merge_bboxes(bboxes: list[list[float]]) -> list[float]:
     max_lat = max(b[3] for b in bboxes)
 
     return [min_lon, min_lat, max_lon, max_lat]
+
+
+def make_intervals(datetimes: list[datetime]) -> list[list[datetime]]:
+    """
+    Converts a list of datetimes into list of lists of datetimes in format of [start,end]
+    where end is next element in original list minus 1 second
+    """
+    intervals = []
+    n = len(datetimes)
+    for i in range(n):
+        start = datetimes[i]
+        if i < n - 1:
+            # end is next datetime minus one second
+            end = datetimes[i + 1] - timedelta(seconds=1)
+        else:
+            prev_interval = timedelta(seconds=0)
+            # last item: use previous interval length added to last start
+            if n > 1:
+                prev_interval = datetimes[-1] - datetimes[-2]
+            end = start + prev_interval
+        intervals.append([start, end])
+    LOGGER.info(intervals)
+    return intervals

{eodash_catalog-0.3.1.dist-info → eodash_catalog-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eodash_catalog
-Version: 0.3.1
+Version: 0.3.3
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues

eodash_catalog-0.3.3.dist-info/RECORD

@@ -0,0 +1,14 @@
+eodash_catalog/__about__.py,sha256=jakR3FNEOvU2GrvtecX6moFoNPby5Ok50p6_6JcZoEQ,137
+eodash_catalog/__init__.py,sha256=_W_9emPYf6FUqc0P8L2SmADx6hGSd7PlQV3yRmCk5uM,115
+eodash_catalog/duration.py,sha256=TBG7v1lCpbYowADK5uJ2M8kPxsvQneFAFi1NIE26dy4,10754
+eodash_catalog/endpoints.py,sha256=uI7aapRHQ1eDYQYKlrMdLHi_6aitBS3jutrou2HDWJY,53183
+eodash_catalog/generate_indicators.py,sha256=FPeiZm9TE4PpbTyH6UMegQ7HwaARzO91IrLtzFjFSF0,21900
+eodash_catalog/sh_endpoint.py,sha256=XjZsZJ5jfJZLQenSTqUhiUZ5YAu9M9nv2KL1Qv3Be-I,1219
+eodash_catalog/stac_handling.py,sha256=xMhUK_gvijs6tL-1ecg28DtWBo4msd5NTYZpYtt7FHo,25877
+eodash_catalog/thumbnails.py,sha256=oNbWdRC8KTLUC4PbSMlSaiOeLXfkIpa0j-sOZdn1RGU,2262
+eodash_catalog/utils.py,sha256=C-HQK6IYMNAd3Vfgq9KOqsBlJu-jtnZTcsHGp4kj7Y0,23773
+eodash_catalog-0.3.3.dist-info/METADATA,sha256=H1Npnw-SZ3hAerwBs6ml0fscgSP9MkMDbVTryhPq7kM,3019
+eodash_catalog-0.3.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+eodash_catalog-0.3.3.dist-info/entry_points.txt,sha256=kuUQrDG1PtYd8kPjf5XM6H_NtQd9Ozwl0jjiGtAvZSM,87
+eodash_catalog-0.3.3.dist-info/licenses/LICENSE.txt,sha256=oJCW5zQxnFD-J0hGz6Zh5Lkpdk1oAndmWhseTmV224E,1107
+eodash_catalog-0.3.3.dist-info/RECORD,,

eodash_catalog-0.3.1.dist-info/RECORD

@@ -1,14 +0,0 @@
-eodash_catalog/__about__.py,sha256=l-uZOyslIByuHXe6EqfhKvRxyfPaTvVIobrJDf86Ai8,137
-eodash_catalog/__init__.py,sha256=_W_9emPYf6FUqc0P8L2SmADx6hGSd7PlQV3yRmCk5uM,115
-eodash_catalog/duration.py,sha256=TBG7v1lCpbYowADK5uJ2M8kPxsvQneFAFi1NIE26dy4,10754
-eodash_catalog/endpoints.py,sha256=Ckvg0MnaXQjTy7hd55kqnGlPmlz5UYGJ2c3HXdvLVNs,49168
-eodash_catalog/generate_indicators.py,sha256=FPeiZm9TE4PpbTyH6UMegQ7HwaARzO91IrLtzFjFSF0,21900
-eodash_catalog/sh_endpoint.py,sha256=XjZsZJ5jfJZLQenSTqUhiUZ5YAu9M9nv2KL1Qv3Be-I,1219
-eodash_catalog/stac_handling.py,sha256=waw8qRjwjdbDBRtialc4bG3WSjXAATklc-W5kLKywqE,25548
-eodash_catalog/thumbnails.py,sha256=oNbWdRC8KTLUC4PbSMlSaiOeLXfkIpa0j-sOZdn1RGU,2262
-eodash_catalog/utils.py,sha256=wEh0hvBVp20NKhdJMOQ4dmiiOrFUOTDP2sRtfe0FBpQ,22563
-eodash_catalog-0.3.1.dist-info/METADATA,sha256=OgbqxdohyQr9kO3WwjwJ3FRs5-5khvd6T7mmivVzpe8,3019
-eodash_catalog-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-eodash_catalog-0.3.1.dist-info/entry_points.txt,sha256=kuUQrDG1PtYd8kPjf5XM6H_NtQd9Ozwl0jjiGtAvZSM,87
-eodash_catalog-0.3.1.dist-info/licenses/LICENSE.txt,sha256=oJCW5zQxnFD-J0hGz6Zh5Lkpdk1oAndmWhseTmV224E,1107
-eodash_catalog-0.3.1.dist-info/RECORD,,