eodash_catalog 0.3.0__tar.gz → 0.3.17__tar.gz

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.3.17
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/Dockerfile

@@ -29,4 +29,4 @@ RUN eodash_catalog --help
 
 CMD ["eodash_catalog"]
 
-LABEL version="0.3.0"
+LABEL version="0.3.17"

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eodash_catalog
-Version: 0.3.0
+Version: 0.3.17
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.3.0"
+__version__ = "0.3.17"

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/endpoints.py

@@ -1,4 +1,6 @@
+import copy
 import importlib
+import io
 import json
 import os
 import sys
@@ -7,7 +9,9 @@ from collections.abc import Callable
 from datetime import datetime, timedelta
 from itertools import groupby
 from operator import itemgetter
+from urllib.parse import urlparse
 
+import pyarrow.parquet as pq
 import requests
 from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries, TemporalExtent
 from pystac_client import Client
@@ -17,6 +21,8 @@ from structlog import get_logger
 
 from eodash_catalog.sh_endpoint import get_SH_token
 from eodash_catalog.stac_handling import (
+    add_authentication,
+    add_base_overlay_info,
     add_collection_information,
     add_example_info,
     add_process_info_child_collection,
@@ -28,6 +34,7 @@ from eodash_catalog.thumbnails import generate_thumbnail
 from eodash_catalog.utils import (
     Options,
     create_geometry_from_bbox,
+    extract_extent_from_geoparquet,
     filter_time_entries,
     format_datetime_to_isostring_zulu,
     generate_veda_cog_link,
@@ -169,7 +176,7 @@ def handle_STAC_based_endpoint(
             catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
         )
         for location in collection_config["Locations"]:
-            identifier = location.get("Identifier", uuid.uuid4())
+            identifier = location.get("Identifier", str(uuid.uuid4()))
             collection = process_STACAPI_Endpoint(
                 catalog_config=catalog_config,
                 endpoint_config=endpoint_config,
@@ -209,11 +216,13 @@ def handle_STAC_based_endpoint(
                         location["OverwriteBBox"],
                     ]
                 )
+            add_collection_information(catalog_config, collection, collection_config)
+            add_base_overlay_info(collection, catalog_config, collection_config)
         update_extents_from_collection_children(root_collection)
     else:
         bbox = None
-        if endpoint_config.get("Bbox"):
-            bbox = ",".join(map(str, endpoint_config["Bbox"]))
+        if endpoint_config.get("OverwriteBBox"):
+            bbox = ",".join(map(str, endpoint_config["OverwriteBBox"]))
         root_collection = process_STACAPI_Endpoint(
             catalog_config=catalog_config,
             endpoint_config=endpoint_config,
@@ -388,6 +397,7 @@ def handle_collection_only(
                 properties={},
                 geometry=None,
                 datetime=dt,
+                assets={"dummy_asset": Asset(href="")},
             )
             link = collection.add_item(item)
             link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
@@ -460,12 +470,14 @@ def handle_SH_WMS_endpoint(
                 LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
             add_visualization_info(collection, collection_config, endpoint_config)
             add_process_info_child_collection(collection, catalog_config, collection_config)
+            add_collection_information(catalog_config, collection, collection_config)
+            add_base_overlay_info(collection, catalog_config, collection_config)
         update_extents_from_collection_children(root_collection)
     else:
         # if locations are not provided, treat the collection as a
         # general proxy to the sentinel hub layer
         datetimes = get_collection_datetimes_from_config(endpoint_config)
-        bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
+        bbox = endpoint_config.get("OverwriteBBox", [-180, -85, 180, 85])
         items = []
         for dt in datetimes:
             item = Item(
@@ -536,6 +548,117 @@ def handle_rasdaman_endpoint(
     return collection
 
 
+def handle_GeoDB_Features_endpoint(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    coll_path_rel_to_root_catalog: str,
+    catalog: Catalog,
+    options: Options,
+) -> Collection:
+    # ID of collection is data["Name"] instead of CollectionId to be able to
+    # create more STAC collections from one geoDB table
+    collection = get_or_create_collection(
+        catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
+    )
+    coll_path_rel_to_root_catalog = f'{coll_path_rel_to_root_catalog}/{collection_config["Name"]}'
+    select = f'?select={endpoint_config["TimeParameter"]}'
+    url = (
+        endpoint_config["EndPoint"]
+        + endpoint_config["Database"]
+        + "_{}".format(endpoint_config["CollectionId"])
+        + select
+    )
+    response = json.loads(requests.get(url).text)
+    # Use aggregation value to group datetime results
+    aggregation = endpoint_config.get("Aggregation", "day")
+    unique_datetimes = set()
+    for value in response:
+        time_object = datetime.fromisoformat(value[endpoint_config["TimeParameter"]])
+        match aggregation:
+            case "hour":
+                unique_datetimes.add(
+                    datetime(
+                        time_object.year,
+                        time_object.month,
+                        time_object.day,
+                        time_object.hour,
+                    )
+                )
+            case "day":
+                unique_datetimes.add(
+                    datetime(time_object.year, time_object.month, time_object.day).date()
+                )
+            case "month":
+                unique_datetimes.add(datetime(time_object.year, time_object.month, 1).date())
+            case "year":
+                unique_datetimes.add(datetime(time_object.year, 1, 1).date())
+            case _:
+                # default to day
+                unique_datetimes.add(
+                    datetime(time_object.year, time_object.month, time_object.day).date()
+                )
+    # go over unique datetimes and create items
+    items = []
+    for dt in sorted(unique_datetimes):
+        item_datetime = dt if isinstance(dt, datetime) else datetime(dt.year, dt.month, dt.day)
+        matching_string = ""
+        match aggregation:
+            case "hour":
+                matching_string = item_datetime.strftime("%Y-%m-%dT%H:00:00Z")
+            case "day":
+                matching_string = item_datetime.strftime("%Y-%m-%d")
+            case "month":
+                matching_string = item_datetime.strftime("%Y-%m")
+            case "year":
+                matching_string = item_datetime.strftime("%Y")
+        updated_query = endpoint_config["Query"].replace("{{date_time}}", matching_string)
+        assets = {
+            "geodbfeatures": Asset(
+                href=f"{endpoint_config['EndPoint']}{endpoint_config['Database']}_{endpoint_config['CollectionId']}?{updated_query}",
+                media_type="application/geodb+json",
+                roles=["data"],
+            )
+        }
+        item = Item(
+            id=format_datetime_to_isostring_zulu(item_datetime),
+            bbox=endpoint_config.get("OverwriteBBox", [-180, -90, 180, 90]),
+            properties={},
+            geometry=create_geometry_from_bbox(
+                endpoint_config.get("OverwriteBBox", [-180, -90, 180, 90])
+            ),
+            datetime=item_datetime,
+            stac_extensions=[],
+            assets=assets,
+        )
+        # add eodash style visualization info if Style has been provided
+        if endpoint_config.get("Style"):
+            ep_st = endpoint_config.get("Style")
+            style_link = Link(
+                rel="style",
+                target=ep_st
+                if ep_st.startswith("http")
+                else f"{catalog_config['assets_endpoint']}/{ep_st}",
+                media_type="text/vector-styles",
+                extra_fields={
+                    "asset:keys": list(assets),
+                },
+            )
+            item.add_link(style_link)
+        add_projection_info(endpoint_config, item)
+        items.append(item)
+    save_items(
+        collection,
+        items,
+        options.outputpath,
+        catalog_config["id"],
+        coll_path_rel_to_root_catalog,
+        options.gp,
+    )
+    add_collection_information(catalog_config, collection, collection_config)
+    return collection
+
+
 def handle_GeoDB_endpoint(
     catalog_config: dict,
     endpoint_config: dict,
@@ -602,33 +725,96 @@ def handle_GeoDB_endpoint(
         locations_collection = get_or_create_collection(
             collection, key, sc_config, catalog_config, endpoint_config
         )
-        if input_data:
+        # check if input data is none
+        if input_data is None:
+            input_data = []
+        if len(input_data) > 0 or endpoint_config.get("FeatureCollection"):
             items = []
-            for v in values:
+            content_for_individual_datetimes = values
+            if endpoint_config.get("MapTimesCollection"):
+                # extract datetimes from another table if configured so and match it based on aoi_id
+                # special for E13d
+                select = f"?select=time&aoi_id=eq.{key}"
+                url = (
+                    endpoint_config["EndPoint"]
+                    + endpoint_config["Database"]
+                    + "_{}".format(endpoint_config["MapTimesCollection"])
+                    + select
+                )
+                response = json.loads(requests.get(url).text)
+                content_for_individual_datetimes = []
+                for response_obj in response:
+                    time_object = datetime.fromisoformat(response_obj["time"])
+                    for searched_row in values:
+                        search_datetime = datetime.fromisoformat(searched_row["time"])
+                        if (
+                            search_datetime.month == time_object.month
+                            and search_datetime.year == time_object.year
+                        ):
+                            break
+                    insert_row = copy.deepcopy(searched_row)
+                    # overwrite time with one from another collection and save
+                    insert_row["time"] = response_obj["time"]
+                    content_for_individual_datetimes.append(insert_row)
+            for v in content_for_individual_datetimes:
                 # add items based on inputData fields for each time step available in values
-                first_match = next(
+                first_match: dict = next(
                     (item for item in input_data if item.get("Identifier") == v["input_data"]), None
                 )
                 time_object = datetime.fromisoformat(v["time"])
+                if endpoint_config.get("MapReplaceDates"):
+                    # get mapping of AOI_ID to list of dates
+                    available_dates_for_aoi_id = endpoint_config.get("MapReplaceDates").get(
+                        v["aoi_id"]
+                    )
+                    if available_dates_for_aoi_id:
+                        formatted_datetime = time_object.strftime("%Y-%m-%d")
+                        if formatted_datetime not in available_dates_for_aoi_id:
+                            # discard this date because not in available map dates
+                            continue
                 # extract wkt geometry from sub_aoi
                 if "sub_aoi" in v and v["sub_aoi"] != "/":
                     # create geometry from wkt
-                    geometry = mapping(wkt.loads(v["sub_aoi"]))
+                    shapely_geometry = wkt.loads(v["sub_aoi"])
+                    geometry = mapping(shapely_geometry)
                     # converting multipolygon to polygon to avoid shapely throwing an exception
                     # in collection extent from geoparquet table generation
                     # while trying to create a multipolygon extent of all multipolygons
                     if geometry["type"] == "MultiPolygon":
                         geometry = {"type": "Polygon", "coordinates": geometry["coordinates"][0]}
+                    bbox = shapely_geometry.bounds
                 else:
                     geometry = create_geometry_from_bbox(bbox)
+
+                assets = {"dummy_asset": Asset(href="")}
+                if endpoint_config.get("FeatureCollection"):
+                    assets["geodbfeatures"] = Asset(
+                        href=f"{endpoint_config['EndPoint']}{endpoint_config['Database']}_{endpoint_config['FeatureCollection']}?aoi_id=eq.{v['aoi_id']}&time=eq.{v['time']}",
+                        media_type="application/geodb+json",
+                        roles=["data"],
+                    )
                 item = Item(
                     id=v["time"],
                     bbox=bbox,
                     properties={},
                     geometry=geometry,
                     datetime=time_object,
-                    assets={"dummy_asset": Asset(href="")},
+                    assets=assets,
                 )
+                # make sure to also add Style link if FeatureCollection and Style has been provided
+                if endpoint_config.get("FeatureCollection") and endpoint_config.get("Style"):
+                    ep_st = endpoint_config.get("Style")
+                    style_link = Link(
+                        rel="style",
+                        target=ep_st
+                        if ep_st.startswith("http")
+                        else f"{catalog_config['assets_endpoint']}/{ep_st}",
+                        media_type="text/vector-styles",
+                        extra_fields={
+                            "asset:keys": list(assets),
+                        },
+                    )
+                    item.add_link(style_link)
                 if first_match:
                     match first_match["Type"]:
                         case "WMS":
@@ -637,7 +823,7 @@ def handle_GeoDB_endpoint(
                                 "wms:layers": [first_match["Layers"]],
                                 "role": ["data"],
                             }
-                            if url.startswith("https://services.sentinel-hub.com/ogc/wms/"):
+                            if "sentinel-hub.com" in url:
                                 instanceId = os.getenv("SH_INSTANCE_ID")
                                 if "InstanceId" in endpoint_config:
                                     instanceId = endpoint_config["InstanceId"]
@@ -652,7 +838,7 @@ def handle_GeoDB_endpoint(
                                     {"wms:dimensions": {"TIME": f"{start_date}/{end_date}"}}
                                 )
                                 # we add the instance id to the url
-                                url = f"https://services.sentinel-hub.com/ogc/wms/{instanceId}"
+                                url = f"{url}{instanceId}"
                             else:
                                 extra_fields.update({"wms:dimensions": {"TIME": v["time"]}})
                             link = Link(
@@ -664,6 +850,50 @@ def handle_GeoDB_endpoint(
                             )
                             item.add_link(link)
                             items.append(item)
+                        case "XYZ":
+                            # handler for NASA apis
+                            url = first_match["Url"]
+                            extra_fields = {}
+                            # replace time to a formatted version
+                            date_formatted = time_object.strftime(
+                                first_match.get("DateFormat", "%Y_%m_%d")
+                            )
+                            target_url = url.replace("{time}", date_formatted)
+                            if SiteMapping := first_match.get("SiteMapping"):
+                                # match with aoi_id
+                                site = SiteMapping.get(v["aoi_id"])
+                                # replace in URL
+                                if site:
+                                    target_url = target_url.replace("{site}", site)
+                                else:
+                                    LOGGER.info(
+                                        f"Warning: no match for SiteMapping in config for {site}"
+                                    )
+                            link = Link(
+                                rel="xyz",
+                                target=target_url,
+                                media_type="image/png",
+                                title=collection_config["Name"],
+                                extra_fields=extra_fields,
+                            )
+                            item.add_link(link)
+                            items.append(item)
+                elif endpoint_config.get("FeatureCollection"):
+                    # no input data match found, just add the item with asset only
+                    assets["geodbfeatures"] = Asset(
+                        href=f"{endpoint_config['EndPoint']}{endpoint_config['Database']}_{endpoint_config['FeatureCollection']}?aoi_id=eq.{v['aoi_id']}&time=eq.{v['time']}",
+                        media_type="application/geodb+json",
+                        roles=["data"],
+                    )
+                    item = Item(
+                        id=v["time"],
+                        bbox=bbox,
+                        properties={},
+                        geometry=geometry,
+                        datetime=time_object,
+                        assets=assets,
+                    )
+                    items.append(item)
             save_items(
                 locations_collection,
                 items,
@@ -687,6 +917,8 @@ def handle_GeoDB_endpoint(
         link.extra_fields["latlng"] = latlon
         link.extra_fields["country"] = country
         link.extra_fields["name"] = city
+        add_collection_information(catalog_config, locations_collection, collection_config)
+        add_base_overlay_info(locations_collection, catalog_config, collection_config)
 
     if "yAxis" not in collection_config:
         # fetch yAxis and store it to data, preventing need to save it per dataset in yml
@@ -725,7 +957,8 @@ def handle_SH_endpoint(
 ) -> Collection:
     token = get_SH_token(endpoint_config)
     headers = {"Authorization": f"Bearer {token}"}
-    endpoint_config["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
+    endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
+    endpoint_config["EndPoint"] = f"https://{endpoint_url_parts.netloc}/api/v1/catalog/1.0.0/"
     # Overwrite collection id with type, such as ZARR or BYOC
     if endpoint_config.get("Type"):
         endpoint_config["CollectionId"] = (
@@ -762,12 +995,14 @@ def handle_WMS_endpoint(
         # some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
         # have to process full service capabilities XML
         capabilities_url = endpoint_config["EndPoint"]
-        spatial_extent, datetimes = retrieveExtentFromWMSWMTS(
+        spatial_extent, datetimes_retrieved = retrieveExtentFromWMSWMTS(
             capabilities_url,
             endpoint_config["LayerId"],
             version=endpoint_config.get("Version", "1.1.1"),
             wmts=wmts,
         )
+        if datetimes_retrieved:
+            datetimes = datetimes_retrieved
     # optionally filter time results
     if query := endpoint_config.get("Query"):
         datetimes = filter_time_entries(datetimes, query)
@@ -778,19 +1013,24 @@ def handle_WMS_endpoint(
     # Create an item per time to allow visualization in stac clients
     if len(datetimes) > 0:
         for dt in datetimes:
+            # case of wms interval coming from config
+            dt_item = dt[0] if isinstance(dt, list) else dt
             item = Item(
-                id=format_datetime_to_isostring_zulu(dt),
+                id=format_datetime_to_isostring_zulu(dt_item),
                 bbox=spatial_extent,
                 properties={},
                 geometry=create_geometry_from_bbox(spatial_extent),
-                datetime=dt,
+                datetime=dt_item,
                 stac_extensions=[
                     "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
                 ],
                 assets={"dummy_asset": Asset(href="")},
             )
             add_projection_info(endpoint_config, item)
-            add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
+            dt_visualization = dt if isinstance(dt, list) else [dt]
+            add_visualization_info(
+                item, collection_config, endpoint_config, datetimes=dt_visualization
+            )
             items.append(item)
     else:
         LOGGER.warn(f"NO datetimes returned for collection: {collection_config['Name']}!")
@@ -826,11 +1066,19 @@ def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
     color_formula = ""
     if endpoint_config.get("ColorFormula"):
         color_formula = "&color_formula={}".format(endpoint_config["ColorFormula"])
+    rescale = ""
+    if endpoint_config.get("Rescale"):
+        for rescale in endpoint_config["Rescale"]:
+            rescale += f"&rescale={rescale}"
     no_data = ""
     if endpoint_config.get("NoData"):
         no_data = "&no_data={}".format(endpoint_config["NoData"])
     item = item if item else "{item}"
-    target_url = f"https://openveda.cloud/api/raster/collections/{collection}/items/{item}/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}"
+    target_url_base = endpoint_config["EndPoint"].replace("/stac/", "")
+    target_url = (
+        f"{target_url_base}/raster/collections/{collection}/items/{item}"
+        f"/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}{rescale}"
+    )
     return target_url
 
 
@@ -871,7 +1119,6 @@ def add_visualization_info(
             start_isostring = format_datetime_to_isostring_zulu(dt)
             # SH WMS for public collections needs time interval, we use full day here
             end = dt + timedelta(days=1) - timedelta(milliseconds=1)
-            # we have start_datetime and end_datetime
             if len(datetimes) == 2:
                 end = datetimes[1]
             end_isostring = format_datetime_to_isostring_zulu(end)
@@ -880,9 +1127,10 @@ def add_visualization_info(
 
         if dimensions != {}:
             extra_fields["wms:dimensions"] = dimensions
+        endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
         link = Link(
             rel="wms",
-            target=f"https://services.sentinel-hub.com/ogc/wms/{instanceId}",
+            target=f"https://{endpoint_url_parts.netloc}/ogc/wms/{instanceId}",
             media_type=(endpoint_config.get("MimeType", "image/png")),
             title=collection_config["Name"],
             extra_fields=extra_fields,
@@ -899,6 +1147,14 @@ def add_visualization_info(
                 "role": ["data"],
             }
         )
+        if collection_config.get("EodashIdentifier") == "FNF":
+            extra_fields.update(
+                {
+                    "wms:layers": endpoint_config.get("LayerId", "").replace(
+                        "{time}", (datetimes is not None and str(datetimes[0].year)) or "2020"
+                    ),
+                }
+            )
         dimensions = {}
         if dimensions_config := endpoint_config.get("Dimensions", {}):
             for key, value in dimensions_config.items():
@@ -909,7 +1165,13 @@ def add_visualization_info(
                     )
                 dimensions[key] = value
         if datetimes is not None:
-            dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
+            if len(datetimes) > 1:
+                start = format_datetime_to_isostring_zulu(datetimes[0])
+                end = format_datetime_to_isostring_zulu(datetimes[1])
+                interval = f"{start}/{end}"
+                dimensions["TIME"] = interval
+            else:
+                dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
         if dimensions != {}:
             extra_fields["wms:dimensions"] = dimensions
         if endpoint_config.get("Styles"):
@@ -966,28 +1228,6 @@ def add_visualization_info(
             link,
         )
         stac_object.add_link(link)
-    elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
-        target_url = "{}".format(endpoint_config.get("EndPoint"))
-        # custom time just for this special case as a default for collection wmts
-        time = None
-        if datetimes is not None:
-            time = datetimes[0]
-        extra_fields.update(
-            {
-                "wmts:layer": endpoint_config.get("LayerId", "").replace(
-                    "{time}", (time and str(time.year)) or "2017"
-                )
-            }
-        )
-        stac_object.add_link(
-            Link(
-                rel="wmts",
-                target=target_url,
-                media_type="image/png",
-                title="wmts capabilities",
-                extra_fields=extra_fields,
-            )
-        )
     elif endpoint_config["Name"] == "xcube":
         if endpoint_config["Type"] == "zarr":
             # either preset ColormapName of left as a template
@@ -1214,6 +1454,121 @@ def handle_raw_source(
         # eodash v4 compatibility, adding last referenced style to collection
         if style_link:
             collection.add_link(style_link)
+    elif endpoint_config.get("ParquetSource"):
+        # if parquet source is provided, download it and create items from it
+        parquet_source = endpoint_config["ParquetSource"]
+        if parquet_source.startswith("http"):
+            # download parquet file
+            parquet_file = requests.get(parquet_source)
+            if parquet_file.status_code != 200:
+                LOGGER.error(f"Failed to download parquet file from {parquet_source}")
+                return collection
+            try:
+                table = pq.read_table(io.BytesIO(parquet_file.content))
+            except Exception as e:
+                LOGGER.error(f"Failed to read parquet file: {e}")
+                return collection
+            extents = extract_extent_from_geoparquet(table)
+            collection.extent.temporal = extents[0]
+            collection.extent.spatial = extents[1]
+            collection.add_asset(
+                "geoparquet",
+                Asset(
+                    href=parquet_source,
+                    media_type="application/vnd.apache.parquet",
+                    title="GeoParquet Items",
+                    roles=["collection-mirror"],
+                ),
+            )
+
+    else:
+        LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
+
+    add_collection_information(catalog_config, collection, collection_config)
+    return collection
+
+
+def handle_vector_tile_source(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    coll_path_rel_to_root_catalog: str,
+    catalog: Catalog,
+    options: Options,
+) -> Collection:
+    collection = get_or_create_collection(
+        catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
+    )
+    coll_path_rel_to_root_catalog = f'{coll_path_rel_to_root_catalog}/{collection_config["Name"]}'
+    if len(endpoint_config.get("TimeEntries", [])) > 0:
+        items = []
+        style_link = None
+        for time_entry in endpoint_config["TimeEntries"]:
+            # create Item for each time entry
+            media_type = "application/vnd.mapbox-vector-tile"
+            style_type = "text/vector-styles"
+            bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
+            dt = parse_datestring_to_tz_aware_datetime(time_entry["Time"])
+
+            item = Item(
+                id=format_datetime_to_isostring_zulu(dt),
+                bbox=bbox,
+                properties={},
+                geometry=create_geometry_from_bbox(bbox),
+                datetime=dt,
+                extra_fields={},
+                assets={"dummy_asset": Asset(href="")},
+            )
+            extra_fields_link = {}
+            add_authentication(item, time_entry["Url"], extra_fields_link)
+            # add mapbox vector tile link
+            identifier = str(uuid.uuid4())
+            extra_fields_link["key"] = identifier
+            if vector_tile_id_property := endpoint_config.get("idProperty"):
+                extra_fields_link["idProperty"] = vector_tile_id_property
+            link = Link(
+                rel="vector-tile",
+                target=time_entry["Url"],
+                media_type=media_type,
+                title=collection_config["Name"],
+                extra_fields=extra_fields_link,
+            )
+            add_projection_info(
+                endpoint_config,
+                link,
+            )
+            item.add_link(link)
+            add_projection_info(
+                endpoint_config,
+                item,
+            )
+            if endpoint_config.get("Attribution"):
+                item.stac_extensions.append(
+                    "https://stac-extensions.github.io/attribution/v0.1.0/schema.json"
+                )
+                item.extra_fields["attribution"] = endpoint_config["Attribution"]
+            # add style
+            if ep_st := endpoint_config.get("Style"):
+                style_link = Link(
+                    rel="style",
+                    target=ep_st
+                    if ep_st.startswith("http")
+                    else f"{catalog_config['assets_endpoint']}/{ep_st}",
+                    media_type=style_type,
+                    extra_fields={"links:keys": [identifier]},
+                )
+                item.add_link(style_link)
+            items.append(item)
+
+        save_items(
+            collection,
+            items,
+            options.outputpath,
+            catalog_config["id"],
+            coll_path_rel_to_root_catalog,
+            options.gp,
+        )
+
     else:
         LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
 

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/generate_indicators.py

@@ -19,10 +19,12 @@ from eodash_catalog.endpoints import (
     handle_collection_only,
     handle_custom_endpoint,
     handle_GeoDB_endpoint,
+    handle_GeoDB_Features_endpoint,
     handle_rasdaman_endpoint,
     handle_raw_source,
     handle_SH_endpoint,
     handle_SH_WMS_endpoint,
+    handle_vector_tile_source,
     handle_VEDA_endpoint,
     handle_WMS_endpoint,
     handle_xcube_endpoint,
@@ -241,6 +243,15 @@ def process_collection_file(
                         catalog,
                         options,
                     )
+                elif endpoint_config["Name"] == "GeoDB Features":
+                    collection = handle_GeoDB_Features_endpoint(
+                        catalog_config,
+                        endpoint_config,
+                        collection_config,
+                        coll_path_rel_to_root_catalog,
+                        catalog,
+                        options,
+                    )
                 elif endpoint_config["Name"] == "VEDA":
                     collection = handle_VEDA_endpoint(
                         catalog_config,
@@ -282,16 +293,14 @@ def process_collection_file(
                         catalog,
                         options,
                     )
-                elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
-                    # somewhat one off creation of individual WMTS layers as individual items
-                    collection = handle_WMS_endpoint(
+                elif endpoint_config["Name"] == "VectorTile source":
+                    collection = handle_vector_tile_source(
                         catalog_config,
                         endpoint_config,
                         collection_config,
                         coll_path_rel_to_root_catalog,
                         catalog,
                         options,
-                        wmts=True,
                     )
                 elif endpoint_config["Name"] == "Collection-only":
                     collection = handle_collection_only(
@@ -350,9 +359,11 @@ def process_collection_file(
                     countries.extend(sub_coll_def["Country"])
                 else:
                     countries.append(sub_coll_def["Country"])
-                coll_path_rel_to_root_catalog = (
-                    f"{coll_path_rel_to_root_catalog}/{sub_coll_def['Collection']}"
-                )
+                # commented out intentionally, because otherwise paths further down did
+                # not match, parquet file was one level deeper
+                # coll_path_rel_to_root_catalog = (
+                #     f"{coll_path_rel_to_root_catalog}/{sub_coll_def['Collection']}"
+                # )
                 process_collection_file(
                     catalog_config,
                     "{}/{}".format(options.collectionspath, sub_coll_def["Collection"]),
@@ -411,6 +422,7 @@ def process_collection_file(
         add_collection_information(catalog_config, parent_collection, collection_config, True)
         add_process_info(parent_collection, catalog_config, collection_config)
         update_extents_from_collection_children(parent_collection)
+        add_base_overlay_info(parent_collection, catalog_config, collection_config)
         # Fill summaries for locations
         parent_collection.summaries = Summaries(
             {

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/sh_endpoint.py

@@ -1,10 +1,10 @@
 import os
 import time
+from urllib.parse import urlparse
 
 from oauthlib.oauth2 import BackendApplicationClient
 from requests_oauthlib import OAuth2Session
 
-SH_TOKEN_URL = "https://services.sentinel-hub.com/oauth/token"
 _token_cache: dict[str, dict] = {}
 
 
@@ -24,6 +24,8 @@ def get_SH_token(endpoint_config: dict) -> str:
     client = BackendApplicationClient(client_id=client_id)
     oauth = OAuth2Session(client=client)
     # Get token for the session
+    endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
+    SH_TOKEN_URL = f"https://{endpoint_url_parts.netloc}/oauth/token"
     token = oauth.fetch_token(
         token_url=SH_TOKEN_URL,
         client_secret=client_secret,

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/stac_handling.py

@@ -1,3 +1,4 @@
+import uuid
 from datetime import datetime, timezone
 
 import requests
@@ -18,6 +19,7 @@ from structlog import get_logger
 from eodash_catalog.utils import (
     generateDatetimesFromInterval,
     get_full_url,
+    make_intervals,
     parse_datestring_to_tz_aware_datetime,
     read_config_file,
 )
@@ -121,11 +123,14 @@ def create_service_link(endpoint_config: dict, catalog_config: dict) -> Link:
     return sl
 
 
-def create_web_map_link(layer_config: dict, role: str) -> Link:
+def create_web_map_link(
+    collection: Collection, catalog_config: dict, layer_config: dict, role: str
+) -> Link:
     extra_fields = {
         "roles": [role],
         "id": layer_config["id"],
     }
+    media_type = (layer_config.get("media_type", "image/png"),)
     if layer_config.get("default"):
         extra_fields["roles"].append("default")
     if layer_config.get("visible"):
@@ -145,12 +150,32 @@ def create_web_map_link(layer_config: dict, role: str) -> Link:
             extra_fields["wmts:layer"] = layer_config["layer"]
             if layer_config.get("dimensions"):
                 extra_fields["wmts:dimensions"] = layer_config["dimensions"]
+        case "vector-tile":
+            identifier = str(uuid.uuid4())
+            extra_fields["key"] = identifier
+            media_type = "application/vnd.mapbox-vector-tile"
+            if vector_tile_id_property := layer_config.get("idProperty"):
+                extra_fields["idProperty"] = vector_tile_id_property
+            if ep_st := layer_config.get("Style"):
+                style_link = Link(
+                    rel="style",
+                    target=ep_st
+                    if ep_st.startswith("http")
+                    else f"{catalog_config['assets_endpoint']}/{ep_st}",
+                    media_type="text/vector-styles",
+                    extra_fields={"links:keys": [identifier]},
+                )
+                collection.add_link(style_link)
+            add_authentication(collection, layer_config["url"], extra_fields)
+
     if layer_config.get("Attribution"):
         extra_fields["attribution"] = layer_config["Attribution"]
+    if layer_config.get("Colorlegend"):
+        extra_fields["eox:colorlegend"] = layer_config["Colorlegend"]
     wml = Link(
         rel=layer_config["protocol"],
         target=layer_config["url"],
-        media_type=layer_config.get("media_type", "image/png"),
+        media_type=media_type,
         title=layer_config["name"],
         extra_fields=extra_fields,
     )
@@ -306,18 +331,21 @@ def add_collection_information(
             ),
         )
     if collection_config.get("Image"):
+        # Check if absolute URL or relative path
+        if collection_config["Image"].startswith("http"):
+            image_url = collection_config["Image"]
+        else:
+            image_url = f'{catalog_config["assets_endpoint"]}/{collection_config["Image"]}'
         collection.add_asset(
             "thumbnail",
             Asset(
-                href=f'{catalog_config["assets_endpoint"]}/{collection_config["Image"]}',
+                href=image_url,
                 media_type="image/png",
                 roles=["thumbnail"],
             ),
         )
         # Bubble up thumbnail to extra fields
-        collection.extra_fields["thumbnail"] = (
-            f'{catalog_config["assets_endpoint"]}/' f'{collection_config["Image"]}'
-        )
+        collection.extra_fields["thumbnail"] = image_url
     # Add extra fields to collection if available
     add_extra_fields(collection, collection_config, is_root_collection)
 
@@ -472,23 +500,31 @@ def add_base_overlay_info(
     collection: Collection, catalog_config: dict, collection_config: dict
 ) -> None:
     # add custom baselayers specially for this indicator
-    if collection_config.get("BaseLayers"):
+    if "BaseLayers" in collection_config:
         for layer in collection_config["BaseLayers"]:
-            collection.add_link(create_web_map_link(layer, role="baselayer"))
+            collection.add_link(
+                create_web_map_link(collection, catalog_config, layer, role="baselayer")
+            )
     # alternatively use default base layers defined
     elif catalog_config.get("default_base_layers"):
         base_layers = read_config_file(catalog_config["default_base_layers"])
         for layer in base_layers:
-            collection.add_link(create_web_map_link(layer, role="baselayer"))
+            collection.add_link(
+                create_web_map_link(collection, catalog_config, layer, role="baselayer")
+            )
     # add custom overlays just for this indicator
-    if collection_config.get("OverlayLayers"):
+    if "OverlayLayers" in collection_config:
         for layer in collection_config["OverlayLayers"]:
-            collection.add_link(create_web_map_link(layer, role="overlay"))
+            collection.add_link(
+                create_web_map_link(collection, catalog_config, layer, role="overlay")
+            )
     # check if default overlay layers defined
     elif catalog_config.get("default_overlay_layers"):
         overlay_layers = read_config_file(catalog_config["default_overlay_layers"])
         for layer in overlay_layers:
-            collection.add_link(create_web_map_link(layer, role="overlay"))
+            collection.add_link(
+                create_web_map_link(collection, catalog_config, layer, role="overlay")
+            )
 
 
 def add_extra_fields(
@@ -535,16 +571,22 @@ def add_extra_fields(
 def get_collection_datetimes_from_config(endpoint_config: dict) -> list[datetime]:
     times_datetimes: list[datetime] = []
     if endpoint_config:
+        interval_between_dates = endpoint_config.get("WMSIntervalsBetweenDates")
         if endpoint_config.get("Times"):
             times = list(endpoint_config.get("Times", []))
             times_datetimes = sorted(
                 [parse_datestring_to_tz_aware_datetime(time) for time in times]
             )
+            if interval_between_dates:
+                # convert to list of datetime_start and datetime_end
+                times_datetimes = make_intervals(times_datetimes)
         elif endpoint_config.get("DateTimeInterval"):
             start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00Z")
             end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00Z")
             timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
-            times_datetimes = generateDatetimesFromInterval(start, end, timedelta_config)
+            times_datetimes = generateDatetimesFromInterval(
+                start, end, timedelta_config, interval_between_dates
+            )
     return times_datetimes
 
 
@@ -572,3 +614,20 @@ def add_projection_info(
             stac_object.extra_fields["eodash:proj4_def"] = proj
         else:
             raise Exception(f"Incorrect type of proj definition {proj}")
+
+
+def add_authentication(stac_object: Item | Collection | Catalog, url: str, extra_fields_link: dict):
+    if "api.mapbox" in url:
+        # add authentication info
+        auth_extension = "https://stac-extensions.github.io/authentication/v1.1.0/schema.json"
+        if auth_extension not in stac_object.stac_extensions:
+            stac_object.stac_extensions.append(auth_extension)
+        stac_object.extra_fields["auth:schemes"] = {
+            "mapboxauth": {
+                "type": "apiKey",
+                "name": "access_token",
+                "in": "query",
+            }
+        }
+        extra_fields_link["auth:refs"] = ["mapboxauth"]
+    pass

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/thumbnails.py

@@ -1,6 +1,7 @@
 import os
 import re
 from pathlib import Path
+from urllib.parse import urlparse
 
 import requests
 from pystac import (
@@ -46,7 +47,9 @@ def generate_thumbnail(
         # if it is start and end datetime have to exist
         if item_datetime:
             time = format_datetime_to_isostring_zulu(item_datetime)
-        url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format(
+        endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
+        url = "https://{}/ogc/wms/{}?{}&layers={}&time={}&{}".format(
+            endpoint_url_parts,
             instanceId,
             wms_config,
             endpoint_config["LayerId"],

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/src/eodash_catalog/utils.py

@@ -18,7 +18,7 @@ from dateutil import parser
 from owslib.wcs import WebCoverageService
 from owslib.wms import WebMapService
 from owslib.wmts import WebMapTileService
-from pystac import Asset, Catalog, Collection, Item, Link, RelType, SpatialExtent, TemporalExtent
+from pystac import Asset, Catalog, Collection, Item, RelType, SpatialExtent, TemporalExtent
 from pytz import timezone as pytztimezone
 from shapely import geometry as sgeom
 from shapely import wkb
@@ -214,7 +214,7 @@ def parse_duration(datestring):
 
 
 def generateDatetimesFromInterval(
-    start: str, end: str, timedelta_config: dict | None = None
+    start: str, end: str, timedelta_config: dict | None = None, interval_between_dates: bool = False
 ) -> list[datetime]:
     if timedelta_config is None:
         timedelta_config = {}
@@ -226,7 +226,10 @@ def generateDatetimesFromInterval(
     delta = timedelta(**timedelta_config)
     dates = []
     while start_dt <= end_dt:
-        dates.append(start_dt)
+        if interval_between_dates:
+            dates.append([start_dt, start_dt + delta - timedelta(seconds=1)])
+        else:
+            dates.append(start_dt)
         start_dt += delta
     return dates
 
@@ -290,8 +293,12 @@ def generate_veda_cog_link(endpoint_config: dict, file_url: str | None) -> str:
         )
 
     file_url = f"url={file_url}&" if file_url else ""
-
-    target_url = f"https://openveda.cloud/api/raster/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{file_url}resampling_method=nearest{bidx}{colormap}{colormap_name}{rescale}{Nodata}"
+    target_url_base = endpoint_config["EndPoint"].replace("/stac/", "")
+    target_url = (
+        f"{target_url_base}/raster/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?"
+        f"{file_url}resampling_method=nearest"
+        f"{bidx}{colormap}{colormap_name}{rescale}{Nodata}"
+    )
     return target_url
 
 
@@ -321,6 +328,7 @@ def add_single_item_if_collection_empty(endpoint_config: dict, collection: Colle
             datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
             start_datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
             end_datetime=datetime.now(tz=pytztimezone("UTC")),
+            assets={"dummy_asset": Asset(href="")},
         )
         collection.add_item(item)
         if not endpoint_config.get("OverwriteBBox"):
@@ -440,8 +448,37 @@ def update_extents_from_collection_children(collection: Collection):
         ):
             individual_datetimes.extend(c_child.extent.temporal.intervals[0])  # type: ignore
     individual_datetimes = list(filter(lambda x: x is not None, individual_datetimes))
-    time_extent = [min(individual_datetimes), max(individual_datetimes)]
-    collection.extent.temporal = TemporalExtent([time_extent])
+    if individual_datetimes:
+        time_extent = [min(individual_datetimes), max(individual_datetimes)]
+        collection.extent.temporal = TemporalExtent([time_extent])
+
+
+def extract_extent_from_geoparquet(table) -> tuple[TemporalExtent, SpatialExtent]:
+    """
+    Extract spatial and temporal extents from a GeoParquet file.
+    Args:
+        table (pyarrow.Table): The table containing the GeoParquet data.
+    Returns:
+        tuple: A tuple containing spatial and temporal extents.
+    """
+    # add extent information to the collection
+    min_datetime = pc.min(table["datetime"]).as_py()
+    max_datetime = pc.max(table["datetime"]).as_py()
+    if not min_datetime:
+        # cases when datetime was null
+        # fallback to start_datetime
+        min_datetime = pc.min(table["start_datetime"]).as_py()
+        max_datetime = pc.max(table["start_datetime"]).as_py()
+    # Making sure time extent is timezone aware
+    if min_datetime and min_datetime.tzinfo is None:
+        min_datetime = min_datetime.replace(tzinfo=timezone.utc)
+    if max_datetime and max_datetime.tzinfo is None:
+        max_datetime = max_datetime.replace(tzinfo=timezone.utc)
+    temporal = TemporalExtent([min_datetime, max_datetime])
+    geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
+    bbox = sgeom.MultiPolygon(geoms).bounds
+    spatial = SpatialExtent([bbox])
+    return [temporal, spatial]
 
 
 def save_items(
@@ -493,25 +530,9 @@ def save_items(
         output_path = f"{buildcatpath}/{colpath}"
         os.makedirs(output_path, exist_ok=True)
         stacgp.arrow.to_parquet(table, f"{output_path}/items.parquet")
-        gp_link = Link(
-            rel="items",
-            target="./items.parquet",
-            media_type="application/vnd.apache.parquet",
-            title="GeoParquet Items",
-        )
-        collection.add_link(gp_link)
-        # add extent information to the collection
-        min_datetime = pc.min(table["datetime"]).as_py()
-        max_datetime = pc.max(table["datetime"]).as_py()
-        if not min_datetime:
-            # cases when datetime was null
-            # fallback to start_datetime
-            min_datetime = pc.min(table["start_datetime"]).as_py()
-            max_datetime = pc.max(table["start_datetime"]).as_py()
-        collection.extent.temporal = TemporalExtent([min_datetime, max_datetime])
-        geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
-        bbox = sgeom.MultiPolygon(geoms).bounds
-        collection.extent.spatial = SpatialExtent([bbox])
+        extents = extract_extent_from_geoparquet(table)
+        collection.extent.temporal = extents[0]
+        collection.extent.spatial = extents[1]
         # Make sure to also reference the geoparquet as asset
         collection.add_asset(
             "geoparquet",
@@ -604,3 +625,25 @@ def merge_bboxes(bboxes: list[list[float]]) -> list[float]:
     max_lat = max(b[3] for b in bboxes)
 
     return [min_lon, min_lat, max_lon, max_lat]
+
+
+def make_intervals(datetimes: list[datetime]) -> list[list[datetime]]:
+    """
+    Converts a list of datetimes into list of lists of datetimes in format of [start,end]
+    where end is next element in original list minus 1 second
+    """
+    intervals = []
+    n = len(datetimes)
+    for i in range(n):
+        start = datetimes[i]
+        if i < n - 1:
+            # end is next datetime minus one second
+            end = datetimes[i + 1] - timedelta(seconds=1)
+        else:
+            prev_interval = timedelta(seconds=0)
+            # last item: use previous interval length added to last start
+            if n > 1:
+                prev_interval = datetimes[-1] - datetimes[-2]
+            end = start + prev_interval
+        intervals.append([start, end])
+    return intervals

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/tests/test_geoparquet.py

@@ -45,11 +45,11 @@ def test_geoparquet_geojson_items(catalog_output_folder):
 
     with open(os.path.join(child_collection_path, "collection.json")) as fp:
         collection_json = json.load(fp)
-        items_links = [link for link in collection_json["links"] if link["rel"] == "items"]
-        assert len(items_links) == 1
-        gp_link = items_links[0]
-        assert gp_link["type"] == "application/vnd.apache.parquet"
-        items_path = os.path.join(child_collection_path, gp_link["href"].split("/")[-1])
+        # check if parquet source is present in assets
+        assert "geoparquet" in collection_json["assets"]
+        parquet_asset = collection_json["assets"]["geoparquet"]
+        assert parquet_asset["type"] == "application/vnd.apache.parquet"
+        items_path = os.path.join(child_collection_path, parquet_asset["href"].split("/")[-1])
         assert os.path.exists(items_path)
 
     with open(items_path, "rb") as fp:

{eodash_catalog-0.3.0 → eodash_catalog-0.3.17}/tests/testing-collections/test_sh_wms.json

@@ -33,7 +33,7 @@
       "Name": "Sentinel Hub WMS",
       "CollectionId": "sentinel-1-grd",
       "LayerId": "SENTINEL_1_IW_VV",
-      "Bbox": [
+      "OverwriteBBox": [
         101.938,
         11.945,
         106.37,
@@ -65,4 +65,4 @@
       "Url": "https://www.copernicus.eu/en/access-data/conventional-data-access-hubs"
     }
   ]
-}
+}