eodash_catalog 0.1.12__tar.gz → 0.1.13__tar.gz

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.1.12
+current_version = 0.1.13
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/.github/workflows/ci.yml

@@ -10,6 +10,9 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
 
+    permissions:
+      contents: read
+      packages: write
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -25,3 +28,11 @@ jobs:
         run: python -m pip install .
       - name: Test
         run: cd tests && python -m pytest -p no:cacheprovider
+      - if: github.ref == 'refs/heads/main'
+        name: Build and push latest docker image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.repository }}
+          VERSION=${{ github.ref_name }}
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+          docker build -t $IMAGE_ID:latest .
+          docker push $IMAGE_ID:latest

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/.github/workflows/python-publish.yml

@@ -1,7 +1,7 @@
 # This workflows will upload a Python Package using Twine when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
-name: Upload Python Package and Docker image
+name: Upload Python Package and Docker image on tag
 
 on:
     push:
@@ -37,6 +37,5 @@ jobs:
         IMAGE_ID=ghcr.io/${{ github.repository }}
         VERSION=${{ github.ref_name }}
         echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
-        docker build -t $IMAGE_ID:latest -t $IMAGE_ID:$VERSION .
-        docker push $IMAGE_ID:latest
+        docker build -t $IMAGE_ID:$VERSION .
         docker push $IMAGE_ID:$VERSION

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/Dockerfile

@@ -29,4 +29,4 @@ RUN eodash_catalog --help
 
 CMD ["eodash_catalog"]
 
-LABEL version="0.1.12"
+LABEL version="0.1.13"

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eodash_catalog
-Version: 0.1.12
+Version: 0.1.13
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/src/eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.12"
+__version__ = "0.1.13"

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/src/eodash_catalog/endpoints.py

@@ -11,6 +11,8 @@ from operator import itemgetter
 import requests
 from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries
 from pystac_client import Client
+from shapely import wkt
+from shapely.geometry import mapping
 from structlog import get_logger
 
 from eodash_catalog.sh_endpoint import get_SH_token
@@ -26,7 +28,6 @@ from eodash_catalog.thumbnails import generate_thumbnail
 from eodash_catalog.utils import (
     Options,
     create_geojson_from_bbox,
-    create_geojson_point,
     filter_time_entries,
     format_datetime_to_isostring_zulu,
     generate_veda_cog_link,
@@ -147,7 +148,7 @@ def handle_STAC_based_endpoint(
     options: Options,
     headers=None,
 ) -> Collection:
-    if "Locations" in collection_config:
+    if collection_config.get("Locations"):
         root_collection = get_or_create_collection(
             catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
         )
@@ -169,7 +170,7 @@ def handle_STAC_based_endpoint(
             collection.id = location.get("Identifier", uuid.uuid4())
             collection.title = location.get("Name")
             # See if description should be overwritten
-            if "Description" in location:
+            if location.get("Description"):
                 collection.description = location["Description"]
             else:
                 collection.description = location["Name"]
@@ -184,7 +185,7 @@ def handle_STAC_based_endpoint(
             # eodash v4 compatibility
             add_visualization_info(collection, collection_config, endpoint_config)
             add_process_info_child_collection(collection, catalog_config, collection_config)
-            if "OverwriteBBox" in location:
+            if location.get("OverwriteBBox"):
                 collection.extent.spatial = SpatialExtent(
                     [
                         location["OverwriteBBox"],
@@ -197,7 +198,7 @@ def handle_STAC_based_endpoint(
                 root_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
     else:
         bbox = None
-        if "Bbox" in endpoint_config:
+        if endpoint_config.get("Bbox"):
             bbox = ",".join(map(str, endpoint_config["Bbox"]))
         root_collection = process_STACAPI_Endpoint(
             catalog_config=catalog_config,
@@ -263,17 +264,17 @@ def process_STACAPI_Endpoint(
             added_times[iso_date] = True
         link = collection.add_item(item)
         if options.tn:
-            if "cog_default" in item.assets:
+            if item.assets.get("cog_default"):
                 generate_thumbnail(
                     item, collection_config, endpoint_config, item.assets["cog_default"].href
                 )
             else:
                 generate_thumbnail(item, collection_config, endpoint_config)
         # Check if we can create visualization link
-        if "Assets" in endpoint_config:
+        if endpoint_config.get("Assets"):
             add_visualization_info(item, collection_config, endpoint_config, item.id)
             link.extra_fields["item"] = item.id
-        elif "cog_default" in item.assets:
+        elif item.assets.get("cog_default"):
             add_visualization_info(
                 item, collection_config, endpoint_config, item.assets["cog_default"].href
             )
@@ -282,7 +283,7 @@ def process_STACAPI_Endpoint(
             add_visualization_info(
                 item, collection_config, endpoint_config, datetimes=[item_datetime]
             )
-        elif "start_datetime" in item.properties and "end_datetime" in item.properties:
+        elif item.properties.get("start_datetime") and item.properties.get("end_datetime"):
             add_visualization_info(
                 item,
                 collection_config,
@@ -323,7 +324,7 @@ def process_STACAPI_Endpoint(
     add_collection_information(catalog_config, collection, collection_config)
 
     # Check if we need to overwrite the bbox after update from items
-    if "OverwriteBBox" in endpoint_config:
+    if endpoint_config.get("OverwriteBBox"):
         collection.extent.spatial = SpatialExtent(
             [
                 endpoint_config["OverwriteBBox"],
@@ -377,7 +378,7 @@ def handle_SH_WMS_endpoint(
     root_collection = get_or_create_collection(
         catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
     )
-    if "Locations" in collection_config:
+    if collection_config.get("Locations"):
         for location in collection_config["Locations"]:
             # create  and populate location collections based on times
             # TODO: Should we add some new description per location?
@@ -483,7 +484,7 @@ def handle_GeoDB_endpoint(
     collection = get_or_create_collection(
         catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
     )
-    select = "?select=aoi,aoi_id,country,city,time"
+    select = "?select=aoi,aoi_id,country,city,time,input_data,sub_aoi"
     url = (
         endpoint_config["EndPoint"]
         + endpoint_config["Database"]
@@ -501,12 +502,12 @@ def handle_GeoDB_endpoint(
     for key, value in groupby(sorted_locations, key=itemgetter("aoi_id")):
         # Finding min and max values for date
         values = list(value)
-        times = [datetime.fromisoformat(t["time"]) for t in values]
         unique_values = next(iter({v["aoi_id"]: v for v in values}.values()))
         country = unique_values["country"]
         city = unique_values["city"]
         IdKey = endpoint_config.get("IdKey", "city")
         IdValue = unique_values[IdKey]
+
         if country not in countries:
             countries.append(country)
         # sanitize unique key identifier to be sure it is saveable as a filename
@@ -520,28 +521,92 @@ def handle_GeoDB_endpoint(
             IdValue = key
         if city not in cities:
             cities.append(city)
-        min_date = min(times)
-        max_date = max(times)
         latlon = unique_values["aoi"]
         [lat, lon] = [float(x) for x in latlon.split(",")]
         # create item for unique locations
         buff = 0.01
         bbox = [lon - buff, lat - buff, lon + buff, lat + buff]
-        item = Item(
-            id=IdValue,
-            bbox=bbox,
-            properties={},
-            geometry=create_geojson_point(lon, lat)["geometry"],
-            datetime=None,
-            start_datetime=min_date,
-            end_datetime=max_date,
+
+        # create collection per available inputdata information
+        sc_config = {
+            "Title": city,
+            "Description": f"{city} - {country}",
+        }
+        locations_collection = get_or_create_collection(
+            collection, key, sc_config, catalog_config, endpoint_config
         )
-        link = collection.add_item(item)
+        input_data = endpoint_config.get("InputData")
+        if input_data:
+            for v in values:
+                # add items based on inputData fields for each time step available in values
+                first_match = next(
+                    (item for item in input_data if item.get("Identifier") == v["input_data"]), None
+                )
+                time_object = datetime.fromisoformat(v["time"])
+                # extract wkt geometry from sub_aoi
+                if "sub_aoi" in v and v["sub_aoi"] != "/":
+                    # create geometry from wkt
+                    geometry = mapping(wkt.loads(v["sub_aoi"]))
+                else:
+                    geometry = create_geojson_from_bbox(bbox)
+                item = Item(
+                    id=v["time"],
+                    bbox=bbox,
+                    properties={},
+                    geometry=geometry,
+                    datetime=time_object,
+                )
+                if first_match:
+                    match first_match["Type"]:
+                        case "WMS":
+                            url = first_match["Url"]
+                            extra_fields = {
+                                "wms:layers": [first_match["Layers"]],
+                                "role": ["data"],
+                            }
+                            if url.startswith("https://services.sentinel-hub.com/ogc/wms/"):
+                                instanceId = os.getenv("SH_INSTANCE_ID")
+                                if "InstanceId" in endpoint_config:
+                                    instanceId = endpoint_config["InstanceId"]
+                                start_date = format_datetime_to_isostring_zulu(time_object)
+                                used_delta = timedelta(days=1)
+                                if "TimeDelta" in first_match:
+                                    used_delta = timedelta(minutes=first_match["TimeDelta"])
+                                end_date = format_datetime_to_isostring_zulu(
+                                    time_object + used_delta - timedelta(milliseconds=1)
+                                )
+                                extra_fields.update(
+                                    {"wms:dimensions": {"TIME": f"{start_date}/{end_date}"}}
+                                )
+                                # we add the instance id to the url
+                                url = f"https://services.sentinel-hub.com/ogc/wms/{instanceId}"
+                            else:
+                                extra_fields.update({"wms:dimensions": {"TIME": v["time"]}})
+                            link = Link(
+                                rel="wms",
+                                target=url,
+                                media_type=(endpoint_config.get("MimeType", "image/png")),
+                                title=collection_config["Name"],
+                                extra_fields=extra_fields,
+                            )
+                            item.add_link(link)
+                            itemlink = locations_collection.add_item(item)
+                            itemlink.extra_fields["datetime"] = (
+                                f"{format_datetime_to_isostring_zulu(time_object)}Z"
+                            )
+
+            # add_visualization_info(
+            #     item, collection_config, endpoint_config, file_url=first_match.get("FileUrl")
+            # )
+        locations_collection.extra_fields["subcode"] = key
+        link = collection.add_child(locations_collection)
+        locations_collection.update_extent_from_items()
+        # collection.update_extent_from_items()
         # bubble up information we want to the link
         link.extra_fields["id"] = key
         link.extra_fields["latlng"] = latlon
         link.extra_fields["country"] = country
-        link.extra_fields["city"] = city
+        link.extra_fields["name"] = city
 
     if "yAxis" not in collection_config:
         # fetch yAxis and store it to data, preventing need to save it per dataset in yml
@@ -557,7 +622,7 @@ def handle_GeoDB_endpoint(
         collection_config["yAxis"] = yAxis
     add_collection_information(catalog_config, collection, collection_config)
     add_example_info(collection, collection_config, endpoint_config, catalog_config)
-    collection.extra_fields["geoDBID"] = endpoint_config["CollectionId"]
+    collection.extra_fields["locations"] = True
 
     collection.update_extent_from_items()
     collection.summaries = Summaries(
@@ -580,7 +645,7 @@ def handle_SH_endpoint(
     headers = {"Authorization": f"Bearer {token}"}
     endpoint_config["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
     # Overwrite collection id with type, such as ZARR or BYOC
-    if "Type" in endpoint_config:
+    if endpoint_config.get("Type"):
         endpoint_config["CollectionId"] = (
             endpoint_config["Type"] + "-" + endpoint_config["CollectionId"]
         )
@@ -637,7 +702,7 @@ def handle_WMS_endpoint(
         LOGGER.warn(f"NO datetimes returned for collection: {collection_config['Name']}!")
 
     # Check if we should overwrite bbox
-    if "OverwriteBBox" in endpoint_config:
+    if endpoint_config.get("OverwriteBBox"):
         collection.extent.spatial = SpatialExtent(
             [
                 endpoint_config["OverwriteBBox"],
@@ -655,10 +720,10 @@ def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
     for asset in endpoint_config["Assets"]:
         assets += f"&assets={asset}"
     color_formula = ""
-    if "ColorFormula" in endpoint_config:
+    if endpoint_config.get("ColorFormula"):
         color_formula = "&color_formula={}".format(endpoint_config["ColorFormula"])
     no_data = ""
-    if "NoData" in endpoint_config:
+    if endpoint_config.get("NoData"):
         no_data = "&no_data={}".format(endpoint_config["NoData"])
     item = item if item else "{item}"
     target_url = f"https://openveda.cloud/api/raster/collections/{collection}/items/{item}/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}"
@@ -673,7 +738,7 @@ def add_visualization_info(
     datetimes: list[datetime] | None = None,
 ) -> None:
     extra_fields: dict[str, list[str] | dict[str, str]] = {}
-    if "Attribution" in endpoint_config:
+    if endpoint_config.get("Attribution"):
         stac_object.stac_extensions.append(
             "https://stac-extensions.github.io/attribution/v0.1.0/schema.json"
         )
@@ -681,7 +746,7 @@ def add_visualization_info(
     # add extension reference
     if endpoint_config["Name"] == "Sentinel Hub" or endpoint_config["Name"] == "Sentinel Hub WMS":
         instanceId = os.getenv("SH_INSTANCE_ID")
-        if "InstanceId" in endpoint_config:
+        if endpoint_config.get("InstanceId"):
             instanceId = endpoint_config["InstanceId"]
         if env_id := endpoint_config.get("CustomSHEnvId"):
             # special handling for custom environment
@@ -733,16 +798,21 @@ def add_visualization_info(
         dimensions = {}
         if dimensions_config := endpoint_config.get("Dimensions", {}):
             for key, value in dimensions_config.items():
+                # special replace for world_settlement_footprint
+                if collection_config["EodashIdentifier"] == "WSF":
+                    value = value.replace(
+                        "{time}", datetimes is not None and str(datetimes[0].year) or "{time}"
+                    )
                 dimensions[key] = value
         if datetimes is not None:
             dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
         if dimensions != {}:
             extra_fields["wms:dimensions"] = dimensions
-        if "Styles" in endpoint_config:
+        if endpoint_config.get("Styles"):
             extra_fields["wms:styles"] = endpoint_config["Styles"]
-        if "TileSize" in endpoint_config:
+        if endpoint_config.get("TileSize"):
             extra_fields["wms:tilesize"] = endpoint_config["TileSize"]
-        if "Version" in endpoint_config:
+        if endpoint_config.get("Version"):
             extra_fields["wms:version"] = endpoint_config["Version"]
         media_type = endpoint_config.get("MediaType", "image/jpeg")
         endpoint_url = endpoint_config["EndPoint"]
@@ -775,7 +845,7 @@ def add_visualization_info(
             dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
         if dimensions != {}:
             extra_fields["wms:dimensions"] = dimensions
-        if "Styles" in endpoint_config:
+        if endpoint_config.get("Styles"):
             extra_fields["wms:styles"] = endpoint_config["Styles"]
         media_type = endpoint_config.get("MediaType", "image/png")
         endpoint_url = endpoint_config["EndPoint"]
@@ -821,14 +891,18 @@ def add_visualization_info(
             # either preset Rescale of left as a template
             vmin = "{vmin}"
             vmax = "{vmax}"
-            if "Rescale" in endpoint_config:
+            if endpoint_config.get("Rescale"):
                 vmin = endpoint_config["Rescale"][0]
                 vmax = endpoint_config["Rescale"][1]
             # depending on numerical input only
             data_projection = str(endpoint_config.get("DataProjection", 3857))
             epsg_prefix = "" if "EPSG:" in data_projection else "EPSG:"
             crs = f"{epsg_prefix}{data_projection}"
-            time = stac_object.get_datetime() if isinstance(stac_object, Item) else "{time}"
+            time = (
+                stac_object.get_datetime().strftime("%Y-%m-%dT%H:%M:%SZ")  # type: ignore
+                if isinstance(stac_object, Item)
+                else "{time}"
+            )
             target_url = (
                 "{}/tiles/{}/{}/{{z}}/{{y}}/{{x}}" "?crs={}&time={}&vmin={}&vmax={}&cbar={}"
             ).format(
@@ -998,7 +1072,7 @@ def handle_raw_source(
                 assets=assets,
                 extra_fields={},
             )
-            if "Attribution" in endpoint_config:
+            if endpoint_config.get("Attribution"):
                 item.stac_extensions.append(
                     "https://stac-extensions.github.io/attribution/v0.1.0/schema.json"
                 )

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/src/eodash_catalog/generate_indicators.py

@@ -39,7 +39,7 @@ from eodash_catalog.utils import (
     Options,
     RaisingThread,
     add_single_item_if_collection_empty,
-    iter_len_at_least,
+    merge_bboxes,
     read_config_file,
     recursive_save,
     retry,
@@ -88,7 +88,7 @@ def process_catalog_file(file_path: str, options: Options):
                 )
             except FileNotFoundError:
                 LOGGER.info(f"Warning: neither collection nor indicator found for {collection}")
-    if "MapProjection" in catalog_config:
+    if catalog_config.get("MapProjection"):
         catalog.extra_fields["eodash:mapProjection"] = catalog_config["MapProjection"]
 
     strategy = TemplateLayoutStrategy(item_template="${collection}/${year}")
@@ -163,10 +163,11 @@ def process_indicator_file(
 ):
     LOGGER.info(f"Processing indicator: {file_path}")
     indicator_config = read_config_file(file_path)
+
     parent_indicator = get_or_create_collection(
         catalog, indicator_config["Name"], indicator_config, catalog_config, {}
     )
-    if "Collections" in indicator_config:
+    if indicator_config.get("Collections"):
         for collection in indicator_config["Collections"]:
             process_collection_file(
                 catalog_config,
@@ -179,11 +180,20 @@ def process_indicator_file(
         # we assume that collection files can also be loaded directly
         process_collection_file(catalog_config, file_path, parent_indicator, options)
     add_collection_information(catalog_config, parent_indicator, indicator_config, True)
-    if iter_len_at_least(parent_indicator.get_items(recursive=True), 1):
-        parent_indicator.update_extent_from_items()
+    # get shared extent of all of the collections
+    # we do not want to use update_extent_from_items() because
+    # they might have OverwriteBBox and that would discard it for indicator
+    merged_bbox = merge_bboxes(
+        [
+            c_child.extent.spatial.bboxes[0]
+            for c_child in parent_indicator.get_children()
+            if isinstance(c_child, Collection)
+        ]
+    )
+    parent_indicator.extent.spatial.bboxes = [merged_bbox]
     # Add bbox extents from children
     for c_child in parent_indicator.get_children():
-        if isinstance(c_child, Collection):  # typing reason
+        if isinstance(c_child, Collection) and merged_bbox != c_child.extent.spatial.bboxes[0]:
             parent_indicator.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
     # extract collection information and add it to summary indicator level
     extract_indicator_info(parent_indicator)
@@ -203,7 +213,7 @@ def process_collection_file(
 ):
     LOGGER.info(f"Processing collection: {file_path}")
     collection_config = read_config_file(file_path)
-    if "Resources" in collection_config:
+    if collection_config.get("Resources"):
         for endpoint_config in collection_config["Resources"]:
             try:
                 collection = None
@@ -275,7 +285,7 @@ def process_collection_file(
                 LOGGER.warn(f"""Exception: {e.args[0]} with config: {endpoint_config}""")
                 raise e
 
-    elif "Subcollections" in collection_config:
+    elif collection_config.get("Subcollections"):
         # if no endpoint is specified we check for definition of subcollections
         parent_collection = get_or_create_collection(
             catalog, collection_config["Name"], collection_config, catalog_config, {}
@@ -286,7 +296,7 @@ def process_collection_file(
         for sub_coll_def in collection_config["Subcollections"]:
             # Subcollection has only data on one location which
             # is defined for the entire collection
-            if "Name" in sub_coll_def and "Point" in sub_coll_def:
+            if sub_coll_def.get("Name") and sub_coll_def.get("Point"):
                 locations.append(sub_coll_def["Name"])
                 if isinstance(sub_coll_def["Country"], list):
                     countries.extend(sub_coll_def["Country"])
@@ -302,7 +312,7 @@ def process_collection_file(
                 for link in parent_collection.links:
                     if (
                         link.rel == "child"
-                        and "id" in link.extra_fields
+                        and link.extra_fields.get("id")
                         and link.extra_fields["id"] == sub_coll_def["Identifier"]
                     ):
                         latlng = "{},{}".format(
@@ -330,9 +340,9 @@ def process_collection_file(
                 links = tmp_catalog.get_child(sub_coll_def["Identifier"]).get_links()  # type: ignore
                 for link in links:
                     # extract summary information
-                    if "city" in link.extra_fields:
+                    if link.extra_fields.get("city"):
                         locations.append(link.extra_fields["city"])
-                    if "country" in link.extra_fields:
+                    if link.extra_fields.get("country"):
                         if isinstance(link.extra_fields["country"], list):
                             countries.extend(link.extra_fields["country"])
                         else:
@@ -340,7 +350,7 @@ def process_collection_file(
 
                 parent_collection.add_links(links)
 
-        add_collection_information(catalog_config, parent_collection, collection_config)
+        add_collection_information(catalog_config, parent_collection, collection_config, True)
         add_process_info(parent_collection, catalog_config, collection_config)
         parent_collection.update_extent_from_items()
         # Add bbox extents from children
@@ -368,7 +378,7 @@ def add_to_catalog(
 
     link: Link = catalog.add_child(collection)
     # bubble fields we want to have up to collection link and add them to collection
-    if endpoint and "Type" in endpoint:
+    if endpoint and endpoint.get("Type") and endpoint["Type"] not in ["GeoDB"]:
         collection.extra_fields["endpointtype"] = "{}_{}".format(
             endpoint["Name"],
             endpoint["Type"],
@@ -380,12 +390,12 @@ def add_to_catalog(
     elif endpoint:
         collection.extra_fields["endpointtype"] = endpoint["Name"]
         link.extra_fields["endpointtype"] = endpoint["Name"]
-    if "Subtitle" in collection_config:
+    if collection_config.get("Subtitle"):
         link.extra_fields["subtitle"] = collection_config["Subtitle"]
     link.extra_fields["title"] = collection.title
     link.extra_fields["code"] = collection_config["EodashIdentifier"]
     link.extra_fields["id"] = collection_config["Name"]
-    if "Themes" in collection_config:
+    if collection_config.get("Themes"):
         link.extra_fields["themes"] = collection_config["Themes"]
     # Check for summaries and bubble up info
     if disable:

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/src/eodash_catalog/stac_handling.py

@@ -51,9 +51,9 @@ def get_or_create_collection(
             temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
 
     extent = Extent(spatial=spatial_extent, temporal=temporal_extent)
-
+    description = ""
     # Check if description is link to markdown file
-    if "Description" in collection_config:
+    if collection_config.get("Description"):
         description = collection_config["Description"]
         if description.endswith((".md", ".MD")):
             if description.startswith("http"):
@@ -61,7 +61,7 @@ def get_or_create_collection(
                 response = requests.get(description)
                 if response.status_code == 200:
                     description = response.text
-                elif "Subtitle" in collection_config:
+                elif collection_config.get("Subtitle"):
                     LOGGER.warn("Markdown file could not be fetched")
                     description = collection_config["Subtitle"]
             else:
@@ -69,10 +69,10 @@ def get_or_create_collection(
                 response = requests.get(f'{catalog_config["assets_endpoint"]}/{description}')
                 if response.status_code == 200:
                     description = response.text
-                elif "Subtitle" in collection_config:
+                elif collection_config.get("Subtitle"):
                     LOGGER.warn("Markdown file could not be fetched")
                     description = collection_config["Subtitle"]
-    elif "Subtitle" in collection_config:
+    elif collection_config.get("Subtitle"):
         # Try to use at least subtitle to fill some information
         description = collection_config["Subtitle"]
 
@@ -90,12 +90,28 @@ def create_service_link(endpoint_config: dict, catalog_config: dict) -> Link:
         "id": endpoint_config["Identifier"],
         "method": endpoint_config.get("Method", "GET"),
     }
-    if "EndPoint" in endpoint_config:
+    if endpoint_config.get("EndPoint"):
         extra_fields["endpoint"] = endpoint_config["EndPoint"]
-    if "Body" in endpoint_config:
+    if endpoint_config.get("Body"):
         extra_fields["body"] = get_full_url(endpoint_config["Body"], catalog_config)
-    if "Flatstyle" in endpoint_config:
-        extra_fields["eox:flatstyle"] = get_full_url(endpoint_config["Flatstyle"], catalog_config)
+    if endpoint_config.get("Flatstyle"):
+        # either a string
+        if isinstance(endpoint_config["Flatstyle"], str):
+            # update URL if needed
+            extra_fields["eox:flatstyle"] = get_full_url(
+                endpoint_config["Flatstyle"], catalog_config
+            )
+        elif isinstance(endpoint_config["Flatstyle"], list):
+            # or a list of objects - update URL if needed
+            extra_fields["eox:flatstyle"] = []
+            for flatstyle_config in endpoint_config["Flatstyle"]:
+                flatstyle_obj = {
+                    "Identifier": flatstyle_config.get("Identifier"),
+                    "Url": get_full_url(flatstyle_config.get("Url"), catalog_config),
+                }
+                extra_fields["eox:flatstyle"].append(flatstyle_obj)
+        else:
+            LOGGER.warn("Flatstyle is invalid type", endpoint_config["Flatstyle"])
     sl = Link(
         rel="service",
         target=endpoint_config["Url"],
@@ -121,15 +137,15 @@ def create_web_map_link(layer_config: dict, role: str) -> Link:
         case "wms":
             # handle wms special config options
             extra_fields["wms:layers"] = layer_config["layers"]
-            if "styles" in layer_config:
+            if layer_config.get("styles"):
                 extra_fields["wms:styles"] = layer_config["styles"]
-            if "dimensions" in layer_config:
+            if layer_config.get("dimensions"):
                 extra_fields["wms:dimensions"] = layer_config["dimensions"]
         case "wmts":
             extra_fields["wmts:layer"] = layer_config["layer"]
-            if "dimensions" in layer_config:
+            if layer_config.get("dimensions"):
                 extra_fields["wmts:dimensions"] = layer_config["dimensions"]
-    if "Attribution" in layer_config:
+    if layer_config.get("Attribution"):
         extra_fields["attribution"] = layer_config["Attribution"]
     wml = Link(
         rel=layer_config["protocol"],
@@ -148,7 +164,7 @@ def add_example_info(
     endpoint_config: dict,
     catalog_config: dict,
 ) -> None:
-    if "Services" in collection_config:
+    if collection_config.get("Services"):
         for service in collection_config["Services"]:
             if service["Name"] == "Statistical API":
                 service_type = service.get("Type", "byoc")
@@ -182,7 +198,7 @@ def add_example_info(
                     Link(
                         rel="example",
                         target=service["Url"],
-                        title=(service["Title"] if "Title" in service else service["Name"]),
+                        title=service.get("Title", service.get("Name")),
                         media_type="application/x-ipynb+json",
                         extra_fields={
                             "example:language": "Jupyter Notebook",
@@ -190,26 +206,6 @@ def add_example_info(
                         },
                     )
                 )
-    elif "Resources" in collection_config:
-        for service in collection_config["Resources"]:
-            if service.get("Name") == "xcube":
-                target_url = "{}/timeseries/{}/{}?aggMethods=median".format(
-                    endpoint_config["EndPoint"],
-                    endpoint_config["DatacubeId"],
-                    endpoint_config["Variable"],
-                )
-                stac_object.add_link(
-                    Link(
-                        rel="example",
-                        target=target_url,
-                        title=service["Name"] + " analytics",
-                        media_type="application/json",
-                        extra_fields={
-                            "example:language": "JSON",
-                            "example:method": "POST",
-                        },
-                    )
-                )
 
 
 def add_collection_information(
@@ -220,7 +216,7 @@ def add_collection_information(
 ) -> None:
     # Add metadata information
     # Check license identifier
-    if "License" in collection_config:
+    if collection_config.get("License"):
         # Check if list was provided
         if isinstance(collection_config["License"], list):
             if len(collection_config["License"]) == 1:
@@ -230,7 +226,7 @@ def add_collection_information(
                     target=collection_config["License"][0]["Url"],
                     media_type=(collection_config["License"][0].get("Type", "text/html")),
                 )
-                if "Title" in collection_config["License"][0]:
+                if collection_config["License"][0].get("Title"):
                     link.title = collection_config["License"][0]["Title"]
                 collection.links.append(link)
             elif len(collection_config["License"]) > 1:
@@ -239,11 +235,9 @@ def add_collection_information(
                     link = Link(
                         rel="license",
                         target=license_entry["Url"],
-                        media_type="text/html"
-                        if "Type" in license_entry
-                        else license_entry["Type"],
+                        media_type=license_entry.get("Type", "text/html"),
                     )
-                    if "Title" in license_entry:
+                    if license_entry.get("Title"):
                         link.title = license_entry["Title"]
                     collection.links.append(link)
         else:
@@ -267,7 +261,7 @@ def add_collection_information(
     else:
         pass
 
-    if "Provider" in collection_config:
+    if collection_config.get("Provider"):
         try:
             collection.providers = [
                 Provider(
@@ -279,21 +273,21 @@ def add_collection_information(
         except Exception:
             LOGGER.warn(f"Issue creating provider information for collection: {collection.id}")
 
-    if "Citation" in collection_config:
-        if "DOI" in collection_config["Citation"]:
+    if collection_config.get("Citation"):
+        if collection_config["Citation"].get("DOI"):
             collection.extra_fields["sci:doi"] = collection_config["Citation"]["DOI"]
-        if "Citation" in collection_config["Citation"]:
+        if collection_config["Citation"].get("Citation"):
             collection.extra_fields["sci:citation"] = collection_config["Citation"]["Citation"]
-        if "Publication" in collection_config["Citation"]:
+        if collection_config["Citation"].get("Publication"):
             collection.extra_fields["sci:publications"] = [
                 # convert keys to lower case
                 {k.lower(): v for k, v in publication.items()}
                 for publication in collection_config["Citation"]["Publication"]
             ]
 
-    if "Subtitle" in collection_config:
+    if collection_config.get("Subtitle"):
         collection.extra_fields["subtitle"] = collection_config["Subtitle"]
-    if "Legend" in collection_config:
+    if collection_config.get("Legend"):
         collection.add_asset(
             "legend",
             Asset(
@@ -302,7 +296,7 @@ def add_collection_information(
                 roles=["metadata"],
             ),
         )
-    if "Story" in collection_config:
+    if collection_config.get("Story"):
         collection.add_asset(
             "story",
             Asset(
@@ -311,7 +305,7 @@ def add_collection_information(
                 roles=["metadata"],
             ),
         )
-    if "Image" in collection_config:
+    if collection_config.get("Image"):
         collection.add_asset(
             "thumbnail",
             Asset(
@@ -327,10 +321,10 @@ def add_collection_information(
     # Add extra fields to collection if available
     add_extra_fields(collection, collection_config, is_root_collection)
 
-    if "References" in collection_config:
+    if collection_config.get("References"):
         generic_counter = 1
         for ref in collection_config["References"]:
-            if "Key" in ref:
+            if ref.get("Key"):
                 key = ref["Key"]
             else:
                 key = f"reference_{generic_counter}"
@@ -344,14 +338,14 @@ def add_collection_information(
                     roles=["metadata"],
                 ),
             )
-    if "Colorlegend" in collection_config:
+    if collection_config.get("Colorlegend"):
         collection.extra_fields["eox:colorlegend"] = collection_config["Colorlegend"]
 
 
 def add_process_info(collection: Collection, catalog_config: dict, collection_config: dict) -> None:
-    if any(key in collection_config for key in ["Locations", "Subcollections"]):
+    if any(collection_config.get(key) for key in ["Locations", "Subcollections"]):
         # add the generic geodb-like selection process on the root collection instead of Processes
-        if "geodb_default_form" in catalog_config:
+        if catalog_config.get("geodb_default_form"):
             # adding default geodb-like map handling for Locations
             collection.extra_fields["eodash:jsonform"] = get_full_url(
                 catalog_config["geodb_default_form"], catalog_config
@@ -369,29 +363,44 @@ def add_process_info(collection: Collection, catalog_config: dict, collection_co
             },
         )
         collection.add_link(sl)
+        # adding additional service links
+        if collection_config["Name"] == "GeoDB" and collection_config.get("Process", {}).get(
+            "EndPoints"
+        ):
+            for endpoint in collection_config["Process"]["EndPoints"]:
+                collection.add_link(create_service_link(endpoint, catalog_config))
+
+        # for geodb collections now based on locations, we want to make sure
+        # also manually defined processes are added to the collection
+        if collection_config["Name"] == "GeoDB" and collection_config.get("Process", {}).get(
+            "VegaDefinition"
+        ):
+            collection.extra_fields["eodash:vegadefinition"] = get_full_url(
+                collection_config["Process"]["VegaDefinition"], catalog_config
+            )
     # elif is intentional for cases when Process is defined on collection with Locations
     # then we want to only add it to the "children", not the root
-    elif "Process" in collection_config:
-        if "EndPoints" in collection_config["Process"]:
+    elif collection_config.get("Process"):
+        if collection_config["Process"].get("EndPoints"):
             for endpoint in collection_config["Process"]["EndPoints"]:
                 collection.add_link(create_service_link(endpoint, catalog_config))
-        if "JsonForm" in collection_config["Process"]:
+        if collection_config["Process"].get("JsonForm"):
             collection.extra_fields["eodash:jsonform"] = get_full_url(
                 collection_config["Process"]["JsonForm"], catalog_config
             )
-        if "VegaDefinition" in collection_config["Process"]:
+        if collection_config["Process"].get("VegaDefinition"):
             collection.extra_fields["eodash:vegadefinition"] = get_full_url(
                 collection_config["Process"]["VegaDefinition"], catalog_config
             )
-    elif "Resources" in collection_config:
+    elif collection_config.get("Resources"):
         # see if geodb resource configured use defaults if available
         for resource in collection_config["Resources"]:
             if resource["Name"] == "GeoDB":
-                if "geodb_default_form" in catalog_config:
+                if catalog_config.get("geodb_default_form"):
                     collection.extra_fields["eodash:jsonform"] = get_full_url(
                         catalog_config["geodb_default_form"], catalog_config
                     )
-                if "geodb_default_vega" in catalog_config:
+                if catalog_config.get("geodb_default_vega"):
                     collection.extra_fields["eodash:vegadefinition"] = get_full_url(
                         catalog_config["geodb_default_vega"], catalog_config
                     )
@@ -412,21 +421,49 @@ def add_process_info(collection: Collection, catalog_config: dict, collection_co
                         },
                     )
                 )
+            elif resource["Name"] == "xcube" and catalog_config.get("default_xcube_process"):
+                target_url = "{}/timeseries/{}/{}?aggMethods=median".format(
+                    resource["EndPoint"],
+                    resource["DatacubeId"],
+                    resource["Variable"],
+                )
+                process_endpoint_config = catalog_config["default_xcube_process"]["EndPoints"][0]
+                extra_fields = {
+                    "id": process_endpoint_config["Identifier"],
+                    "method": process_endpoint_config.get("Method", "GET"),
+                }
+                extra_fields["body"] = get_full_url(process_endpoint_config["Body"], catalog_config)
+                if catalog_config["default_xcube_process"].get("JsonForm"):
+                    collection.extra_fields["eodash:jsonform"] = get_full_url(
+                        catalog_config["default_xcube_process"]["JsonForm"], catalog_config
+                    )
+                if catalog_config["default_xcube_process"].get("VegaDefinition"):
+                    collection.extra_fields["eodash:vegadefinition"] = get_full_url(
+                        catalog_config["default_xcube_process"]["VegaDefinition"], catalog_config
+                    )
+
+                sl = Link(
+                    rel="service",
+                    target=target_url,
+                    media_type=process_endpoint_config["Type"],
+                    extra_fields=extra_fields,
+                )
+                collection.add_link(sl)
 
 
 def add_process_info_child_collection(
     collection: Collection, catalog_config: dict, collection_config: dict
 ) -> None:
     # in case of locations, we add the process itself on a child collection
-    if "Process" in collection_config:
-        if "EndPoints" in collection_config["Process"]:
+    if collection_config.get("Process"):
+        if collection_config["Process"].get("EndPoints"):
             for endpoint in collection_config["Process"]["EndPoints"]:
                 collection.add_link(create_service_link(endpoint, catalog_config))
-        if "JsonForm" in collection_config["Process"]:
+        if collection_config["Process"].get("JsonForm"):
             collection.extra_fields["eodash:jsonform"] = get_full_url(
                 collection_config["Process"]["JsonForm"], catalog_config
             )
-        if "VegaDefinition" in collection_config["Process"]:
+        if collection_config["Process"].get("VegaDefinition"):
             collection.extra_fields["eodash:vegadefinition"] = get_full_url(
                 collection_config["Process"]["VegaDefinition"], catalog_config
             )
@@ -436,20 +473,20 @@ def add_base_overlay_info(
     collection: Collection, catalog_config: dict, collection_config: dict
 ) -> None:
     # add custom baselayers specially for this indicator
-    if "BaseLayers" in collection_config:
+    if collection_config.get("BaseLayers"):
         for layer in collection_config["BaseLayers"]:
             collection.add_link(create_web_map_link(layer, role="baselayer"))
     # alternatively use default base layers defined
-    elif "default_base_layers" in catalog_config:
+    elif catalog_config.get("default_base_layers"):
         base_layers = read_config_file(catalog_config["default_base_layers"])
         for layer in base_layers:
             collection.add_link(create_web_map_link(layer, role="baselayer"))
     # add custom overlays just for this indicator
-    if "OverlayLayers" in collection_config:
+    if collection_config.get("OverlayLayers"):
         for layer in collection_config["OverlayLayers"]:
             collection.add_link(create_web_map_link(layer, role="overlay"))
     # check if default overlay layers defined
-    elif "default_overlay_layers" in catalog_config:
+    elif catalog_config.get("default_overlay_layers"):
         overlay_layers = read_config_file(catalog_config["default_overlay_layers"])
         for layer in overlay_layers:
             collection.add_link(create_web_map_link(layer, role="overlay"))
@@ -458,41 +495,41 @@ def add_base_overlay_info(
 def add_extra_fields(
     stac_object: Collection | Link, collection_config: dict, is_root_collection: bool = False
 ) -> None:
-    if "yAxis" in collection_config:
+    if collection_config.get("yAxis"):
         stac_object.extra_fields["yAxis"] = collection_config["yAxis"]
-    if "Themes" in collection_config:
+    if collection_config.get("Themes"):
         stac_object.extra_fields["themes"] = collection_config["Themes"]
     if (
-        "Locations" in collection_config or "Subcollections" in collection_config
+        collection_config.get("Locations") or collection_config.get("Subcollections")
     ) and is_root_collection:
         stac_object.extra_fields["locations"] = True
-    if "Tags" in collection_config:
+    if collection_config.get("Tags"):
         stac_object.extra_fields["tags"] = collection_config["Tags"]
-    if "Satellite" in collection_config:
+    if collection_config.get("Satellite"):
         stac_object.extra_fields["satellite"] = collection_config["Satellite"]
-    if "Sensor" in collection_config:
+    if collection_config.get("Sensor"):
         stac_object.extra_fields["sensor"] = collection_config["Sensor"]
-    if "Agency" in collection_config:
+    if collection_config.get("Agency"):
         stac_object.extra_fields["agency"] = collection_config["Agency"]
-    if "EodashIdentifier" in collection_config:
+    if collection_config.get("EodashIdentifier"):
         stac_object.extra_fields["subcode"] = collection_config["EodashIdentifier"]
-    if "CollectionGroup" in collection_config:
+    if collection_config.get("CollectionGroup"):
         stac_object.extra_fields["collection_group"] = collection_config["CollectionGroup"]
-    if "DataSource" in collection_config:
-        if "Spaceborne" in collection_config["DataSource"]:
-            if "Sensor" in collection_config["DataSource"]["Spaceborne"]:
+    if collection_config.get("DataSource"):
+        if collection_config["DataSource"].get("Spaceborne"):
+            if collection_config["DataSource"]["Spaceborne"].get("Sensor"):
                 stac_object.extra_fields["sensor"] = collection_config["DataSource"]["Spaceborne"][
                     "Sensor"
                 ]
-            if "Satellite" in collection_config["DataSource"]["Spaceborne"]:
+            if collection_config["DataSource"]["Spaceborne"].get("Satellite"):
                 stac_object.extra_fields["satellite"] = collection_config["DataSource"][
                     "Spaceborne"
                 ]["Satellite"]
-        if "InSitu" in collection_config["DataSource"]:
+        if collection_config["DataSource"].get("InSitu"):
             stac_object.extra_fields["insituSources"] = collection_config["DataSource"]["InSitu"]
-        if "Other" in collection_config["DataSource"]:
+        if collection_config["DataSource"].get("Other"):
             stac_object.extra_fields["otherSources"] = collection_config["DataSource"]["Other"]
-    if "MapProjection" in collection_config:
+    if collection_config.get("MapProjection"):
         stac_object.extra_fields["eodash:mapProjection"] = collection_config["MapProjection"]
 
 

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/src/eodash_catalog/thumbnails.py

@@ -31,7 +31,7 @@ def generate_thumbnail(
 ) -> None:
     if endpoint_config["Name"] == "Sentinel Hub" or endpoint_config["Name"] == "WMS":
         instanceId = os.getenv("SH_INSTANCE_ID")
-        if "InstanceId" in endpoint_config:
+        if endpoint_config.get("InstanceId"):
             instanceId = endpoint_config["InstanceId"]
         # Build example url
         wms_config = (

{eodash_catalog-0.1.12 → eodash_catalog-0.1.13}/src/eodash_catalog/utils.py

@@ -82,6 +82,7 @@ def retrieveExtentFromWCS(
         template = "An exception of type {0} occurred. Arguments:\n{1!r}"
         message = template.format(type(e).__name__, e.args)
         LOGGER.warn(message)
+        raise e
 
     bbox = [-180.0, -90.0, 180.0, 90.0]
     owsnmspc = "{http://www.opengis.net/ows/2.0}"
@@ -143,6 +144,7 @@ def retrieveExtentFromWMSWMTS(
         template = "An exception of type {0} occurred. Arguments:\n{1!r}"
         message = template.format(type(e).__name__, e.args)
         LOGGER.warn(message)
+        raise e
 
     bbox = [-180.0, -90.0, 180.0, 90.0]
     if service and service[layer].boundingBoxWGS84:
@@ -285,7 +287,7 @@ def iter_len_at_least(i, n: int) -> int:
 
 def generate_veda_cog_link(endpoint_config: dict, file_url: str | None) -> str:
     bidx = ""
-    if "Bidx" in endpoint_config:
+    if endpoint_config.get("Bidx"):
         # Check if an array was provided
         if hasattr(endpoint_config["Bidx"], "__len__"):
             for band in endpoint_config["Bidx"]:
@@ -294,25 +296,29 @@ def generate_veda_cog_link(endpoint_config: dict, file_url: str | None) -> str:
             bidx = "&bidx={}".format(endpoint_config["Bidx"])
 
     colormap = ""
-    if "Colormap" in endpoint_config:
+    if endpoint_config.get("Colormap"):
         colormap = "&colormap={}".format(endpoint_config["Colormap"])
         # TODO: For now we assume a already urlparsed colormap definition
         # it could be nice to allow a json and better convert it on the fly
         # colormap = "&colormap=%s"%(urllib.parse.quote(str(endpoint_config["Colormap"])))
 
+    Nodata = ""
+    if endpoint_config.get("Nodata"):
+        Nodata = "&nodata={}".format(endpoint_config["Nodata"])
+
     colormap_name = ""
-    if "ColormapName" in endpoint_config:
+    if endpoint_config.get("ColormapName"):
         colormap_name = "&colormap_name={}".format(endpoint_config["ColormapName"])
 
     rescale = ""
-    if "Rescale" in endpoint_config:
+    if endpoint_config.get("Rescale"):
         rescale = "&rescale={},{}".format(
             endpoint_config["Rescale"][0], endpoint_config["Rescale"][1]
         )
 
     file_url = f"url={file_url}&" if file_url else ""
 
-    target_url = f"https://openveda.cloud/api/raster/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{file_url}resampling_method=nearest{bidx}{colormap}{colormap_name}{rescale}"
+    target_url = f"https://openveda.cloud/api/raster/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{file_url}resampling_method=nearest{bidx}{colormap}{colormap_name}{rescale}{Nodata}"
     return target_url
 
 
@@ -458,3 +464,20 @@ def _load_file(filepath):
             return yaml.safe_load(content)
         except yaml.YAMLError as err:
             raise ValueError(f"Failed to parse '{filepath}' as JSON or YAML: {err}") from err
+
+
+def merge_bboxes(bboxes: list[list[float]]) -> list[float]:
+    """
+    Merge  bounding boxes into one bounding box that contains them all.
+    Returns:
+        A list representing the merged bbox: [min_lon, min_lat, max_lon, max_lat]
+    """
+    if not bboxes:
+        raise ValueError("No bounding boxes provided.")
+
+    min_lon = min(b[0] for b in bboxes)
+    min_lat = min(b[1] for b in bboxes)
+    max_lon = max(b[2] for b in bboxes)
+    max_lat = max(b[3] for b in bboxes)
+
+    return [min_lon, min_lat, max_lon, max_lat]