eodash_catalog 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.1.8"
+__version__ = "0.1.10"

eodash_catalog/endpoints.py

@@ -17,6 +17,7 @@ from eodash_catalog.sh_endpoint import get_SH_token
 from eodash_catalog.stac_handling import (
     add_collection_information,
     add_example_info,
+    add_process_info_child_collection,
     add_projection_info,
     get_collection_datetimes_from_config,
     get_or_create_collection,
@@ -174,7 +175,7 @@ def handle_STAC_based_endpoint(
                 collection.description = location["Name"]
             # TODO: should we remove all assets from sub collections?
             link = root_collection.add_child(collection)
-            latlng = f'{location["Point"][1]},{location["Point"][0]}'
+            latlng = f'{location["Point"][1]},{location["Point"][0]}'.strip()
             # Add extra properties we need
             link.extra_fields["id"] = location["Identifier"]
             link.extra_fields["latlng"] = latlng
@@ -182,6 +183,7 @@ def handle_STAC_based_endpoint(
             add_example_info(collection, collection_config, endpoint_config, catalog_config)
             # eodash v4 compatibility
             add_visualization_info(collection, collection_config, endpoint_config)
+            add_process_info_child_collection(collection, catalog_config, collection_config)
             if "OverwriteBBox" in location:
                 collection.extent.spatial = SpatialExtent(
                     [
@@ -208,7 +210,7 @@ def handle_STAC_based_endpoint(
         )
     # eodash v4 compatibility
     add_visualization_info(root_collection, collection_config, endpoint_config)
-    add_collection_information(catalog_config, root_collection, collection_config)
+    add_collection_information(catalog_config, root_collection, collection_config, True)
     add_example_info(root_collection, collection_config, endpoint_config, catalog_config)
     return root_collection
 
@@ -406,7 +408,7 @@ def handle_SH_WMS_endpoint(
 
             link = root_collection.add_child(collection)
             # bubble up information we want to the link
-            latlng = "{},{}".format(location["Point"][1], location["Point"][0])
+            latlng = "{},{}".format(location["Point"][1], location["Point"][0]).strip()
             link.extra_fields["id"] = location["Identifier"]
             link.extra_fields["latlng"] = latlng
             link.extra_fields["country"] = location["Country"]
@@ -416,6 +418,7 @@ def handle_SH_WMS_endpoint(
             else:
                 LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
             add_visualization_info(collection, collection_config, endpoint_config)
+            add_process_info_child_collection(collection, catalog_config, collection_config)
 
         root_collection.update_extent_from_items()
         # Add bbox extents from children
@@ -443,7 +446,7 @@ def handle_SH_WMS_endpoint(
             item_link = root_collection.add_item(item)
             item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
     # eodash v4 compatibility
-    add_collection_information(catalog_config, root_collection, collection_config)
+    add_collection_information(catalog_config, root_collection, collection_config, True)
     add_visualization_info(root_collection, collection_config, endpoint_config)
     return root_collection
 
@@ -528,7 +531,7 @@ def handle_GeoDB_endpoint(
             id=IdValue,
             bbox=bbox,
             properties={},
-            geometry=create_geojson_point(lon, lat),
+            geometry=create_geojson_point(lon, lat)["geometry"],
             datetime=None,
             start_datetime=min_date,
             end_datetime=max_date,
@@ -990,7 +993,7 @@ def handle_raw_source(
                 id=format_datetime_to_isostring_zulu(dt),
                 bbox=bbox,
                 properties={},
-                geometry=create_geojson_from_bbox(bbox),
+                geometry=create_geojson_from_bbox(bbox)["features"][0]["geometry"],
                 datetime=dt,
                 assets=assets,
                 extra_fields={},

eodash_catalog/generate_indicators.py

@@ -9,13 +9,11 @@ import time
 from typing import Any
 
 import click
-import yaml
 from dotenv import load_dotenv
 from pystac import Catalog, CatalogType, Collection, Link, Summaries
 from pystac.layout import TemplateLayoutStrategy
 from pystac.validation import validate_all
 from structlog import get_logger
-from yaml.loader import SafeLoader
 
 from eodash_catalog.endpoints import (
     handle_collection_only,
@@ -42,6 +40,7 @@ from eodash_catalog.utils import (
     RaisingThread,
     add_single_item_if_collection_empty,
     iter_len_at_least,
+    read_config_file,
     recursive_save,
     retry,
 )
@@ -53,76 +52,72 @@ LOGGER = get_logger(__name__)
 
 def process_catalog_file(file_path: str, options: Options):
     LOGGER.info(f"Processing catalog: {file_path}")
-    with open(file_path) as f:
-        catalog_config: dict = yaml.load(f, Loader=SafeLoader)
-
-        if len(options.collections) > 0:
-            # create only catalogs containing the passed collections
-            process_collections = [
-                c for c in catalog_config["collections"] if c in options.collections
-            ]
-        elif (len(options.collections) == 1 and options.collections == "all") or len(
-            options.collections
-        ) == 0:
-            # create full catalog
-            process_collections = catalog_config["collections"]
-        if len(process_collections) == 0:
-            LOGGER.info("No applicable collections found for catalog, skipping creation")
-            return
-        catalog = Catalog(
-            id=catalog_config["id"],
-            description=catalog_config["description"],
-            title=catalog_config["title"],
-            catalog_type=CatalogType.RELATIVE_PUBLISHED,
-        )
-        for collection in process_collections:
-            file_path = f"{options.collectionspath}/{collection}.yaml"
-            if os.path.isfile(file_path):
-                # if collection file exists process it as indicator
-                # collection will be added as single collection to indicator
-                process_indicator_file(catalog_config, file_path, catalog, options)
-            else:
-                # if not try to see if indicator definition available
-                file_path = f"{options.indicatorspath}/{collection}.yaml"
-                if os.path.isfile(file_path):
-                    process_indicator_file(
-                        catalog_config,
-                        f"{options.indicatorspath}/{collection}.yaml",
-                        catalog,
-                        options,
-                    )
-                else:
-                    LOGGER.info(f"Warning: neither collection nor indicator found for {collection}")
-        if "MapProjection" in catalog_config:
-            catalog.extra_fields["eodash:mapProjection"] = catalog_config["MapProjection"]
+    catalog_config: dict = read_config_file(file_path)
+    if len(options.collections) > 0:
+        # create only catalogs containing the passed collections
+        process_collections = [c for c in catalog_config["collections"] if c in options.collections]
+    elif (len(options.collections) == 1 and options.collections == "all") or len(
+        options.collections
+    ) == 0:
+        # create full catalog
+        process_collections = catalog_config["collections"]
+    if len(process_collections) == 0:
+        LOGGER.info("No applicable collections found for catalog, skipping creation")
+        return
+    catalog = Catalog(
+        id=catalog_config["id"],
+        description=catalog_config["description"],
+        title=catalog_config["title"],
+        catalog_type=CatalogType.RELATIVE_PUBLISHED,
+    )
+    for collection in process_collections:
+        file_path = f"{options.collectionspath}/{collection}"
+        try:
+            # if collection file exists process it as indicator
+            # collection will be added as single collection to indicator
+            process_indicator_file(catalog_config, file_path, catalog, options)
+        except FileNotFoundError:
+            # if not exists try to see if indicator definition available
+            file_path_indicator = f"{options.indicatorspath}/{collection}"
+            try:
+                process_indicator_file(
+                    catalog_config,
+                    file_path_indicator,
+                    catalog,
+                    options,
+                )
+            except FileNotFoundError:
+                LOGGER.info(f"Warning: neither collection nor indicator found for {collection}")
+    if "MapProjection" in catalog_config:
+        catalog.extra_fields["eodash:mapProjection"] = catalog_config["MapProjection"]
 
-        strategy = TemplateLayoutStrategy(item_template="${collection}/${year}")
-        # expecting that the catalog will be hosted online, self url should correspond to that
-        # default to a local folder + catalog id in case not set
+    strategy = TemplateLayoutStrategy(item_template="${collection}/${year}")
+    # expecting that the catalog will be hosted online, self url should correspond to that
+    # default to a local folder + catalog id in case not set
 
-        LOGGER.info("Started creation of collection files")
-        start = time.time()
-        if options.ni:
-            catalog_self_href = f'{options.outputpath}/{catalog_config["id"]}'
-            catalog.normalize_hrefs(catalog_self_href, strategy=strategy)
-            recursive_save(catalog, options.ni)
-        else:
-            # For full catalog save with items this still seems to be faster
-            catalog_self_href = catalog_config.get(
-                "endpoint", "{}/{}".format(options.outputpath, catalog_config["id"])
-            )
-            catalog.normalize_hrefs(catalog_self_href, strategy=strategy)
-            catalog.save(dest_href="{}/{}".format(options.outputpath, catalog_config["id"]))
-        end = time.time()
-        LOGGER.info(f"Catalog {catalog_config['id']}: Time consumed in saving: {end - start}")
+    LOGGER.info("Started creation of collection files")
+    start = time.time()
+    if options.ni or options.gp:
+        catalog_self_href = f'{options.outputpath}/{catalog_config["id"]}'
+        catalog.normalize_hrefs(catalog_self_href, strategy=strategy)
+        recursive_save(catalog, options.ni, options.gp)
+    else:
+        # For full catalog save with items this still seems to be faster
+        catalog_self_href = catalog_config.get(
+            "endpoint", "{}/{}".format(options.outputpath, catalog_config["id"])
+        )
+        catalog.normalize_hrefs(catalog_self_href, strategy=strategy)
+        catalog.save(dest_href="{}/{}".format(options.outputpath, catalog_config["id"]))
+    end = time.time()
+    LOGGER.info(f"Catalog {catalog_config['id']}: Time consumed in saving: {end - start}")
 
-        if options.vd:
-            # try to validate catalog if flag was set
-            LOGGER.info(f"Running validation of catalog {file_path}")
-            try:
-                validate_all(catalog.to_dict(), href=catalog_config["endpoint"])
-            except Exception as e:
-                LOGGER.info(f"Issue validation collection: {e}")
+    if options.vd:
+        # try to validate catalog if flag was set
+        LOGGER.info(f"Running validation of catalog {file_path}")
+        try:
+            validate_all(catalog.to_dict(), href=catalog_config["endpoint"])
+        except Exception as e:
+            LOGGER.info(f"Issue validation collection: {e}")
 
 
 def extract_indicator_info(parent_collection: Collection):
@@ -166,37 +161,36 @@ def extract_indicator_info(parent_collection: Collection):
 def process_indicator_file(
     catalog_config: dict, file_path: str, catalog: Catalog, options: Options
 ):
-    with open(file_path) as f:
-        LOGGER.info(f"Processing indicator: {file_path}")
-        indicator_config: dict = yaml.load(f, Loader=SafeLoader)
-        parent_indicator = get_or_create_collection(
-            catalog, indicator_config["Name"], indicator_config, catalog_config, {}
-        )
-        if "Collections" in indicator_config:
-            for collection in indicator_config["Collections"]:
-                process_collection_file(
-                    catalog_config,
-                    f"{options.collectionspath}/{collection}.yaml",
-                    parent_indicator,
-                    options,
-                    "Disable" in indicator_config and collection in indicator_config["Disable"],
-                )
-        else:
-            # we assume that collection files can also be loaded directly
-            process_collection_file(catalog_config, file_path, parent_indicator, options)
-        add_collection_information(catalog_config, parent_indicator, indicator_config)
-        if iter_len_at_least(parent_indicator.get_items(recursive=True), 1):
-            parent_indicator.update_extent_from_items()
-        # Add bbox extents from children
-        for c_child in parent_indicator.get_children():
-            if isinstance(c_child, Collection):  # typing reason
-                parent_indicator.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
-        # extract collection information and add it to summary indicator level
-        extract_indicator_info(parent_indicator)
-        add_process_info(parent_indicator, catalog_config, indicator_config)
-        # add baselayer and overview information to indicator collection
-        add_base_overlay_info(parent_indicator, catalog_config, indicator_config)
-        add_to_catalog(parent_indicator, catalog, {}, indicator_config)
+    LOGGER.info(f"Processing indicator: {file_path}")
+    indicator_config = read_config_file(file_path)
+    parent_indicator = get_or_create_collection(
+        catalog, indicator_config["Name"], indicator_config, catalog_config, {}
+    )
+    if "Collections" in indicator_config:
+        for collection in indicator_config["Collections"]:
+            process_collection_file(
+                catalog_config,
+                f"{options.collectionspath}/{collection}",
+                parent_indicator,
+                options,
+                "Disable" in indicator_config and collection in indicator_config["Disable"],
+            )
+    else:
+        # we assume that collection files can also be loaded directly
+        process_collection_file(catalog_config, file_path, parent_indicator, options)
+    add_collection_information(catalog_config, parent_indicator, indicator_config, True)
+    if iter_len_at_least(parent_indicator.get_items(recursive=True), 1):
+        parent_indicator.update_extent_from_items()
+    # Add bbox extents from children
+    for c_child in parent_indicator.get_children():
+        if isinstance(c_child, Collection):  # typing reason
+            parent_indicator.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
+    # extract collection information and add it to summary indicator level
+    extract_indicator_info(parent_indicator)
+    add_process_info(parent_indicator, catalog_config, indicator_config)
+    # add baselayer and overview information to indicator collection
+    add_base_overlay_info(parent_indicator, catalog_config, indicator_config)
+    add_to_catalog(parent_indicator, catalog, {}, indicator_config)
 
 
 @retry((Exception), tries=3, delay=5, backoff=2, logger=LOGGER)
@@ -208,164 +202,159 @@ def process_collection_file(
     disable=False,
 ):
     LOGGER.info(f"Processing collection: {file_path}")
-    with open(file_path) as f:
-        collection_config: dict = yaml.load(f, Loader=SafeLoader)
-        if "Resources" in collection_config:
-            for endpoint_config in collection_config["Resources"]:
-                try:
-                    collection = None
-                    if endpoint_config["Name"] == "Sentinel Hub":
-                        collection = handle_SH_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog, options
-                        )
-                    elif endpoint_config["Name"] == "Sentinel Hub WMS":
-                        collection = handle_SH_WMS_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    elif endpoint_config["Name"] == "GeoDB":
-                        collection = handle_GeoDB_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    elif endpoint_config["Name"] == "VEDA":
-                        collection = handle_VEDA_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog, options
-                        )
-                    elif endpoint_config["Name"] == "marinedatastore":
-                        collection = handle_WMS_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog, wmts=True
-                        )
-                    elif endpoint_config["Name"] == "xcube":
-                        collection = handle_xcube_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    elif endpoint_config["Name"] == "rasdaman":
-                        collection = handle_rasdaman_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    elif endpoint_config["Name"] == "WMS":
-                        collection = handle_WMS_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
-                        # somewhat one off creation of individual WMTS layers as individual items
-                        collection = handle_WMS_endpoint(
-                            catalog_config, endpoint_config, collection_config, catalog, wmts=True
-                        )
-                    elif endpoint_config["Name"] == "Collection-only":
-                        collection = handle_collection_only(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    elif endpoint_config["Name"] == "Custom-Endpoint":
-                        collection = handle_custom_endpoint(
-                            catalog_config,
-                            endpoint_config,
-                            collection_config,
-                            catalog,
-                        )
-                    elif endpoint_config["Name"] in [
-                        "COG source",
-                        "GeoJSON source",
-                        "FlatGeobuf source",
-                    ]:
-                        collection = handle_raw_source(
-                            catalog_config, endpoint_config, collection_config, catalog
-                        )
-                    else:
-                        raise ValueError("Type of Resource is not supported")
-                    if collection:
-                        add_single_item_if_collection_empty(collection)
-                        add_projection_info(endpoint_config, collection)
-                        add_to_catalog(
-                            collection, catalog, endpoint_config, collection_config, disable
-                        )
-                    else:
-                        raise Exception(
-                            f"No collection was generated for resource {endpoint_config}"
-                        )
-                except Exception as e:
-                    LOGGER.warn(f"""Exception: {e.args[0]} with config: {endpoint_config}""")
-                    raise e
-
-        elif "Subcollections" in collection_config:
-            # if no endpoint is specified we check for definition of subcollections
-            parent_collection = get_or_create_collection(
-                catalog, collection_config["Name"], collection_config, catalog_config, {}
-            )
-
-            locations = []
-            countries = []
-            for sub_coll_def in collection_config["Subcollections"]:
-                # Subcollection has only data on one location which
-                # is defined for the entire collection
-                if "Name" in sub_coll_def and "Point" in sub_coll_def:
-                    locations.append(sub_coll_def["Name"])
-                    if isinstance(sub_coll_def["Country"], list):
-                        countries.extend(sub_coll_def["Country"])
-                    else:
-                        countries.append(sub_coll_def["Country"])
-                    process_collection_file(
-                        catalog_config,
-                        "{}/{}.yaml".format(options.collectionspath, sub_coll_def["Collection"]),
-                        parent_collection,
-                        options,
+    collection_config = read_config_file(file_path)
+    if "Resources" in collection_config:
+        for endpoint_config in collection_config["Resources"]:
+            try:
+                collection = None
+                if endpoint_config["Name"] == "Sentinel Hub":
+                    collection = handle_SH_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog, options
                     )
-                    # find link in parent collection to update metadata
-                    for link in parent_collection.links:
-                        if (
-                            link.rel == "child"
-                            and "id" in link.extra_fields
-                            and link.extra_fields["id"] == sub_coll_def["Identifier"]
-                        ):
-                            latlng = "{},{}".format(
-                                sub_coll_def["Point"][1],
-                                sub_coll_def["Point"][0],
-                            )
-                            link.extra_fields["id"] = sub_coll_def["Identifier"]
-                            link.extra_fields["latlng"] = latlng
-                            link.extra_fields["name"] = sub_coll_def["Name"]
-                    # Update title of collection to use location name
-                    sub_collection = parent_collection.get_child(id=sub_coll_def["Identifier"])
-                    if sub_collection:
-                        sub_collection.title = sub_coll_def["Name"]
-                # The subcollection has multiple locations which need to be extracted
-                # and elevated to parent collection level
-                else:
-                    # create temp catalog to save collection
-                    tmp_catalog = Catalog(id="tmp_catalog", description="temp catalog placeholder")
-                    process_collection_file(
+                elif endpoint_config["Name"] == "Sentinel Hub WMS":
+                    collection = handle_SH_WMS_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                elif endpoint_config["Name"] == "GeoDB":
+                    collection = handle_GeoDB_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                elif endpoint_config["Name"] == "VEDA":
+                    collection = handle_VEDA_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog, options
+                    )
+                elif endpoint_config["Name"] == "marinedatastore":
+                    collection = handle_WMS_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog, wmts=True
+                    )
+                elif endpoint_config["Name"] == "xcube":
+                    collection = handle_xcube_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                elif endpoint_config["Name"] == "rasdaman":
+                    collection = handle_rasdaman_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                elif endpoint_config["Name"] == "WMS":
+                    collection = handle_WMS_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
+                    # somewhat one off creation of individual WMTS layers as individual items
+                    collection = handle_WMS_endpoint(
+                        catalog_config, endpoint_config, collection_config, catalog, wmts=True
+                    )
+                elif endpoint_config["Name"] == "Collection-only":
+                    collection = handle_collection_only(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                elif endpoint_config["Name"] == "Custom-Endpoint":
+                    collection = handle_custom_endpoint(
                         catalog_config,
-                        "{}/{}.yaml".format(options.collectionspath, sub_coll_def["Collection"]),
-                        tmp_catalog,
-                        options,
+                        endpoint_config,
+                        collection_config,
+                        catalog,
                     )
-                    links = tmp_catalog.get_child(sub_coll_def["Identifier"]).get_links()  # type: ignore
-                    for link in links:
-                        # extract summary information
-                        if "city" in link.extra_fields:
-                            locations.append(link.extra_fields["city"])
-                        if "country" in link.extra_fields:
-                            if isinstance(link.extra_fields["country"], list):
-                                countries.extend(link.extra_fields["country"])
-                            else:
-                                countries.append(link.extra_fields["country"])
+                elif endpoint_config["Name"] in [
+                    "COG source",
+                    "GeoJSON source",
+                    "FlatGeobuf source",
+                ]:
+                    collection = handle_raw_source(
+                        catalog_config, endpoint_config, collection_config, catalog
+                    )
+                else:
+                    raise ValueError("Type of Resource is not supported")
+                if collection:
+                    add_single_item_if_collection_empty(collection)
+                    add_projection_info(endpoint_config, collection)
+                    add_to_catalog(collection, catalog, endpoint_config, collection_config, disable)
+                else:
+                    raise Exception(f"No collection was generated for resource {endpoint_config}")
+            except Exception as e:
+                LOGGER.warn(f"""Exception: {e.args[0]} with config: {endpoint_config}""")
+                raise e
 
-                    parent_collection.add_links(links)
+    elif "Subcollections" in collection_config:
+        # if no endpoint is specified we check for definition of subcollections
+        parent_collection = get_or_create_collection(
+            catalog, collection_config["Name"], collection_config, catalog_config, {}
+        )
 
-            add_collection_information(catalog_config, parent_collection, collection_config)
-            add_process_info(catalog_config, parent_collection, collection_config)
-            parent_collection.update_extent_from_items()
-            # Add bbox extents from children
-            for c_child in parent_collection.get_children():
-                if isinstance(c_child, Collection):
-                    parent_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
-            # Fill summaries for locations
-            parent_collection.summaries = Summaries(
-                {
-                    "cities": list(set(locations)),
-                    "countries": list(set(countries)),
-                }
-            )
-            add_to_catalog(parent_collection, catalog, {}, collection_config)
+        locations = []
+        countries = []
+        for sub_coll_def in collection_config["Subcollections"]:
+            # Subcollection has only data on one location which
+            # is defined for the entire collection
+            if "Name" in sub_coll_def and "Point" in sub_coll_def:
+                locations.append(sub_coll_def["Name"])
+                if isinstance(sub_coll_def["Country"], list):
+                    countries.extend(sub_coll_def["Country"])
+                else:
+                    countries.append(sub_coll_def["Country"])
+                process_collection_file(
+                    catalog_config,
+                    "{}/{}".format(options.collectionspath, sub_coll_def["Collection"]),
+                    parent_collection,
+                    options,
+                )
+                # find link in parent collection to update metadata
+                for link in parent_collection.links:
+                    if (
+                        link.rel == "child"
+                        and "id" in link.extra_fields
+                        and link.extra_fields["id"] == sub_coll_def["Identifier"]
+                    ):
+                        latlng = "{},{}".format(
+                            sub_coll_def["Point"][1],
+                            sub_coll_def["Point"][0],
+                        )
+                        link.extra_fields["id"] = sub_coll_def["Identifier"]
+                        link.extra_fields["latlng"] = latlng
+                        link.extra_fields["name"] = sub_coll_def["Name"]
+                # Update title of collection to use location name
+                sub_collection = parent_collection.get_child(id=sub_coll_def["Identifier"])
+                if sub_collection:
+                    sub_collection.title = sub_coll_def["Name"]
+            # The subcollection has multiple locations which need to be extracted
+            # and elevated to parent collection level
+            else:
+                # create temp catalog to save collection
+                tmp_catalog = Catalog(id="tmp_catalog", description="temp catalog placeholder")
+                process_collection_file(
+                    catalog_config,
+                    "{}/{}".format(options.collectionspath, sub_coll_def["Collection"]),
+                    tmp_catalog,
+                    options,
+                )
+                links = tmp_catalog.get_child(sub_coll_def["Identifier"]).get_links()  # type: ignore
+                for link in links:
+                    # extract summary information
+                    if "city" in link.extra_fields:
+                        locations.append(link.extra_fields["city"])
+                    if "country" in link.extra_fields:
+                        if isinstance(link.extra_fields["country"], list):
+                            countries.extend(link.extra_fields["country"])
+                        else:
+                            countries.append(link.extra_fields["country"])
+
+                parent_collection.add_links(links)
+
+        add_collection_information(catalog_config, parent_collection, collection_config)
+        add_process_info(parent_collection, catalog_config, collection_config)
+        parent_collection.update_extent_from_items()
+        # Add bbox extents from children
+        for c_child in parent_collection.get_children():
+            if isinstance(c_child, Collection):
+                parent_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
+        # Fill summaries for locations
+        parent_collection.summaries = Summaries(
+            {
+                "cities": list(set(locations)),
+                "countries": list(set(countries)),
+            }
+        )
+        add_to_catalog(parent_collection, catalog, {}, collection_config)
 
 
 def add_to_catalog(
@@ -451,6 +440,11 @@ def add_to_catalog(
     is_flag=True,
     help="generate additionally thumbnail image for supported collections",
 )
+@click.option(
+    "-gp",
+    is_flag=True,
+    help="generates the items in .parquet format",
+)
 @click.argument(
     "collections",
     nargs=-1,
@@ -464,6 +458,7 @@ def process_catalogs(
     vd,
     ni,
     tn,
+    gp,
     collections,
 ):
     """STAC generator and harvester:
@@ -478,6 +473,7 @@ def process_catalogs(
         ni=ni,
         tn=tn,
         collections=collections,
+        gp=gp,
     )
     tasks = []
     for file_name in os.listdir(catalogspath):

eodash_catalog/stac_handling.py

@@ -2,7 +2,6 @@ from datetime import datetime
 
 import requests
 import spdx_lookup as lookup
-import yaml
 from pystac import (
     Asset,
     Catalog,
@@ -15,12 +14,12 @@ from pystac import (
     TemporalExtent,
 )
 from structlog import get_logger
-from yaml.loader import SafeLoader
 
 from eodash_catalog.utils import (
     generateDatetimesFromInterval,
     get_full_url,
     parse_datestring_to_tz_aware_datetime,
+    read_config_file,
 )
 
 LOGGER = get_logger(__name__)
@@ -214,7 +213,10 @@ def add_example_info(
 
 
 def add_collection_information(
-    catalog_config: dict, collection: Collection, collection_config: dict
+    catalog_config: dict,
+    collection: Collection,
+    collection_config: dict,
+    is_root_collection: bool = False,
 ) -> None:
     # Add metadata information
     # Check license identifier
@@ -323,7 +325,7 @@ def add_collection_information(
             f'{catalog_config["assets_endpoint"]}/' f'{collection_config["Image"]}'
         )
     # Add extra fields to collection if available
-    add_extra_fields(collection, collection_config)
+    add_extra_fields(collection, collection_config, is_root_collection)
 
     if "References" in collection_config:
         generic_counter = 1
@@ -347,7 +349,29 @@ def add_collection_information(
 
 
 def add_process_info(collection: Collection, catalog_config: dict, collection_config: dict) -> None:
-    if "Process" in collection_config:
+    if any(key in collection_config for key in ["Locations", "Subcollections"]):
+        # add the generic geodb-like selection process on the root collection instead of Processes
+        if "geodb_default_form" in catalog_config:
+            # adding default geodb-like map handling for Locations
+            collection.extra_fields["eodash:jsonform"] = get_full_url(
+                catalog_config["geodb_default_form"], catalog_config
+            )
+        # link a process definition for getting a collection with {{feature}} placeholder
+        sl = Link(
+            rel="service",
+            target="./" + collection.id + "/{{feature}}/collection.json",
+            media_type="application/json; profile=collection",
+            extra_fields={
+                "id": "locations",
+                "method": "GET",
+                "type": "application/json; profile=collection",
+                "endpoint": "STAC",
+            },
+        )
+        collection.add_link(sl)
+    # elif is intentional for cases when Process is defined on collection with Locations
+    # then we want to only add it to the "children", not the root
+    elif "Process" in collection_config:
         if "EndPoints" in collection_config["Process"]:
             for endpoint in collection_config["Process"]["EndPoints"]:
                 collection.add_link(create_service_link(endpoint, catalog_config))
@@ -390,6 +414,24 @@ def add_process_info(collection: Collection, catalog_config: dict, collection_co
                 )
 
 
+def add_process_info_child_collection(
+    collection: Collection, catalog_config: dict, collection_config: dict
+) -> None:
+    # in case of locations, we add the process itself on a child collection
+    if "Process" in collection_config:
+        if "EndPoints" in collection_config["Process"]:
+            for endpoint in collection_config["Process"]["EndPoints"]:
+                collection.add_link(create_service_link(endpoint, catalog_config))
+        if "JsonForm" in collection_config["Process"]:
+            collection.extra_fields["eodash:jsonform"] = get_full_url(
+                collection_config["Process"]["JsonForm"], catalog_config
+            )
+        if "VegaDefinition" in collection_config["Process"]:
+            collection.extra_fields["eodash:vegadefinition"] = get_full_url(
+                collection_config["Process"]["VegaDefinition"], catalog_config
+            )
+
+
 def add_base_overlay_info(
     collection: Collection, catalog_config: dict, collection_config: dict
 ) -> None:
@@ -399,28 +441,30 @@ def add_base_overlay_info(
             collection.add_link(create_web_map_link(layer, role="baselayer"))
     # alternatively use default base layers defined
     elif "default_base_layers" in catalog_config:
-        with open(f'{catalog_config["default_base_layers"]}.yaml') as f:
-            base_layers = yaml.load(f, Loader=SafeLoader)
-            for layer in base_layers:
-                collection.add_link(create_web_map_link(layer, role="baselayer"))
+        base_layers = read_config_file(catalog_config["default_base_layers"])
+        for layer in base_layers:
+            collection.add_link(create_web_map_link(layer, role="baselayer"))
     # add custom overlays just for this indicator
     if "OverlayLayers" in collection_config:
         for layer in collection_config["OverlayLayers"]:
             collection.add_link(create_web_map_link(layer, role="overlay"))
     # check if default overlay layers defined
     elif "default_overlay_layers" in catalog_config:
-        with open("{}.yaml".format(catalog_config["default_overlay_layers"])) as f:
-            overlay_layers = yaml.load(f, Loader=SafeLoader)
-            for layer in overlay_layers:
-                collection.add_link(create_web_map_link(layer, role="overlay"))
+        overlay_layers = read_config_file(catalog_config["default_overlay_layers"])
+        for layer in overlay_layers:
+            collection.add_link(create_web_map_link(layer, role="overlay"))
 
 
-def add_extra_fields(stac_object: Collection | Link, collection_config: dict) -> None:
+def add_extra_fields(
+    stac_object: Collection | Link, collection_config: dict, is_root_collection: bool = False
+) -> None:
     if "yAxis" in collection_config:
         stac_object.extra_fields["yAxis"] = collection_config["yAxis"]
     if "Themes" in collection_config:
         stac_object.extra_fields["themes"] = collection_config["Themes"]
-    if "Locations" in collection_config or "Subcollections" in collection_config:
+    if (
+        "Locations" in collection_config or "Subcollections" in collection_config
+    ) and is_root_collection:
         stac_object.extra_fields["locations"] = True
     if "Tags" in collection_config:
         stac_object.extra_fields["tags"] = collection_config["Tags"]

eodash_catalog/utils.py

@@ -1,3 +1,4 @@
+import json
 import os
 import re
 import threading
@@ -10,11 +11,13 @@ from decimal import Decimal
 from functools import reduce, wraps
 from typing import Any
 
+import stac_geoparquet as stacgp
+import yaml
 from dateutil import parser
 from owslib.wcs import WebCoverageService
 from owslib.wms import WebMapService
 from owslib.wmts import WebMapTileService
-from pystac import Catalog, Collection, Item, RelType
+from pystac import Asset, Catalog, Collection, Item, Link, RelType
 from pytz import timezone as pytztimezone
 from six import string_types
 from structlog import get_logger
@@ -233,14 +236,47 @@ class RaisingThread(threading.Thread):
             raise self._exc
 
 
-def recursive_save(stac_object: Catalog, no_items: bool = False) -> None:
-    stac_object.save_object()
+def recursive_save(stac_object: Catalog, no_items: bool = False, geo_parquet: bool = False) -> None:
     for child in stac_object.get_children():
-        recursive_save(child, no_items)
+        recursive_save(child, no_items, geo_parquet)
     if not no_items:
-        # try to save items if available
-        for item in stac_object.get_items():
-            item.save_object()
+        if geo_parquet:
+            create_geoparquet_items(stac_object)
+        else:
+            for item in stac_object.get_items():
+                item.save_object()
+    stac_object.save_object()
+
+
+def create_geoparquet_items(stacObject: Catalog):
+    if iter_len_at_least(stacObject.get_items(), 1):
+        stac_dir_arr = stacObject.self_href.split("/")
+        stac_dir_arr.pop()
+        stac_dir_path = "/".join(stac_dir_arr)
+        items_stacgp_path = f"{stac_dir_path}/items.parquet"
+        to_stac_geoparquet(stacObject, items_stacgp_path)
+        gp_link = Link(
+            rel="items",
+            target=items_stacgp_path,
+            media_type="application/vnd.apache.parquet",
+            title="GeoParquet Items",
+        )
+        stacObject.clear_links(rel="item")
+        stacObject.add_links([gp_link])
+
+
+def to_stac_geoparquet(stacObject: Catalog, path: str):
+    items = []
+    for item in stacObject.get_items():
+        if not item.geometry:
+            item.geometry = create_geojson_point(0, 0)["geometry"]
+        if not item.assets:
+            item.assets = {"dummy_asset": Asset(href="")}
+        items.append(item.to_dict())
+    record_batch_reader = stacgp.arrow.parse_stac_items_to_arrow(items)
+    table = record_batch_reader.read_all()
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    stacgp.arrow.to_parquet(table, path)
 
 
 def iter_len_at_least(i, n: int) -> int:
@@ -289,6 +325,7 @@ class Options:
     vd: bool
     ni: bool
     tn: bool
+    gp: bool
     collections: list[str]
 
 
@@ -392,3 +429,32 @@ def get_full_url(url: str, catalog_config) -> str:
         return url
     else:
         return f'{catalog_config["assets_endpoint"]}{url}'
+
+
+def read_config_file(path: str) -> dict:
+    # If the given path exists directly, use it
+    if os.path.exists(path):
+        return _load_file(path)
+
+    # Otherwise, try appending supported suffixes
+    for suffix in [".json", ".yaml", ".yml", ".JSON", ".YAML", ".YML"]:
+        candidate = path + suffix
+        if os.path.exists(candidate):
+            return _load_file(candidate)
+
+    raise FileNotFoundError(
+        f"No file found for '{path}' with or without supported suffixes (.json/.yaml/.yml)"
+    )
+
+
+def _load_file(filepath):
+    with open(filepath) as file:
+        content = file.read()
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            pass
+        try:
+            return yaml.safe_load(content)
+        except yaml.YAMLError as err:
+            raise ValueError(f"Failed to parse '{filepath}' as JSON or YAML: {err}") from err

{eodash_catalog-0.1.8.dist-info → eodash_catalog-0.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eodash_catalog
-Version: 0.1.8
+Version: 0.1.10
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
@@ -33,6 +33,7 @@ Requires-Dist: requests-oauthlib<1.3.2
 Requires-Dist: requests<3
 Requires-Dist: setuptools<71
 Requires-Dist: spdx-lookup<=0.3.3
+Requires-Dist: stac-geoparquet<=0.6.0
 Requires-Dist: structlog<22.0
 Requires-Dist: swiftspec==0.0.2
 Provides-Extra: dev

eodash_catalog-0.1.10.dist-info/RECORD

@@ -0,0 +1,14 @@
+eodash_catalog/__about__.py,sha256=tt63_uEHezTdcfiE37lvE0cZyuSRlO26qsS2Zzcj3-4,138
+eodash_catalog/__init__.py,sha256=_W_9emPYf6FUqc0P8L2SmADx6hGSd7PlQV3yRmCk5uM,115
+eodash_catalog/duration.py,sha256=B6XOZfvNU7SuqpxuVtT1kNKODoOQJXDI6mocvA_U1ik,10816
+eodash_catalog/endpoints.py,sha256=fQ3kaDCcdPypxrZzvAtWYqSt5nVq8cbCDOfH6BlbIc8,41835
+eodash_catalog/generate_indicators.py,sha256=V2a6r3lS_BoZvwoywHFaT-gBu3INRp4E4zLZioD6ang,19493
+eodash_catalog/sh_endpoint.py,sha256=XjZsZJ5jfJZLQenSTqUhiUZ5YAu9M9nv2KL1Qv3Be-I,1219
+eodash_catalog/stac_handling.py,sha256=TuwJ6AOA-1aJglAFtWZmtimZO5_wvIkpU64XHcKI7rk,23254
+eodash_catalog/thumbnails.py,sha256=qZDcpQe80ki6lEMKYdZtSnnHH0PUpcoXTvU9bYdPlzU,2260
+eodash_catalog/utils.py,sha256=zZ4PT1h1lWCMvEIfg5Xlw4re86pwR40BX8jEDweiTJ0,16115
+eodash_catalog-0.1.10.dist-info/METADATA,sha256=40cVmtNiUbJbofbuUDh1-I4rLThX7qO_D7BqW3STcnE,3234
+eodash_catalog-0.1.10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+eodash_catalog-0.1.10.dist-info/entry_points.txt,sha256=kuUQrDG1PtYd8kPjf5XM6H_NtQd9Ozwl0jjiGtAvZSM,87
+eodash_catalog-0.1.10.dist-info/licenses/LICENSE.txt,sha256=oJCW5zQxnFD-J0hGz6Zh5Lkpdk1oAndmWhseTmV224E,1107
+eodash_catalog-0.1.10.dist-info/RECORD,,

eodash_catalog-0.1.8.dist-info/RECORD

@@ -1,14 +0,0 @@
-eodash_catalog/__about__.py,sha256=Suv8M7NmJ_IR_KeM80feexn6iHCp66qCdMfm0fV59ac,137
-eodash_catalog/__init__.py,sha256=_W_9emPYf6FUqc0P8L2SmADx6hGSd7PlQV3yRmCk5uM,115
-eodash_catalog/duration.py,sha256=B6XOZfvNU7SuqpxuVtT1kNKODoOQJXDI6mocvA_U1ik,10816
-eodash_catalog/endpoints.py,sha256=b0vIQOLGqANm7qaaAap0IeW11yPzjo6-VXqcKCY41ag,41543
-eodash_catalog/generate_indicators.py,sha256=jqPns3KOZA8GEcxGYD6NcYa5iGtjKcrKkW7LLv13PRQ,20634
-eodash_catalog/sh_endpoint.py,sha256=XjZsZJ5jfJZLQenSTqUhiUZ5YAu9M9nv2KL1Qv3Be-I,1219
-eodash_catalog/stac_handling.py,sha256=8_W45SIXV4jAUa9J-f2KZ2eor6MNfPfy0n1d-932eAE,21234
-eodash_catalog/thumbnails.py,sha256=qZDcpQe80ki6lEMKYdZtSnnHH0PUpcoXTvU9bYdPlzU,2260
-eodash_catalog/utils.py,sha256=QhZeMXRC1uvE3VzC3T_nMi9hn_3RR05s5rkzs8tCeQc,13866
-eodash_catalog-0.1.8.dist-info/METADATA,sha256=7ETkKNNhQX2F6eH6scnpd0gBfS-VWUBkG7piWHQfmJU,3195
-eodash_catalog-0.1.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-eodash_catalog-0.1.8.dist-info/entry_points.txt,sha256=kuUQrDG1PtYd8kPjf5XM6H_NtQd9Ozwl0jjiGtAvZSM,87
-eodash_catalog-0.1.8.dist-info/licenses/LICENSE.txt,sha256=oJCW5zQxnFD-J0hGz6Zh5Lkpdk1oAndmWhseTmV224E,1107
-eodash_catalog-0.1.8.dist-info/RECORD,,