eodash_catalog 0.0.24__tar.gz → 0.0.26__tar.gz

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.24
+current_version = 0.0.26
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/.github/workflows/ci.yml

@@ -17,7 +17,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
-      # You can test your matrix by printing the current Python version
       - name: Install dependencies
         run: python -m pip install -r requirements.txt
       - name: Lint

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eodash_catalog
-Version: 0.0.24
+Version: 0.0.26
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/src/eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.24"
+__version__ = "0.0.26"

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/src/eodash_catalog/endpoints.py

@@ -12,6 +12,7 @@ import requests
 from dateutil import parser
 from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries
 from pystac_client import Client
+from structlog import get_logger
 
 from eodash_catalog.sh_endpoint import get_SH_token
 from eodash_catalog.stac_handling import (
@@ -31,6 +32,8 @@ from eodash_catalog.utils import (
     retrieveExtentFromWMSWMTS,
 )
 
+LOGGER = get_logger(__name__)
+
 
 def process_STAC_Datacube_Endpoint(
     catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
@@ -774,7 +777,7 @@ def add_visualization_info(
             )
         )
     else:
-        print("Visualization endpoint not supported")
+        LOGGER.info(f"Visualization endpoint not supported {endpoint_config['Name']}")
 
 
 def handle_custom_endpoint(
@@ -797,7 +800,7 @@ def handle_custom_endpoint(
             importlib.import_module(module_name), func_name
         )
     except ModuleNotFoundError as e:
-        print(
+        LOGGER.warn(
             f"""function {func_name} from module {module_name} can not be imported.
             Check if you are specifying relative path inside the
             catalog repository or catalog generator repository."""
@@ -865,6 +868,8 @@ def handle_raw_source(
             link = collection.add_item(item)
             link.extra_fields["datetime"] = time_entry["Time"]
             link.extra_fields["assets"] = [a["File"] for a in time_entry["Assets"]]
+        # eodash v4 compatibility, adding last referenced style to collection
+        collection.add_link(style_link)
     add_collection_information(catalog_config, collection, collection_config)
     collection.update_extent_from_items()
     return collection

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/src/eodash_catalog/generate_indicators.py

@@ -14,6 +14,7 @@ from dotenv import load_dotenv
 from pystac import Catalog, CatalogType, Collection, Link, Summaries
 from pystac.layout import TemplateLayoutStrategy
 from pystac.validation import validate_all
+from structlog import get_logger
 from yaml.loader import SafeLoader
 
 from eodash_catalog.endpoints import (
@@ -40,14 +41,16 @@ from eodash_catalog.utils import (
     add_single_item_if_collection_empty,
     iter_len_at_least,
     recursive_save,
+    retry,
 )
 
 # make sure we are loading the env local definition
 load_dotenv()
+LOGGER = get_logger(__name__)
 
 
 def process_catalog_file(file_path: str, options: Options):
-    print("Processing catalog:", file_path)
+    LOGGER.info(f"Processing catalog: {file_path}")
     with open(file_path) as f:
         catalog_config: dict = yaml.load(f, Loader=SafeLoader)
 
@@ -62,7 +65,7 @@ def process_catalog_file(file_path: str, options: Options):
             # create full catalog
             process_collections = catalog_config["collections"]
         if len(process_collections) == 0:
-            print("No applicable collections found for catalog, skipping creation")
+            LOGGER.info("No applicable collections found for catalog, skipping creation")
             return
         catalog = Catalog(
             id=catalog_config["id"],
@@ -87,7 +90,7 @@ def process_catalog_file(file_path: str, options: Options):
                         options,
                     )
                 else:
-                    print(f"Warning: neither collection nor indicator found for {collection}")
+                    LOGGER.info(f"Warning: neither collection nor indicator found for {collection}")
         if "MapProjection" in catalog_config:
             catalog.extra_fields["eodash:mapProjection"] = catalog_config["MapProjection"]
 
@@ -95,7 +98,7 @@ def process_catalog_file(file_path: str, options: Options):
         # expecting that the catalog will be hosted online, self url should correspond to that
         # default to a local folder + catalog id in case not set
 
-        print("Started creation of collection files")
+        LOGGER.info("Started creation of collection files")
         start = time.time()
         if options.ni:
             catalog_self_href = f'{options.outputpath}/{catalog_config["id"]}'
@@ -109,15 +112,15 @@ def process_catalog_file(file_path: str, options: Options):
             catalog.normalize_hrefs(catalog_self_href, strategy=strategy)
             catalog.save(dest_href="{}/{}".format(options.outputpath, catalog_config["id"]))
         end = time.time()
-        print(f"Catalog {catalog_config['id']}: Time consumed in saving: {end - start}")
+        LOGGER.info(f"Catalog {catalog_config['id']}: Time consumed in saving: {end - start}")
 
         if options.vd:
             # try to validate catalog if flag was set
-            print(f"Running validation of catalog {file_path}")
+            LOGGER.info(f"Running validation of catalog {file_path}")
             try:
                 validate_all(catalog.to_dict(), href=catalog_config["endpoint"])
             except Exception as e:
-                print(f"Issue validation collection: {e}")
+                LOGGER.info(f"Issue validation collection: {e}")
 
 
 def extract_indicator_info(parent_collection: Collection):
@@ -161,7 +164,7 @@ def process_indicator_file(
     catalog_config: dict, file_path: str, catalog: Catalog, options: Options
 ):
     with open(file_path) as f:
-        print("Processing indicator:", file_path)
+        LOGGER.info(f"Processing indicator: {file_path}")
         indicator_config: dict = yaml.load(f, Loader=SafeLoader)
         parent_indicator = get_or_create_collection(
             catalog, indicator_config["Name"], indicator_config, catalog_config, {}
@@ -191,71 +194,79 @@ def process_indicator_file(
         add_to_catalog(parent_indicator, catalog, {}, indicator_config)
 
 
+@retry((Exception), tries=3, delay=5, backoff=2, logger=LOGGER)
 def process_collection_file(
     catalog_config: dict, file_path: str, catalog: Catalog | Collection, options: Options
 ):
-    print("Processing collection:", file_path)
+    LOGGER.info(f"Processing collection: {file_path}")
     with open(file_path) as f:
         collection_config: dict = yaml.load(f, Loader=SafeLoader)
         if "Resources" in collection_config:
             for endpoint_config in collection_config["Resources"]:
-                collection = None
-                if endpoint_config["Name"] == "Sentinel Hub":
-                    collection = handle_SH_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog, options
-                    )
-                elif endpoint_config["Name"] == "Sentinel Hub WMS":
-                    collection = handle_SH_WMS_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog
-                    )
-                elif endpoint_config["Name"] == "GeoDB":
-                    collection = handle_GeoDB_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog
-                    )
-                elif endpoint_config["Name"] == "VEDA":
-                    collection = handle_VEDA_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog, options
-                    )
-                elif endpoint_config["Name"] == "marinedatastore":
-                    collection = handle_WMS_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog, wmts=True
-                    )
-                elif endpoint_config["Name"] == "xcube":
-                    collection = handle_xcube_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog
-                    )
-                elif endpoint_config["Name"] == "WMS":
-                    collection = handle_WMS_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog
-                    )
-                elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
-                    # somewhat one off creation of individual WMTS layers as individual items
-                    collection = handle_WMS_endpoint(
-                        catalog_config, endpoint_config, collection_config, catalog, wmts=True
-                    )
-                elif endpoint_config["Name"] == "Collection-only":
-                    collection = handle_collection_only(
-                        catalog_config, endpoint_config, collection_config, catalog
-                    )
-                elif endpoint_config["Name"] == "Custom-Endpoint":
-                    collection = handle_custom_endpoint(
-                        catalog_config,
-                        endpoint_config,
-                        collection_config,
-                        catalog,
-                    )
-                elif endpoint_config["Name"] in ["COG source", "GeoJSON source"]:
-                    collection = handle_raw_source(
-                        catalog_config, endpoint_config, collection_config, catalog
-                    )
-                else:
-                    raise ValueError("Type of Resource is not supported")
-                if collection:
-                    add_single_item_if_collection_empty(collection)
-                    add_projection_info(endpoint_config, collection)
-                    add_to_catalog(collection, catalog, endpoint_config, collection_config)
-                else:
-                    raise Exception(f"No collection was generated for resource {endpoint_config}")
+                try:
+                    collection = None
+                    if endpoint_config["Name"] == "Sentinel Hub":
+                        collection = handle_SH_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog, options
+                        )
+                    elif endpoint_config["Name"] == "Sentinel Hub WMS":
+                        collection = handle_SH_WMS_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog
+                        )
+                    elif endpoint_config["Name"] == "GeoDB":
+                        collection = handle_GeoDB_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog
+                        )
+                    elif endpoint_config["Name"] == "VEDA":
+                        collection = handle_VEDA_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog, options
+                        )
+                    elif endpoint_config["Name"] == "marinedatastore":
+                        collection = handle_WMS_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog, wmts=True
+                        )
+                    elif endpoint_config["Name"] == "xcube":
+                        collection = handle_xcube_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog
+                        )
+                    elif endpoint_config["Name"] == "WMS":
+                        collection = handle_WMS_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog
+                        )
+                    elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
+                        # somewhat one off creation of individual WMTS layers as individual items
+                        collection = handle_WMS_endpoint(
+                            catalog_config, endpoint_config, collection_config, catalog, wmts=True
+                        )
+                    elif endpoint_config["Name"] == "Collection-only":
+                        collection = handle_collection_only(
+                            catalog_config, endpoint_config, collection_config, catalog
+                        )
+                    elif endpoint_config["Name"] == "Custom-Endpoint":
+                        collection = handle_custom_endpoint(
+                            catalog_config,
+                            endpoint_config,
+                            collection_config,
+                            catalog,
+                        )
+                    elif endpoint_config["Name"] in ["COG source", "GeoJSON source"]:
+                        collection = handle_raw_source(
+                            catalog_config, endpoint_config, collection_config, catalog
+                        )
+                    else:
+                        raise ValueError("Type of Resource is not supported")
+                    if collection:
+                        add_single_item_if_collection_empty(collection)
+                        add_projection_info(endpoint_config, collection)
+                        add_to_catalog(collection, catalog, endpoint_config, collection_config)
+                    else:
+                        raise Exception(
+                            f"No collection was generated for resource {endpoint_config}"
+                        )
+                except Exception as e:
+                    LOGGER.warn(f"""Exception: {e.args[0]} with config: {endpoint_config}""")
+                    raise e
+
         elif "Subcollections" in collection_config:
             # if no endpoint is specified we check for definition of subcollections
             parent_collection = get_or_create_collection(

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/src/eodash_catalog/stac_handling.py

@@ -15,10 +15,13 @@ from pystac import (
     SpatialExtent,
     TemporalExtent,
 )
+from structlog import get_logger
 from yaml.loader import SafeLoader
 
 from eodash_catalog.utils import generateDateIsostringsFromInterval
 
+LOGGER = get_logger(__name__)
+
 
 def get_or_create_collection(
     catalog: Catalog,
@@ -65,7 +68,7 @@ def get_or_create_collection(
                 if response.status_code == 200:
                     description = response.text
                 elif "Subtitle" in collection_config:
-                    print("WARNING: Markdown file could not be fetched")
+                    LOGGER.warn("Markdown file could not be fetched")
                     description = collection_config["Subtitle"]
             else:
                 # relative path to assets was given
@@ -73,7 +76,7 @@ def get_or_create_collection(
                 if response.status_code == 200:
                     description = response.text
                 elif "Subtitle" in collection_config:
-                    print("WARNING: Markdown file could not be fetched")
+                    LOGGER.warn("Markdown file could not be fetched")
                     description = collection_config["Subtitle"]
     elif "Subtitle" in collection_config:
         # Try to use at least subtitle to fill some information
@@ -242,10 +245,9 @@ def add_collection_information(
                         )
             else:
                 # fallback to proprietary
-                print("WARNING: License could not be parsed, falling back to proprietary")
+                LOGGER.warn("License could not be parsed, falling back to proprietary")
                 collection.license = "proprietary"
     else:
-        # print("WARNING: No license was provided, falling back to proprietary")
         pass
 
     if "Provider" in collection_config:
@@ -258,7 +260,7 @@ def add_collection_information(
                 for provider in collection_config["Provider"]
             ]
         except Exception:
-            print(f"WARNING: Issue creating provider information for collection: {collection.id}")
+            LOGGER.warn(f"Issue creating provider information for collection: {collection.id}")
 
     if "Citation" in collection_config:
         if "DOI" in collection_config["Citation"]:

{eodash_catalog-0.0.24 → eodash_catalog-0.0.26}/src/eodash_catalog/utils.py

@@ -1,12 +1,13 @@
 import os
 import re
 import threading
+import time
 import uuid
 from collections.abc import Iterator
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from decimal import Decimal
-from functools import reduce
+from functools import reduce, wraps
 from typing import Any
 
 from dateutil import parser
@@ -14,6 +15,7 @@ from owslib.wms import WebMapService
 from owslib.wmts import WebMapTileService
 from pystac import Catalog, Collection, Item, RelType
 from six import string_types
+from structlog import get_logger
 
 from eodash_catalog.duration import Duration
 
@@ -30,6 +32,8 @@ ISO8601_PERIOD_REGEX = re.compile(
 )
 # regular expression to parse ISO duartion strings.
 
+LOGGER = get_logger(__name__)
+
 
 def create_geojson_point(lon: int | float, lat: int | float) -> dict[str, Any]:
     point = {"type": "Point", "coordinates": [lon, lat]}
@@ -84,10 +88,10 @@ def retrieveExtentFromWMSWMTS(
             # get unique times
             times = reduce(lambda re, x: [*re, x] if x not in re else re, times, [])
     except Exception as e:
-        print("Issue extracting information from service capabilities")
+        LOGGER.warn("Issue extracting information from service capabilities")
         template = "An exception of type {0} occurred. Arguments:\n{1!r}"
         message = template.format(type(e).__name__, e.args)
-        print(message)
+        LOGGER.warn(message)
 
     bbox = [-180.0, -90.0, 180.0, 90.0]
     if service and service[layer].boundingBoxWGS84:
@@ -116,7 +120,6 @@ def parse_duration(datestring):
         if key not in ("separator", "sign"):
             if val is None:
                 groups[key] = "0n"
-            # print groups[key]
             if key in ("years", "months"):
                 groups[key] = Decimal(groups[key][:-1].replace(",", "."))
             else:
@@ -268,3 +271,39 @@ def replace_with_env_variables(s: str) -> str:
 
     # Use re.sub with the replacement function
     return re.sub(pattern, replacer, s)
+
+
+def retry(exceptions, tries=3, delay=2, backoff=1, logger=None):
+    """
+    Retry decorator for retrying exceptions.
+
+    :param exceptions: Exception or tuple of exceptions to catch.
+    :param tries: Number of attempts. Default is 3.
+    :param delay: Initial delay between attempts in seconds. Default is 2.
+    :param backoff: Multiplier applied to delay between attempts. Default is 1 (no backoff).
+    :param logger: Logger to use. If None, print. Default is None.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            _tries, _delay = tries, delay
+            while _tries > 0:
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    _tries -= 1
+                    if _tries == 0:
+                        raise
+                    else:
+                        msg = f"{e}, Try: {tries-_tries+1}/{tries}, retry in {_delay} seconds..."
+                        if logger:
+                            logger.warning(msg)
+                        else:
+                            print(msg)
+                        time.sleep(_delay)
+                        _delay *= backoff
+
+        return wrapper
+
+    return decorator