eodash_catalog 0.0.16__tar.gz → 0.0.18__tar.gz

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.16
+current_version = 0.0.18
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eodash_catalog
-Version: 0.0.16
+Version: 0.0.18
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/ruff.toml

@@ -5,4 +5,4 @@ target-version = "py312"
 
 [lint]
 select = ["E", "F", "UP", "B", "SIM", "I", "W", "C90", "ASYNC", "A", "C4", "PERF", "RUF"]
-ignore = ["B019", "C901"]
+ignore = ["B019", "C901", "UP017"]

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/src/eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.16"
+__version__ = "0.0.18"

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/src/eodash_catalog/endpoints.py

@@ -4,7 +4,7 @@ import os
 import sys
 import uuid
 from collections.abc import Callable
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from itertools import groupby
 from operator import itemgetter
 
@@ -27,6 +27,7 @@ from eodash_catalog.utils import (
     create_geojson_from_bbox,
     create_geojson_point,
     generate_veda_cog_link,
+    replace_with_env_variables,
     retrieveExtentFromWMSWMTS,
 )
 
@@ -293,7 +294,7 @@ def handle_collection_only(
         catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
     )
     times = get_collection_times_from_config(endpoint_config)
-    if len(times) > 0 and not endpoint_config.get("Disable_Items"):
+    if len(times) > 0:
         for t in times:
             item = Item(
                 id=t,
@@ -473,7 +474,7 @@ def handle_SH_endpoint(
     catalog: Catalog,
     options: Options,
 ) -> Collection:
-    token = get_SH_token()
+    token = get_SH_token(endpoint_config)
     headers = {"Authorization": f"Bearer {token}"}
     endpoint_config["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
     # Overwrite collection id with type, such as ZARR or BYOC
@@ -509,8 +510,21 @@ def handle_WMS_endpoint(
             version=endpoint_config.get("Version", "1.1.1"),
             wmts=wmts,
         )
+    # optionally filter time results
+    if query := endpoint_config.get("Query"):
+        datetime_query = [times[0], times[-1]]
+        if start := query.get("Start"):
+            datetime_query[0] = parser.isoparse(start).replace(tzinfo=timezone.utc)
+        if end := query.get("End"):
+            datetime_query[1] = parser.isoparse(end).replace(tzinfo=timezone.utc)
+        # filter times based on query Start/End
+        times = [
+            datetime_str
+            for datetime_str in times
+            if datetime_query[0] <= parser.isoparse(datetime_str) < datetime_query[1]
+        ]
     # Create an item per time to allow visualization in stac clients
-    if len(times) > 0 and not endpoint_config.get("Disable_Items"):
+    if len(times) > 0:
         for t in times:
             item = Item(
                 id=t,
@@ -567,6 +581,10 @@ def add_visualization_info(
         instanceId = os.getenv("SH_INSTANCE_ID")
         if "InstanceId" in endpoint_config:
             instanceId = endpoint_config["InstanceId"]
+        if env_id := endpoint_config.get("CustomSHEnvId"):
+            # special handling for custom environment
+            # (will take SH_INSTANCE_ID_{env_id}) as ENV VAR
+            instanceId = os.getenv(f"SH_INSTANCE_ID_{env_id}")
         extra_fields: dict[str, list[str] | dict[str, str]] = {
             "wms:layers": [endpoint_config["LayerId"]],
             "role": ["data"],
@@ -604,10 +622,13 @@ def add_visualization_info(
         media_type = "image/jpeg"
         if "MediaType" in endpoint_config:
             media_type = endpoint_config["MediaType"]
+        endpoint_url = endpoint_config["EndPoint"]
+        # custom replacing of all ENV VARS present as template in URL as {VAR}
+        endpoint_url = replace_with_env_variables(endpoint_url)
         stac_object.add_link(
             Link(
                 rel="wms",
-                target=endpoint_config["EndPoint"],
+                target=endpoint_url,
                 media_type=media_type,
                 title=collection_config["Name"],
                 extra_fields=extra_fields,

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/src/eodash_catalog/generate_indicators.py

@@ -88,6 +88,8 @@ def process_catalog_file(file_path: str, options: Options):
                     )
                 else:
                     print(f"Warning: neither collection nor indicator found for {collection}")
+        if "MapProjection" in catalog_config:
+            catalog.extra_fields["eodash:mapProjection"] = catalog_config["MapProjection"]
 
         strategy = TemplateLayoutStrategy(item_template="${collection}/${year}")
         # expecting that the catalog will be hosted online, self url should correspond to that
@@ -196,64 +198,64 @@ def process_collection_file(
     with open(file_path) as f:
         collection_config: dict = yaml.load(f, Loader=SafeLoader)
         if "Resources" in collection_config:
-            for resource in collection_config["Resources"]:
+            for endpoint_config in collection_config["Resources"]:
                 collection = None
-                if resource["Name"] == "Sentinel Hub":
+                if endpoint_config["Name"] == "Sentinel Hub":
                     collection = handle_SH_endpoint(
-                        catalog_config, resource, collection_config, catalog, options
+                        catalog_config, endpoint_config, collection_config, catalog, options
                     )
-                elif resource["Name"] == "Sentinel Hub WMS":
+                elif endpoint_config["Name"] == "Sentinel Hub WMS":
                     collection = handle_SH_WMS_endpoint(
-                        catalog_config, resource, collection_config, catalog
+                        catalog_config, endpoint_config, collection_config, catalog
                     )
-                elif resource["Name"] == "GeoDB":
+                elif endpoint_config["Name"] == "GeoDB":
                     collection = handle_GeoDB_endpoint(
-                        catalog_config, resource, collection_config, catalog
+                        catalog_config, endpoint_config, collection_config, catalog
                     )
-                elif resource["Name"] == "VEDA":
+                elif endpoint_config["Name"] == "VEDA":
                     collection = handle_VEDA_endpoint(
-                        catalog_config, resource, collection_config, catalog, options
+                        catalog_config, endpoint_config, collection_config, catalog, options
                     )
-                elif resource["Name"] == "marinedatastore":
+                elif endpoint_config["Name"] == "marinedatastore":
                     collection = handle_WMS_endpoint(
-                        catalog_config, resource, collection_config, catalog, wmts=True
+                        catalog_config, endpoint_config, collection_config, catalog, wmts=True
                     )
-                elif resource["Name"] == "xcube":
+                elif endpoint_config["Name"] == "xcube":
                     collection = handle_xcube_endpoint(
-                        catalog_config, resource, collection_config, catalog
+                        catalog_config, endpoint_config, collection_config, catalog
                     )
-                elif resource["Name"] == "WMS":
+                elif endpoint_config["Name"] == "WMS":
                     collection = handle_WMS_endpoint(
-                        catalog_config, resource, collection_config, catalog
+                        catalog_config, endpoint_config, collection_config, catalog
                     )
-                elif resource["Name"] == "JAXA_WMTS_PALSAR":
+                elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
                     # somewhat one off creation of individual WMTS layers as individual items
                     collection = handle_WMS_endpoint(
-                        catalog_config, resource, collection_config, catalog, wmts=True
+                        catalog_config, endpoint_config, collection_config, catalog, wmts=True
                     )
-                elif resource["Name"] == "Collection-only":
+                elif endpoint_config["Name"] == "Collection-only":
                     collection = handle_collection_only(
-                        catalog_config, resource, collection_config, catalog
+                        catalog_config, endpoint_config, collection_config, catalog
                     )
-                elif resource["Name"] == "Custom-Endpoint":
+                elif endpoint_config["Name"] == "Custom-Endpoint":
                     collection = handle_custom_endpoint(
                         catalog_config,
-                        resource,
+                        endpoint_config,
                         collection_config,
                         catalog,
                     )
-                elif resource["Name"] in ["COG source", "GeoJSON source"]:
+                elif endpoint_config["Name"] in ["COG source", "GeoJSON source"]:
                     collection = handle_raw_source(
-                        catalog_config, resource, collection_config, catalog
+                        catalog_config, endpoint_config, collection_config, catalog
                     )
                 else:
                     raise ValueError("Type of Resource is not supported")
                 if collection:
                     add_single_item_if_collection_empty(collection)
-                    add_projection_info(resource, collection)
-                    add_to_catalog(collection, catalog, resource, collection_config)
+                    add_projection_info(endpoint_config, collection)
+                    add_to_catalog(collection, catalog, endpoint_config, collection_config)
                 else:
-                    raise Exception(f"No collection was generated for resource {resource}")
+                    raise Exception(f"No collection was generated for resource {endpoint_config}")
         elif "Subcollections" in collection_config:
             # if no endpoint is specified we check for definition of subcollections
             parent_collection = get_or_create_collection(

eodash_catalog-0.0.18/src/eodash_catalog/sh_endpoint.py

@@ -0,0 +1,30 @@
+import os
+
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2Session
+
+SH_TOKEN_URL = "https://services.sentinel-hub.com/oauth/token"
+_token_cache: dict[str, str] = {}
+
+
+def get_SH_token(endpoint_config: dict) -> str:
+    # Your client credentials
+    client_id = os.getenv("SH_CLIENT_ID", "")
+    client_secret = os.getenv("SH_CLIENT_SECRET", "")
+    if env_id := endpoint_config.get("CustomSHEnvId"):
+        client_id = os.getenv(f"SH_CLIENT_ID_{env_id}", "")
+        client_secret = os.getenv(f"SH_CLIENT_SECRET_{env_id}", "")
+    if client_id in _token_cache:
+        return _token_cache[client_id]
+    # Create a session
+    client = BackendApplicationClient(client_id=client_id)
+    oauth = OAuth2Session(client=client)
+    # Get token for the session
+    token = oauth.fetch_token(
+        token_url=SH_TOKEN_URL,
+        client_secret=client_secret,
+    )
+    access_token = token["access_token"]
+    _token_cache[client_id] = access_token
+
+    return access_token

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/src/eodash_catalog/stac_handling.py

@@ -347,7 +347,7 @@ def add_base_overlay_info(
                 collection.add_link(create_web_map_link(layer, role="overlay"))
 
 
-def add_extra_fields(stac_object: Collection | Catalog | Link, collection_config: dict) -> None:
+def add_extra_fields(stac_object: Collection | Link, collection_config: dict) -> None:
     if "yAxis" in collection_config:
         stac_object.extra_fields["yAxis"] = collection_config["yAxis"]
     if "Themes" in collection_config:
@@ -379,7 +379,7 @@ def add_extra_fields(stac_object: Collection | Catalog | Link, collection_config
         if "Other" in collection_config["DataSource"]:
             stac_object.extra_fields["otherSources"] = collection_config["DataSource"]["Other"]
     if "MapProjection" in collection_config:
-        stac_object.extra_fields["mapProjection"] = collection_config["MapProjection"]
+        stac_object.extra_fields["eodash:mapProjection"] = collection_config["MapProjection"]
 
 
 def get_collection_times_from_config(endpoint_config: dict) -> list[str]:
@@ -416,6 +416,6 @@ def add_projection_info(
             # so we are taking over the DataProjection as is and deal with it in the eodash client
             # in a non-standard compliant way
             # https://github.com/proj4js/proj4js/issues/400
-            stac_object.extra_fields["proj4_def"] = proj
+            stac_object.extra_fields["eodash:proj4_def"] = proj
         else:
             raise Exception(f"Incorrect type of proj definition {proj}")

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/src/eodash_catalog/utils.py

@@ -1,3 +1,4 @@
+import os
 import re
 import threading
 import uuid
@@ -252,3 +253,18 @@ def add_single_item_if_collection_empty(collection: Collection) -> None:
             end_datetime=datetime.now(),
         )
         collection.add_item(item)
+
+
+def replace_with_env_variables(s: str) -> str:
+    # Define the regex pattern to find text within curly brackets
+    pattern = r"\{(\w+)\}"
+
+    # Define the replacement function
+    def replacer(match):
+        # Extract the variable name from the match
+        var_name = match.group(1)
+        # Get the environment variable value, if it doesn't exist, keep the original placeholder
+        return os.getenv(var_name, match.group(0))
+
+    # Use re.sub with the replacement function
+    return re.sub(pattern, replacer, s)

{eodash_catalog-0.0.16 → eodash_catalog-0.0.18}/tests/test_generate.py

@@ -100,7 +100,7 @@ def test_indicator_map_projection_added(catalog_output_folder):
     with open(os.path.join(root_collection_path, "collection.json")) as fp:
         indicator_json = json.load(fp)
         # test that collection has map projection defined
-        assert indicator_json["mapProjection"] == 3035
+        assert indicator_json["eodash:mapProjection"] == 3035
 
 
 def test_baselayers_and_overlays_added(catalog_output_folder):
@@ -184,4 +184,4 @@ def test_baselayer_with_custom_projection_added(catalog_output_folder):
         # overwrites default_baselayers, so there is just 1
         assert len(baselayer_links) == 1
         # test that custom proj4 definition is added to link
-        assert baselayer_links[0]["proj4_def"]["name"] == "ORTHO:680500"
+        assert baselayer_links[0]["eodash:proj4_def"]["name"] == "ORTHO:680500"

eodash_catalog-0.0.16/src/eodash_catalog/sh_endpoint.py

@@ -1,22 +0,0 @@
-import os
-
-from oauthlib.oauth2 import BackendApplicationClient
-from requests_oauthlib import OAuth2Session
-
-SH_TOKEN_URL = "https://services.sentinel-hub.com/oauth/token"
-
-
-def get_SH_token() -> str:
-    # Your client credentials
-    client_id = os.getenv("SH_CLIENT_ID")
-    client_secret = os.getenv("SH_CLIENT_SECRET")
-    # Create a session
-    client = BackendApplicationClient(client_id=client_id)
-    oauth = OAuth2Session(client=client)
-    # Get token for the session
-    token = oauth.fetch_token(
-        token_url=SH_TOKEN_URL,
-        client_secret=client_secret,
-    )
-
-    return token["access_token"]