eodash_catalog 0.0.10__py3-none-any.whl → 0.0.12__py3-none-any.whl

eodash_catalog/endpoints.py

@@ -1,51 +1,62 @@
+import importlib
 import json
 import os
+import sys
+import uuid
+from collections.abc import Callable
 from datetime import datetime, timedelta
 from itertools import groupby
 from operator import itemgetter
 
 import requests
 from dateutil import parser
-from pystac import (
-    Item,
-    Link,
-    SpatialExtent,
-    Summaries,
-)
+from pystac import Catalog, Collection, Item, Link, SpatialExtent, Summaries
 from pystac_client import Client
 
 from eodash_catalog.sh_endpoint import get_SH_token
 from eodash_catalog.stac_handling import (
     add_collection_information,
     add_example_info,
-    get_or_create_collection_and_times,
+    get_collection_times_from_config,
+    get_or_create_collection,
 )
 from eodash_catalog.thumbnails import generate_thumbnail
 from eodash_catalog.utils import (
+    Options,
     create_geojson_point,
     generate_veda_cog_link,
     retrieveExtentFromWMSWMTS,
 )
 
 
-def process_STAC_Datacube_Endpoint(config, endpoint, data, catalog):
-    collection, _ = get_or_create_collection_and_times(
-        catalog, data["Name"], data, config, endpoint
+def process_STAC_Datacube_Endpoint(
+    catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
+) -> Collection:
+    collection = get_or_create_collection(
+        catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
     )
-    add_visualization_info(collection, data, endpoint)
+    add_visualization_info(collection, collection_config, endpoint_config)
 
-    stac_endpoint_url = endpoint["EndPoint"]
-    if endpoint.get("Name") == "xcube":
-        stac_endpoint_url = stac_endpoint_url + endpoint.get("StacEndpoint", "")
+    stac_endpoint_url = endpoint_config["EndPoint"]
+    if endpoint_config.get("Name") == "xcube":
+        stac_endpoint_url = stac_endpoint_url + endpoint_config.get("StacEndpoint", "")
     # assuming /search not implemented
     api = Client.open(stac_endpoint_url)
-    coll = api.get_collection(endpoint.get("CollectionId", "datacubes"))
-    item = coll.get_item(endpoint.get("DatacubeId"))
+    collection_id = endpoint_config.get("CollectionId", "datacubes")
+    coll = api.get_collection(collection_id)
+    if not coll:
+        raise ValueError(f"Collection {collection_id} not found in endpoint {endpoint_config}")
+    item_id = endpoint_config.get("DatacubeId", "")
+    item = coll.get_item(item_id)
+    if not item:
+        raise ValueError(f"Item  {item_id} not found in collection {coll}")
     # slice a datacube along temporal axis to individual items, selectively adding properties
     dimensions = item.properties.get("cube:dimensions", {})
-    variables = item.properties.get("cube:variables")
-    if endpoint.get("Variable") not in variables:
-        raise Exception(f'Variable {endpoint.get("Variable")} not found in datacube {variables}')
+    variables = item.properties.get("cube:variables", {})
+    if endpoint_config.get("Variable") not in variables:
+        raise Exception(
+            f'Variable {endpoint_config.get("Variable")} not found in datacube {variables}'
+        )
     time_dimension = "time"
     for k, v in dimensions.items():
         if v.get("type") == "temporal":
@@ -70,27 +81,34 @@ def process_STAC_Datacube_Endpoint(config, endpoint, data, catalog):
         else:
             link.extra_fields["start_datetime"] = item.properties["start_datetime"]
             link.extra_fields["end_datetime"] = item.properties["end_datetime"]
-    unit = variables.get(endpoint.get("Variable")).get("unit")
-    if unit and "yAxis" not in data:
-        data["yAxis"] = unit
+    unit = variables.get(endpoint_config.get("Variable")).get("unit")
+    if unit and "yAxis" not in collection_config:
+        collection_config["yAxis"] = unit
     collection.update_extent_from_items()
 
-    add_collection_information(config, collection, data)
+    add_collection_information(catalog_config, collection, collection_config)
 
     return collection
 
 
-def handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers=None):
-    if "Locations" in data:
-        root_collection, _ = get_or_create_collection_and_times(
-            catalog, data["Name"], data, config, endpoint
+def handle_STAC_based_endpoint(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    catalog: Catalog,
+    options: Options,
+    headers=None,
+) -> Collection:
+    if "Locations" in collection_config:
+        root_collection = get_or_create_collection(
+            catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
         )
-        for location in data["Locations"]:
+        for location in collection_config["Locations"]:
             if "FilterDates" in location:
                 collection = process_STACAPI_Endpoint(
-                    config=config,
-                    endpoint=endpoint,
-                    data=data,
+                    catalog_config=catalog_config,
+                    endpoint_config=endpoint_config,
+                    collection_config=collection_config,
                     catalog=catalog,
                     options=options,
                     headers=headers,
@@ -100,9 +118,9 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers
                 )
             else:
                 collection = process_STACAPI_Endpoint(
-                    config=config,
-                    endpoint=endpoint,
-                    data=data,
+                    catalog_config=catalog_config,
+                    endpoint_config=endpoint_config,
+                    collection_config=collection_config,
                     catalog=catalog,
                     options=options,
                     headers=headers,
@@ -112,8 +130,8 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers
             # Update identifier to use location as well as title
             # TODO: should we use the name as id? it provides much more
             # information in the clients
-            collection.id = location["Identifier"]
-            collection.title = (location["Name"],)
+            collection.id = location.get("Identifier", uuid.uuid4())
+            collection.title = location.get("Name")
             # See if description should be overwritten
             if "Description" in location:
                 collection.description = location["Description"]
@@ -121,12 +139,12 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers
                 collection.description = location["Name"]
             # TODO: should we remove all assets from sub collections?
             link = root_collection.add_child(collection)
-            latlng = f"{location["Point"][1]},{location["Point"][0]}"
+            latlng = f'{location["Point"][1]},{location["Point"][0]}'
             # Add extra properties we need
             link.extra_fields["id"] = location["Identifier"]
             link.extra_fields["latlng"] = latlng
             link.extra_fields["name"] = location["Name"]
-            add_example_info(collection, data, endpoint, config)
+            add_example_info(collection, collection_config, endpoint_config, catalog_config)
             if "OverwriteBBox" in location:
                 collection.extent.spatial = SpatialExtent(
                     [
@@ -136,55 +154,55 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers
         root_collection.update_extent_from_items()
         # Add bbox extents from children
         for c_child in root_collection.get_children():
-            root_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
+            if isinstance(c_child, Collection):
+                root_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
     else:
-        if "Bbox" in endpoint:
+        if "Bbox" in endpoint_config:
             root_collection = process_STACAPI_Endpoint(
-                config=config,
-                endpoint=endpoint,
-                data=data,
+                catalog_config=catalog_config,
+                endpoint_config=endpoint_config,
+                collection_config=collection_config,
                 catalog=catalog,
                 options=options,
                 headers=headers,
-                bbox=",".join(map(str, endpoint["Bbox"])),
+                bbox=",".join(map(str, endpoint_config["Bbox"])),
             )
         else:
             root_collection = process_STACAPI_Endpoint(
-                config=config,
-                endpoint=endpoint,
-                data=data,
+                catalog_config=catalog_config,
+                endpoint_config=endpoint_config,
+                collection_config=collection_config,
                 catalog=catalog,
                 options=options,
                 headers=headers,
             )
 
-    add_example_info(root_collection, data, endpoint, config)
+    add_example_info(root_collection, collection_config, endpoint_config, catalog_config)
     return root_collection
 
 
 def process_STACAPI_Endpoint(
-    config,
-    endpoint,
-    data,
-    catalog,
-    options,
-    headers=None,
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    catalog: Catalog,
+    options: Options,
+    headers: dict[str, str] | None = None,
     bbox=None,
-    root_collection=None,
-    filter_dates=None,
-):
+    root_collection: Collection | None = None,
+    filter_dates: list[str] | None = None,
+) -> Collection:
     if headers is None:
         headers = {}
-    collection, _ = get_or_create_collection_and_times(
-        catalog, endpoint["CollectionId"], data, config, endpoint
+    collection = get_or_create_collection(
+        catalog, endpoint_config["CollectionId"], collection_config, catalog_config, endpoint_config
     )
-    # add_visualization_info(collection, data, endpoint)
 
-    api = Client.open(endpoint["EndPoint"], headers=headers)
+    api = Client.open(endpoint_config["EndPoint"], headers=headers)
     if bbox is None:
-        bbox = "-180,-90,180,90"
+        bbox = [-180, -90, 180, 90]
     results = api.search(
-        collections=[endpoint["CollectionId"]],
+        collections=[endpoint_config["CollectionId"]],
         bbox=bbox,
         datetime=["1900-01-01T00:00:00Z", "3000-01-01T00:00:00Z"],
     )
@@ -203,37 +221,41 @@ def process_STACAPI_Endpoint(
         link = collection.add_item(item)
         if options.tn:
             if "cog_default" in item.assets:
-                generate_thumbnail(item, data, endpoint, item.assets["cog_default"].href)
+                generate_thumbnail(
+                    item, collection_config, endpoint_config, item.assets["cog_default"].href
+                )
             else:
-                generate_thumbnail(item, data, endpoint)
+                generate_thumbnail(item, collection_config, endpoint_config)
         # Check if we can create visualization link
-        if "Assets" in endpoint:
-            add_visualization_info(item, data, endpoint, item.id)
+        if "Assets" in endpoint_config:
+            add_visualization_info(item, collection_config, endpoint_config, item.id)
             link.extra_fields["item"] = item.id
         elif "cog_default" in item.assets:
-            add_visualization_info(item, data, endpoint, item.assets["cog_default"].href)
+            add_visualization_info(
+                item, collection_config, endpoint_config, item.assets["cog_default"].href
+            )
             link.extra_fields["cog_href"] = item.assets["cog_default"].href
         elif item_datetime:
             time_string = item_datetime.isoformat()[:-6] + "Z"
-            add_visualization_info(item, data, endpoint, time=time_string)
+            add_visualization_info(item, collection_config, endpoint_config, time=time_string)
         elif "start_datetime" in item.properties and "end_datetime" in item.properties:
             add_visualization_info(
                 item,
-                data,
-                endpoint,
+                collection_config,
+                endpoint_config,
                 time="{}/{}".format(
                     item.properties["start_datetime"], item.properties["end_datetime"]
                 ),
             )
         # If a root collection exists we point back to it from the item
-        if root_collection is not None:
+        if root_collection:
             item.set_collection(root_collection)
 
         # bubble up information we want to the link
         # it is possible for datetime to be null, if it is start and end datetime have to exist
         if item_datetime:
             iso_time = item_datetime.isoformat()[:-6] + "Z"
-            if endpoint["Name"] == "Sentinel Hub":
+            if endpoint_config["Name"] == "Sentinel Hub":
                 # for SH WMS we only save the date (no time)
                 link.extra_fields["datetime"] = iso_date
             else:
@@ -245,61 +267,74 @@ def process_STACAPI_Endpoint(
     collection.update_extent_from_items()
 
     # replace SH identifier with catalog identifier
-    collection.id = data["Name"]
-    add_collection_information(config, collection, data)
+    collection.id = collection_config["Name"]
+    add_collection_information(catalog_config, collection, collection_config)
 
     # Check if we need to overwrite the bbox after update from items
-    if "OverwriteBBox" in endpoint:
+    if "OverwriteBBox" in endpoint_config:
         collection.extent.spatial = SpatialExtent(
             [
-                endpoint["OverwriteBBox"],
+                endpoint_config["OverwriteBBox"],
             ]
         )
 
     return collection
 
 
-def handle_VEDA_endpoint(config, endpoint, data, catalog, options):
-    collection = handle_STAC_based_endpoint(config, endpoint, data, catalog, options)
+def handle_VEDA_endpoint(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    catalog: Catalog,
+    options: Options,
+) -> Collection:
+    collection = handle_STAC_based_endpoint(
+        catalog_config, endpoint_config, collection_config, catalog, options
+    )
     return collection
 
 
-def handle_collection_only(config, endpoint, data, catalog):
-    collection, times = get_or_create_collection_and_times(
-        catalog, data["Name"], data, config, endpoint
+def handle_collection_only(
+    catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
+) -> Collection:
+    collection = get_or_create_collection(
+        catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
     )
-    if len(times) > 0 and not endpoint.get("Disable_Items"):
+    times = get_collection_times_from_config(endpoint_config)
+    if len(times) > 0 and not endpoint_config.get("Disable_Items"):
         for t in times:
             item = Item(
                 id=t,
-                bbox=endpoint.get("OverwriteBBox"),
+                bbox=endpoint_config.get("OverwriteBBox"),
                 properties={},
                 geometry=None,
                 datetime=parser.isoparse(t),
             )
             link = collection.add_item(item)
             link.extra_fields["datetime"] = t
-    add_collection_information(config, collection, data)
+    add_collection_information(catalog_config, collection, collection_config)
     return collection
 
 
-def handle_SH_WMS_endpoint(config, endpoint, data, catalog):
+def handle_SH_WMS_endpoint(
+    catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
+) -> Collection:
     # create collection and subcollections (based on locations)
-    if "Locations" in data:
-        root_collection, _ = get_or_create_collection_and_times(
-            catalog, data["Name"], data, config, endpoint
+    if "Locations" in collection_config:
+        root_collection = get_or_create_collection(
+            catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
         )
-        for location in data["Locations"]:
+        for location in collection_config["Locations"]:
             # create  and populate location collections based on times
             # TODO: Should we add some new description per location?
             location_config = {
                 "Title": location["Name"],
                 "Description": "",
             }
-            collection, _ = get_or_create_collection_and_times(
-                catalog, location["Identifier"], location_config, config, endpoint
+            collection = get_or_create_collection(
+                catalog, location["Identifier"], location_config, catalog_config, endpoint_config
             )
-            collection.extra_fields["endpointtype"] = endpoint["Name"]
+            collection.extra_fields["endpointtype"] = endpoint_config["Name"]
             for time in location["Times"]:
                 item = Item(
                     id=time,
@@ -311,7 +346,7 @@ def handle_SH_WMS_endpoint(config, endpoint, data, catalog):
                         "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
                     ],
                 )
-                add_visualization_info(item, data, endpoint, time=time)
+                add_visualization_info(item, collection_config, endpoint_config, time=time)
                 item_link = collection.add_item(item)
                 item_link.extra_fields["datetime"] = time
 
@@ -323,39 +358,44 @@ def handle_SH_WMS_endpoint(config, endpoint, data, catalog):
             link.extra_fields["country"] = location["Country"]
             link.extra_fields["city"] = location["Name"]
             collection.update_extent_from_items()
-            add_visualization_info(collection, data, endpoint)
+            add_visualization_info(collection, collection_config, endpoint_config)
 
         root_collection.update_extent_from_items()
         # Add bbox extents from children
         for c_child in root_collection.get_children():
-            root_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
+            if isinstance(c_child, Collection):
+                root_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
     return root_collection
 
 
-def handle_xcube_endpoint(config, endpoint, data: dict, catalog):
-    root_collection = process_STAC_Datacube_Endpoint(
-        config=config,
-        endpoint=endpoint,
-        data=data,
+def handle_xcube_endpoint(
+    catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
+) -> Collection:
+    collection = process_STAC_Datacube_Endpoint(
+        catalog_config=catalog_config,
+        endpoint_config=endpoint_config,
+        collection_config=collection_config,
         catalog=catalog,
     )
 
-    add_example_info(root_collection, data, endpoint, config)
-    return root_collection
+    add_example_info(collection, collection_config, endpoint_config, catalog_config)
+    return collection
 
 
-def handle_GeoDB_endpoint(config, endpoint, data: dict, catalog):
-    collection, _ = get_or_create_collection_and_times(
-        catalog, endpoint["CollectionId"], data, config, endpoint
+def handle_GeoDB_endpoint(
+    catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
+) -> Collection:
+    collection = get_or_create_collection(
+        catalog, endpoint_config["CollectionId"], collection_config, catalog_config, endpoint_config
     )
     select = "?select=aoi,aoi_id,country,city,time"
     url = (
-        endpoint["EndPoint"]
-        + endpoint["Database"]
-        + "_{}".format(endpoint["CollectionId"])
+        endpoint_config["EndPoint"]
+        + endpoint_config["Database"]
+        + "_{}".format(endpoint_config["CollectionId"])
         + select
     )
-    if additional_query_parameters := endpoint.get("AdditionalQueryString"):
+    if additional_query_parameters := endpoint_config.get("AdditionalQueryString"):
         url += f"&{additional_query_parameters}"
     response = json.loads(requests.get(url).text)
 
@@ -370,7 +410,7 @@ def handle_GeoDB_endpoint(config, endpoint, data: dict, catalog):
         unique_values = next(iter({v["aoi_id"]: v for v in values}.values()))
         country = unique_values["country"]
         city = unique_values["city"]
-        IdKey = endpoint.get("IdKey", "city")
+        IdKey = endpoint_config.get("IdKey", "city")
         IdValue = unique_values[IdKey]
         if country not in countries:
             countries.append(country)
@@ -408,20 +448,20 @@ def handle_GeoDB_endpoint(config, endpoint, data: dict, catalog):
         link.extra_fields["country"] = country
         link.extra_fields["city"] = city
 
-    if "yAxis" not in data:
+    if "yAxis" not in collection_config:
         # fetch yAxis and store it to data, preventing need to save it per dataset in yml
         select = "?select=y_axis&limit=1"
         url = (
-            endpoint["EndPoint"]
-            + endpoint["Database"]
-            + "_{}".format(endpoint["CollectionId"])
+            endpoint_config["EndPoint"]
+            + endpoint_config["Database"]
+            + "_{}".format(endpoint_config["CollectionId"])
             + select
         )
         response = json.loads(requests.get(url).text)
         yAxis = response[0]["y_axis"]
-        data["yAxis"] = yAxis
-    add_collection_information(config, collection, data)
-    add_example_info(collection, data, endpoint, config)
+        collection_config["yAxis"] = yAxis
+    add_collection_information(catalog_config, collection, collection_config)
+    add_example_info(collection, collection_config, endpoint_config, catalog_config)
 
     collection.update_extent_from_items()
     collection.summaries = Summaries(
@@ -433,34 +473,51 @@ def handle_GeoDB_endpoint(config, endpoint, data: dict, catalog):
     return collection
 
 
-def handle_SH_endpoint(config, endpoint, data, catalog, options):
+def handle_SH_endpoint(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    catalog: Catalog,
+    options: Options,
+) -> Collection:
     token = get_SH_token()
     headers = {"Authorization": f"Bearer {token}"}
-    endpoint["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
+    endpoint_config["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
     # Overwrite collection id with type, such as ZARR or BYOC
-    if "Type" in endpoint:
-        endpoint["CollectionId"] = endpoint["Type"] + "-" + endpoint["CollectionId"]
-    collection = handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers)
+    if "Type" in endpoint_config:
+        endpoint_config["CollectionId"] = (
+            endpoint_config["Type"] + "-" + endpoint_config["CollectionId"]
+        )
+    collection = handle_STAC_based_endpoint(
+        catalog_config, endpoint_config, collection_config, catalog, options, headers
+    )
     return collection
 
 
-def handle_WMS_endpoint(config, endpoint, data, catalog, wmts=False):
-    collection, times = get_or_create_collection_and_times(
-        catalog, data["Name"], data, config, endpoint
+def handle_WMS_endpoint(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    catalog: Catalog,
+    wmts: bool = False,
+) -> Collection:
+    collection = get_or_create_collection(
+        catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
     )
+    times = get_collection_times_from_config(endpoint_config)
     spatial_extent = collection.extent.spatial.to_dict().get("bbox", [-180, -90, 180, 90])[0]
-    if endpoint.get("Type") != "OverwriteTimes" or not endpoint.get("OverwriteBBox"):
+    if endpoint_config.get("Type") != "OverwriteTimes" or not endpoint_config.get("OverwriteBBox"):
         # some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
         # have to process full service capabilities XML
-        capabilities_url = endpoint["EndPoint"]
+        capabilities_url = endpoint_config["EndPoint"]
         spatial_extent, times = retrieveExtentFromWMSWMTS(
             capabilities_url,
-            endpoint["LayerId"],
-            version=endpoint.get("Version", "1.1.1"),
+            endpoint_config["LayerId"],
+            version=endpoint_config.get("Version", "1.1.1"),
             wmts=wmts,
         )
     # Create an item per time to allow visualization in stac clients
-    if len(times) > 0 and not endpoint.get("Disable_Items"):
+    if len(times) > 0 and not endpoint_config.get("Disable_Items"):
         for t in times:
             item = Item(
                 id=t,
@@ -472,98 +529,102 @@ def handle_WMS_endpoint(config, endpoint, data, catalog, wmts=False):
                     "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
                 ],
             )
-            add_visualization_info(item, data, endpoint, time=t)
+            add_visualization_info(item, collection_config, endpoint_config, time=t)
             link = collection.add_item(item)
             link.extra_fields["datetime"] = t
         collection.update_extent_from_items()
 
     # Check if we should overwrite bbox
-    if "OverwriteBBox" in endpoint:
+    if "OverwriteBBox" in endpoint_config:
         collection.extent.spatial = SpatialExtent(
             [
-                endpoint["OverwriteBBox"],
+                endpoint_config["OverwriteBBox"],
             ]
         )
-    add_collection_information(config, collection, data)
+    add_collection_information(catalog_config, collection, collection_config)
     return collection
 
 
-def handle_GeoDB_Tiles_endpoint(config, endpoint, data, catalog):
-    raise NotImplementedError
-
-
-def generate_veda_tiles_link(endpoint, item):
-    collection = "collection={}".format(endpoint["CollectionId"])
+def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
+    collection = "collection={}".format(endpoint_config["CollectionId"])
     assets = ""
-    for asset in endpoint["Assets"]:
+    for asset in endpoint_config["Assets"]:
         assets += f"&assets={asset}"
     color_formula = ""
-    if "ColorFormula" in endpoint:
-        color_formula = "&color_formula={}".format(endpoint["ColorFormula"])
+    if "ColorFormula" in endpoint_config:
+        color_formula = "&color_formula={}".format(endpoint_config["ColorFormula"])
     no_data = ""
-    if "NoData" in endpoint:
-        no_data = "&no_data={}".format(endpoint["NoData"])
+    if "NoData" in endpoint_config:
+        no_data = "&no_data={}".format(endpoint_config["NoData"])
     item = f"&item={item}" if item else ""
     target_url = f"https://staging-raster.delta-backend.com/stac/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{collection}{item}{assets}{color_formula}{no_data}"
     return target_url
 
 
-def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None):
+def add_visualization_info(
+    stac_object: Collection | Item,
+    collection_config: dict,
+    endpoint_config: dict,
+    file_url: str | None = None,
+    time: str | None = None,
+) -> None:
     # add extension reference
-    if endpoint["Name"] == "Sentinel Hub" or endpoint["Name"] == "Sentinel Hub WMS":
+    if endpoint_config["Name"] == "Sentinel Hub" or endpoint_config["Name"] == "Sentinel Hub WMS":
         instanceId = os.getenv("SH_INSTANCE_ID")
-        if "InstanceId" in endpoint:
-            instanceId = endpoint["InstanceId"]
-        extra_fields = {
-            "wms:layers": [endpoint["LayerId"]],
+        if "InstanceId" in endpoint_config:
+            instanceId = endpoint_config["InstanceId"]
+        extra_fields: dict[str, list[str] | dict[str, str]] = {
+            "wms:layers": [endpoint_config["LayerId"]],
             "role": ["data"],
         }
         if time is not None:
-            if endpoint["Name"] == "Sentinel Hub WMS":
+            if endpoint_config["Name"] == "Sentinel Hub WMS":
                 # SH WMS for public collections needs time interval, we use full day here
                 datetime_object = datetime.strptime(time, "%Y-%m-%d")
                 start = datetime_object.isoformat()
                 end = (datetime_object + timedelta(days=1) - timedelta(milliseconds=1)).isoformat()
                 time_interval = f"{start}/{end}"
                 extra_fields["wms:dimensions"] = {"TIME": time_interval}
-            if endpoint["Name"] == "Sentinel Hub":
+            if endpoint_config["Name"] == "Sentinel Hub":
                 extra_fields["wms:dimensions"] = {"TIME": time}
         stac_object.add_link(
             Link(
                 rel="wms",
                 target=f"https://services.sentinel-hub.com/ogc/wms/{instanceId}",
-                media_type=(endpoint.get("MimeType", "image/png")),
-                title=data["Name"],
+                media_type=(endpoint_config.get("MimeType", "image/png")),
+                title=collection_config["Name"],
                 extra_fields=extra_fields,
             )
         )
-    elif endpoint["Name"] == "WMS":
+    elif endpoint_config["Name"] == "WMS":
         extra_fields = {
-            "wms:layers": [endpoint["LayerId"]],
+            "wms:layers": [endpoint_config["LayerId"]],
             "role": ["data"],
         }
         if time is not None:
             extra_fields["wms:dimensions"] = {
                 "TIME": time,
             }
-        if "Styles" in endpoint:
-            extra_fields["wms:styles"] = endpoint["Styles"]
+        if "Styles" in endpoint_config:
+            extra_fields["wms:styles"] = endpoint_config["Styles"]
         media_type = "image/jpeg"
-        if "MediaType" in endpoint:
-            media_type = endpoint["MediaType"]
+        if "MediaType" in endpoint_config:
+            media_type = endpoint_config["MediaType"]
         stac_object.add_link(
             Link(
                 rel="wms",
-                target=endpoint["EndPoint"],
+                target=endpoint_config["EndPoint"],
                 media_type=media_type,
-                title=data["Name"],
+                title=collection_config["Name"],
                 extra_fields=extra_fields,
             )
         )
-    elif endpoint["Name"] == "JAXA_WMTS_PALSAR":
-        target_url = "{}".format(endpoint.get("EndPoint"))
+    elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
+        target_url = "{}".format(endpoint_config.get("EndPoint"))
         # custom time just for this special case as a default for collection wmts
-        extra_fields = {"wmts:layer": endpoint.get("LayerId").replace("{time}", time or "2017")}
+        extra_fields = {
+            "wmts:layer": endpoint_config.get("LayerId", "").replace("{time}", time or "2017")
+        }
         stac_object.add_link(
             Link(
                 rel="wmts",
@@ -573,23 +634,23 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
                 extra_fields=extra_fields,
             )
         )
-    elif endpoint["Name"] == "xcube":
-        if endpoint["Type"] == "zarr":
+    elif endpoint_config["Name"] == "xcube":
+        if endpoint_config["Type"] == "zarr":
             # either preset ColormapName of left as a template
-            cbar = endpoint.get("ColormapName", "{cbar}")
+            cbar = endpoint_config.get("ColormapName", "{cbar}")
             # either preset Rescale of left as a template
             vmin = "{vmin}"
             vmax = "{vmax}"
-            if "Rescale" in endpoint:
-                vmin = endpoint["Rescale"][0]
-                vmax = endpoint["Rescale"][1]
-            crs = endpoint.get("Crs", "EPSG:3857")
+            if "Rescale" in endpoint_config:
+                vmin = endpoint_config["Rescale"][0]
+                vmax = endpoint_config["Rescale"][1]
+            crs = endpoint_config.get("Crs", "EPSG:3857")
             target_url = (
                 "{}/tiles/{}/{}/{{z}}/{{y}}/{{x}}" "?crs={}&time={{time}}&vmin={}&vmax={}&cbar={}"
             ).format(
-                endpoint["EndPoint"],
-                endpoint["DatacubeId"],
-                endpoint["Variable"],
+                endpoint_config["EndPoint"],
+                endpoint_config["DatacubeId"],
+                endpoint_config["Variable"],
                 crs,
                 vmin,
                 vmax,
@@ -603,16 +664,16 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
                     title="xcube tiles",
                 )
             )
-    elif endpoint["Type"] == "WMTSCapabilities":
-        target_url = "{}".format(endpoint.get("EndPoint"))
+    elif endpoint_config["Type"] == "WMTSCapabilities":
+        target_url = "{}".format(endpoint_config.get("EndPoint"))
         extra_fields = {
-            "wmts:layer": endpoint.get("LayerId"),
+            "wmts:layer": endpoint_config.get("LayerId", ""),
             "role": ["data"],
         }
         dimensions = {}
         if time is not None:
             dimensions["time"] = time
-        if dimensions_config := endpoint.get("Dimensions", {}):
+        if dimensions_config := endpoint_config.get("Dimensions", {}):
             for key, value in dimensions_config.items():
                 dimensions[key] = value
         if dimensions != {}:
@@ -626,44 +687,80 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
                 extra_fields=extra_fields,
             )
         )
-    elif endpoint["Name"] == "VEDA":
-        if endpoint["Type"] == "cog":
-            target_url = generate_veda_cog_link(endpoint, file_url)
-        elif endpoint["Type"] == "tiles":
-            target_url = generate_veda_tiles_link(endpoint, file_url)
+    elif endpoint_config["Name"] == "VEDA":
+        if endpoint_config["Type"] == "cog":
+            target_url = generate_veda_cog_link(endpoint_config, file_url)
+        elif endpoint_config["Type"] == "tiles":
+            target_url = generate_veda_tiles_link(endpoint_config, file_url)
         if target_url:
             stac_object.add_link(
                 Link(
                     rel="xyz",
                     target=target_url,
                     media_type="image/png",
-                    title=data["Name"],
+                    title=collection_config["Name"],
                 )
             )
-    elif endpoint["Name"] == "GeoDB Vector Tiles":
+    elif endpoint_config["Name"] == "GeoDB Vector Tiles":
         # `${geoserverUrl}${config.layerName}@EPSG%3A${projString}@pbf/{z}/{x}/{-y}.pbf`,
         # 'geodb_debd884d-92f9-4979-87b6-eadef1139394:GTIF_AT_Gemeinden_3857'
         target_url = "{}{}:{}_{}@EPSG:3857@pbf/{{z}}/{{x}}/{{-y}}.pbf".format(
-            endpoint["EndPoint"],
-            endpoint["Instance"],
-            endpoint["Database"],
-            endpoint["CollectionId"],
+            endpoint_config["EndPoint"],
+            endpoint_config["Instance"],
+            endpoint_config["Database"],
+            endpoint_config["CollectionId"],
         )
         stac_object.add_link(
             Link(
                 rel="xyz",
                 target=target_url,
                 media_type="application/pbf",
-                title=data["Name"],
+                title=collection_config["Name"],
                 extra_fields={
-                    "description": data["Title"],
-                    "parameters": endpoint["Parameters"],
-                    "matchKey": endpoint["MatchKey"],
-                    "timeKey": endpoint["TimeKey"],
-                    "source": endpoint["Source"],
+                    "description": collection_config["Title"],
+                    "parameters": endpoint_config["Parameters"],
+                    "matchKey": endpoint_config["MatchKey"],
+                    "timeKey": endpoint_config["TimeKey"],
+                    "source": endpoint_config["Source"],
                     "role": ["data"],
                 },
             )
         )
     else:
         print("Visualization endpoint not supported")
+
+
+def handle_custom_endpoint(
+    catalog_config: dict,
+    endpoint_config: dict,
+    collection_config: dict,
+    catalog: Catalog,
+) -> Collection:
+    collection = get_or_create_collection(
+        catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
+    )
+    # invoke 3rd party code and return
+    function_path = endpoint_config["Python_Function_Location"]
+    module_name, _, func_name = function_path.rpartition(".")
+    # add current working directory to sys path
+    sys.path.append(os.getcwd())
+    try:
+        # import configured function
+        imported_function: Callable[[Collection, dict, dict, dict], Collection] = getattr(
+            importlib.import_module(module_name), func_name
+        )
+    except ModuleNotFoundError as e:
+        print(
+            f"""function {func_name} from module {module_name} can not be imported.
+            Check if you are specifying relative path inside the
+            catalog repository or catalog generator repository."""
+        )
+        raise e
+    # execture the custom handler
+    collection = imported_function(
+        collection,
+        catalog_config,
+        endpoint_config,
+        collection_config,
+    )
+    return collection