eodash_catalog 0.0.1__tar.gz → 0.0.3__tar.gz

{eodash_catalog-0.0.1 → eodash_catalog-0.0.3}/.gitignore

@@ -6,8 +6,8 @@ build/
 dist/
 .pytest*
 *.gfs
-.vscode/
 __pycache__
 catalogs/
 collections/
+layers/
 

eodash_catalog-0.0.3/.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/ambv/black
+    rev: 23.3.0
+    hooks:
+      - id: black
+        language_version: python3

eodash_catalog-0.0.3/.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "ms-python.black-formatter"
+    ]
+}

eodash_catalog-0.0.3/.vscode/settings.json

@@ -0,0 +1,52 @@
+{
+  "editor.formatOnSave": true,
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter"
+  },
+  "cSpell.words": [
+    "bboxes",
+    "bidx",
+    "Bidx",
+    "byoc",
+    "BYOC",
+    "catalogspath",
+    "cbar",
+    "collectionsspath",
+    "Datacube",
+    "datacubes",
+    "dateutil",
+    "debd",
+    "endpointtype",
+    "eodash",
+    "Eodash",
+    "evalscript",
+    "filterdates",
+    "Gemeinden",
+    "geodb",
+    "geoserver",
+    "GTIF",
+    "insitu",
+    "ipynb",
+    "isoparse",
+    "Isostrings",
+    "latlng",
+    "latlon",
+    "marinedatastore",
+    "outputpath",
+    "pystac",
+    "subcollection",
+    "Subcollection",
+    "subcollections",
+    "Subcollections",
+    "timeseries",
+    "urlparsed",
+    "vmax",
+    "vmin",
+    "WMSWMTS",
+    "wmts",
+    "WMTS",
+    "xcube",
+    "zarr",
+    "ZARR"
+  ],
+}

{eodash_catalog-0.0.1 → eodash_catalog-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: eodash_catalog
-Version: 0.0.1
+Version: 0.0.3
 Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
 Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
 Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
@@ -10,8 +10,6 @@ License-Expression: MIT
 License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -37,6 +35,8 @@ Requires-Dist: setuptools<68
 Requires-Dist: spdx-lookup<=0.3.3
 Requires-Dist: structlog<22.0
 Requires-Dist: swiftspec==0.0.2
+Provides-Extra: dev
+Requires-Dist: pre-commit; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # eodash_catalog

{eodash_catalog-0.0.1 → eodash_catalog-0.0.3}/pyproject.toml

@@ -14,8 +14,6 @@ authors = [{ name = "Daniel Santillan", email = "daniel.santillan@eox.at" }]
 classifiers = [
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.8",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -49,6 +47,8 @@ eodash_catalog = "eodash_catalog.generate_indicators:process_catalogs"
 Documentation = "https://github.com/eodash/eodash_catalog#readme"
 Issues = "https://github.com/eodash/eodash_catalog/issues"
 Source = "https://github.com/eodash/eodash_catalog"
+[project.optional-dependencies]
+dev = ["pre-commit"]
 
 [tool.hatch.version]
 path = "src/eodash_catalog/__about__.py"
@@ -64,7 +64,7 @@ cov-report = ["- coverage combine", "coverage report"]
 cov = ["test-cov", "cov-report"]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
+python = ["3.10", "3.11", "3.12"]
 
 [tool.hatch.envs.types]
 dependencies = ["mypy>=1.0.0"]

{eodash_catalog-0.0.1 → eodash_catalog-0.0.3}/src/eodash_catalog/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "0.0.1"
+__version__ = "0.0.3"

{eodash_catalog-0.0.1 → eodash_catalog-0.0.3}/src/eodash_catalog/generate_indicators.py

@@ -86,12 +86,19 @@ def process_catalog_file(file_path, options):
             catalog_type=CatalogType.RELATIVE_PUBLISHED,
         )
         for collection in process_collections:
-            process_collection_file(
-                config,
-                "%s/%s.yaml" % (options.collectionsspath, collection),
-                catalog,
-                options,
-            )
+            file_path = "%s/%s.yaml" % (options.collectionspath, collection)
+            if os.path.isfile(file_path):
+                # if collection file exists process it as indicator
+                # collection will be added as single collection to indicator
+                process_indicator_file(config, file_path, catalog, options)
+            else:
+                # if not try to see if indicator definition available
+                process_indicator_file(
+                    config,
+                    "%s/%s.yaml" % (options.indicatorspath, collection),
+                    catalog,
+                    options,
+                )
 
         strategy = TemplateLayoutStrategy(item_template="${collection}/${year}")
         catalog.normalize_hrefs(
@@ -117,6 +124,81 @@ def process_catalog_file(file_path, options):
                 print("Issue validation collection: %s" % e)
 
 
+def extract_indicator_info(parent_collection):
+    to_extract = [
+        "subcode",
+        "themes",
+        "keywords",
+        "satellite",
+        "sensor",
+        "cities",
+        "countries",
+    ]
+    summaries = {}
+    for key in to_extract:
+        summaries[key] = set()
+
+    for collection in parent_collection.get_collections():
+        for key in to_extract:
+            if key in collection.extra_fields:
+                param = collection.extra_fields[key]
+                if isinstance(param, list):
+                    for p in param:
+                        summaries[key].add(p)
+                else:
+                    summaries[key].add(param)
+            # extract also summary information
+            if collection.summaries.lists:
+                if key in collection.summaries.lists:
+                    for p in collection.summaries.lists[key]:
+                        summaries[key].add(p)
+
+    for key in to_extract:
+        # convert all items back to a list
+        summaries[key] = list(summaries[key])
+        # remove empty ones
+        if len(summaries[key]) == 0:
+            del summaries[key]
+    parent_collection.summaries = Summaries(summaries)
+
+
+def iter_len_at_least(i, n):
+    return sum(1 for _ in zip(range(n), i)) == n
+
+
+def process_indicator_file(config, file_path, catalog, options):
+    with open(file_path) as f:
+        print("Processing indicator:", file_path)
+        data = yaml.load(f, Loader=SafeLoader)
+        parent_indicator, _ = get_or_create_collection(
+            catalog, data["Name"], data, config
+        )
+        if "Collections" in data:
+            for collection in data["Collections"]:
+                process_collection_file(
+                    config,
+                    "%s/%s.yaml" % (options.collectionspath, collection),
+                    catalog,
+                    options,
+                )
+        else:
+            # we assume that collection files can also be loaded directly
+            process_collection_file(config, file_path, parent_indicator, options)
+        add_collection_information(config, parent_indicator, data)
+        if iter_len_at_least(parent_indicator.get_items(recursive=True), 1):
+            parent_indicator.update_extent_from_items()
+        # Add bbox extents from children
+        for c_child in parent_indicator.get_children():
+            parent_indicator.extent.spatial.bboxes.append(
+                c_child.extent.spatial.bboxes[0]
+            )
+        # extract collection information and add it to summary indicator level
+        extract_indicator_info(parent_indicator)
+        # add baselayer and overview information to indicator collection
+        add_base_overlay_info(parent_indicator, config, data)
+        add_to_catalog(parent_indicator, catalog, None, data)
+
+
 def process_collection_file(config, file_path, catalog, options):
     print("Processing collection:", file_path)
     with open(file_path) as f:
@@ -125,7 +207,7 @@ def process_collection_file(config, file_path, catalog, options):
             for resource in data["Resources"]:
                 if "EndPoint" in resource:
                     if resource["Name"] == "Sentinel Hub":
-                        handle_SH_endpoint(config, resource, data, catalog)
+                        handle_SH_endpoint(config, resource, data, catalog, options)
                     elif resource["Name"] == "Sentinel Hub WMS":
                         collection = handle_SH_WMS_endpoint(
                             config, resource, data, catalog
@@ -136,7 +218,7 @@ def process_collection_file(config, file_path, catalog, options):
                         )
                         add_to_catalog(collection, catalog, resource, data)
                     elif resource["Name"] == "VEDA":
-                        handle_VEDA_endpoint(config, resource, data, catalog)
+                        handle_VEDA_endpoint(config, resource, data, catalog, options)
                     elif resource["Name"] == "marinedatastore":
                         handle_WMS_endpoint(config, resource, data, catalog, wmts=True)
                     elif resource["Name"] == "xcube":
@@ -264,7 +346,6 @@ def handle_WMS_endpoint(config, endpoint, data, catalog, wmts=False):
     if not endpoint.get("Type") == "OverwriteTimes" or not endpoint.get(
         "OverwriteBBox"
     ):
-
         # some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
         # have to process full service capabilities XML
         capabilities_url = endpoint["EndPoint"]
@@ -280,6 +361,9 @@ def handle_WMS_endpoint(config, endpoint, data, catalog, wmts=False):
                 properties={},
                 geometry=None,
                 datetime=parser.isoparse(t),
+                stac_extensions=[
+                    "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
+                ],
             )
             add_visualization_info(item, data, endpoint, time=t)
             link = collection.add_item(item)
@@ -293,20 +377,18 @@ def handle_WMS_endpoint(config, endpoint, data, catalog, wmts=False):
                 endpoint["OverwriteBBox"],
             ]
         )
-
-    add_visualization_info(collection, data, endpoint)
     add_collection_information(config, collection, data)
     add_to_catalog(collection, catalog, endpoint, data)
 
 
-def handle_SH_endpoint(config, endpoint, data, catalog):
+def handle_SH_endpoint(config, endpoint, data, catalog, options):
     token = get_SH_token()
     headers = {"Authorization": "Bearer %s" % token}
     endpoint["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
     # Overwrite collection id with type, such as ZARR or BYOC
     if "Type" in endpoint:
         endpoint["CollectionId"] = endpoint["Type"] + "-" + endpoint["CollectionId"]
-    handle_STAC_based_endpoint(config, endpoint, data, catalog, headers)
+    handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers)
 
 
 def handle_SH_WMS_endpoint(config, endpoint, data, catalog):
@@ -333,6 +415,9 @@ def handle_SH_WMS_endpoint(config, endpoint, data, catalog):
                     properties={},
                     geometry=None,
                     datetime=parser.isoparse(time),
+                    stac_extensions=[
+                        "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
+                    ],
                 )
                 add_visualization_info(item, data, endpoint, time=time)
                 item_link = collection.add_item(item)
@@ -358,8 +443,8 @@ def handle_SH_WMS_endpoint(config, endpoint, data, catalog):
     return root_collection
 
 
-def handle_VEDA_endpoint(config, endpoint, data, catalog):
-    handle_STAC_based_endpoint(config, endpoint, data, catalog)
+def handle_VEDA_endpoint(config, endpoint, data, catalog, options):
+    handle_STAC_based_endpoint(config, endpoint, data, catalog, options)
 
 
 def handle_xcube_endpoint(config, endpoint, data, catalog):
@@ -440,11 +525,6 @@ def get_or_create_collection(catalog, collection_id, data, config, endpoint=None
         id=collection_id,
         title=data["Title"],
         description=description,
-        stac_extensions=[
-            "https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
-            "https://stac-extensions.github.io/example-links/v0.0.1/schema.json",
-            "https://stac-extensions.github.io/scientific/v1.0.0/schema.json",
-        ],
         extent=extent,
     )
     return (collection, times)
@@ -479,6 +559,8 @@ def add_to_catalog(collection, catalog, endpoint, data):
     link.extra_fields["title"] = collection.title
     link.extra_fields["code"] = data["EodashIdentifier"]
     link.extra_fields["id"] = data["Name"]
+    if "Themes" in data:
+        link.extra_fields["themes"] = data["Themes"]
     # Check for summaries and bubble up info
     if collection.summaries.lists:
         for sum in collection.summaries.lists:
@@ -505,6 +587,8 @@ def add_extra_fields(stac_object, data):
         stac_object.extra_fields["agency"] = data["Agency"]
     if "yAxis" in data:
         stac_object.extra_fields["yAxis"] = data["yAxis"]
+    if "EodashIdentifier" in data:
+        stac_object.extra_fields["subcode"] = data["EodashIdentifier"]
     if "DataSource" in data:
         if "Spaceborne" in data["DataSource"]:
             if "Sensor" in data["DataSource"]["Spaceborne"]:
@@ -610,7 +694,7 @@ def handle_GeoDB_endpoint(config, endpoint, data, catalog):
     return collection
 
 
-def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None):
+def handle_STAC_based_endpoint(config, endpoint, data, catalog, options, headers=None):
     if "Locations" in data:
         root_collection, _ = get_or_create_collection(
             catalog, data["Name"], data, config, endpoint
@@ -622,6 +706,7 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None):
                     endpoint=endpoint,
                     data=data,
                     catalog=catalog,
+                    options=options,
                     headers=headers,
                     bbox=",".join(map(str, location["Bbox"])),
                     filter_dates=location["FilterDates"],
@@ -633,6 +718,7 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None):
                     endpoint=endpoint,
                     data=data,
                     catalog=catalog,
+                    options=options,
                     headers=headers,
                     bbox=",".join(map(str, location["Bbox"])),
                     root_collection=root_collection,
@@ -674,6 +760,7 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None):
                 endpoint=endpoint,
                 data=data,
                 catalog=catalog,
+                options=options,
                 headers=headers,
                 bbox=",".join(map(str, endpoint["Bbox"])),
             )
@@ -683,6 +770,7 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None):
                 endpoint=endpoint,
                 data=data,
                 catalog=catalog,
+                options=options,
                 headers=headers,
             )
 
@@ -690,6 +778,61 @@ def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None):
     add_to_catalog(root_collection, catalog, endpoint, data)
 
 
+def add_base_overlay_info(collection, config, data):
+    # check if default base layers defined
+    if "default_base_layers" in config:
+        with open("%s.yaml" % config["default_base_layers"]) as f:
+            base_layers = yaml.load(f, Loader=SafeLoader)
+            for layer in base_layers:
+                collection.add_link(create_web_map_link(layer, role="baselayer"))
+    # check if default overlay layers defined
+    if "default_overlay_layers" in config:
+        with open("%s.yaml" % config["default_overlay_layers"]) as f:
+            overlay_layers = yaml.load(f, Loader=SafeLoader)
+            for layer in overlay_layers:
+                collection.add_link(create_web_map_link(layer, role="overlay"))
+    if "BaseLayers" in data:
+        for layer in data["BaseLayers"]:
+            collection.add_link(create_web_map_link(layer, role="baselayer"))
+    if "OverlayLayers" in data:
+        for layer in data["OverlayLayers"]:
+            collection.add_link(create_web_map_link(layer, role="overlay"))
+    # TODO: possibility to overwrite default base and overlay layers
+
+
+def create_web_map_link(layer, role):
+    extra_fields = {
+        "roles": [role],
+        "id": layer["id"],
+    }
+    if "default" in layer and layer["default"]:
+        extra_fields["roles"].append("default")
+    if "visible" in layer and layer["visible"]:
+        extra_fields["roles"].append("visible")
+    if "visible" in layer and not layer["visible"]:
+        extra_fields["roles"].append("invisible")
+
+    match layer["protocol"]:
+        case "wms":
+            # handle wms special config options
+            extra_fields["wms:layers"] = layer["layers"]
+            if "styles" in layer:
+                extra_fields["wms:styles"] = layer["styles"]
+            # TODO: handle wms dimensions extra_fields["wms:dimensions"]
+        case "wmts":
+            extra_fields["wmts:layer"] = layer["layer"]
+            # TODO: handle wmts dimensions extra_fields["wmts:dimensions"]
+
+    wml = Link(
+        rel=layer["protocol"],
+        target=layer["url"],
+        media_type="image/png" if "media_type" not in layer else layer["media_type"],
+        title=layer["name"],
+        extra_fields=extra_fields,
+    )
+    return wml
+
+
 def add_example_info(stac_object, data, endpoint, config):
     if "Services" in data:
         for service in data["Services"]:
@@ -836,6 +979,7 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
             instanceId = endpoint["InstanceId"]
         extra_fields = {
             "wms:layers": [endpoint["LayerId"]],
+            "role": ["data"],
         }
         if time != None:
             if endpoint["Name"] == "Sentinel Hub WMS":
@@ -858,7 +1002,9 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
             Link(
                 rel="wms",
                 target="https://services.sentinel-hub.com/ogc/wms/%s" % (instanceId),
-                media_type="text/xml",
+                media_type=(
+                    endpoint["MimeType"] if "MimeType" in endpoint else "image/png"
+                ),
                 title=data["Name"],
                 extra_fields=extra_fields,
             )
@@ -866,7 +1012,10 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
     # elif resource["Name"] == "GeoDB":
     #     pass
     elif endpoint["Name"] == "WMS":
-        extra_fields = {"wms:layers": [endpoint["LayerId"]]}
+        extra_fields = {
+            "wms:layers": [endpoint["LayerId"]],
+            "role": ["data"],
+        }
         if time != None:
             extra_fields["wms:dimensions"] = {
                 "TIME": time,
@@ -918,7 +1067,10 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
             )
     elif endpoint["Type"] == "WMTSCapabilities":
         target_url = "%s" % (endpoint.get("EndPoint"),)
-        extra_fields = {"wmts:layer": endpoint.get("LayerId")}
+        extra_fields = {
+            "wmts:layer": endpoint.get("LayerId"),
+            "role": ["data"],
+        }
         dimensions = {}
         if time != None:
             dimensions["time"] = time
@@ -971,6 +1123,7 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
                     "matchKey": endpoint["MatchKey"],
                     "timeKey": endpoint["TimeKey"],
                     "source": endpoint["Source"],
+                    "role": ["data"],
                 },
             )
         )
@@ -983,6 +1136,7 @@ def process_STACAPI_Endpoint(
     endpoint,
     data,
     catalog,
+    options,
     headers={},
     bbox=None,
     root_collection=None,
@@ -991,7 +1145,7 @@ def process_STACAPI_Endpoint(
     collection, _ = get_or_create_collection(
         catalog, endpoint["CollectionId"], data, config, endpoint
     )
-    add_visualization_info(collection, data, endpoint)
+    # add_visualization_info(collection, data, endpoint)
 
     api = Client.open(endpoint["EndPoint"], headers=headers)
     if bbox == None:
@@ -1313,7 +1467,8 @@ def add_collection_information(config, collection, data):
 @dataclass
 class Options:
     catalogspath: str
-    collectionsspath: str
+    collectionspath: str
+    indicatorspath: str
     outputpath: str
     vd: bool
     ni: bool
@@ -1329,11 +1484,17 @@ class Options:
     default="./catalogs/",
 )
 @click.option(
-    "--collectionsspath",
+    "--collectionspath",
     "-clp",
     help="path to collection configuration files",
     default="./collections/",
 )
+@click.option(
+    "--indicatorspath",
+    "-inp",
+    help="path to indicaotr configuration files",
+    default="./indicators/",
+)
 @click.option(
     "--outputpath",
     "-o",
@@ -1358,14 +1519,15 @@ class Options:
     nargs=-1,
 )
 def process_catalogs(
-    catalogspath, collectionsspath, outputpath, vd, ni, tn, collections
+    catalogspath, collectionspath, indicatorspath, outputpath, vd, ni, tn, collections
 ):
     """STAC generator and harvester:
     This library goes over configured endpoints extracting as much information
     as possible and generating a STAC catalog with the information"""
     options = Options(
         catalogspath=catalogspath,
-        collectionsspath=collectionsspath,
+        collectionspath=collectionspath,
+        indicatorspath=indicatorspath,
         outputpath=outputpath,
         vd=vd,
         ni=ni,

{eodash_catalog-0.0.1 → eodash_catalog-0.0.3}/src/eodash_catalog/utils.py

@@ -27,12 +27,7 @@ ISO8601_PERIOD_REGEX = re.compile(
 
 def create_geojson_point(lon, lat):
     point = {"type": "Point", "coordinates": [lon, lat]}
-
-    feature = {"type": "Feature", "geometry": point, "properties": {}}
-
-    feature_collection = {"type": "FeatureCollection", "features": [feature]}
-
-    return feature_collection
+    return {"type": "Feature", "geometry": point, "properties": {}}
 
 
 def retrieveExtentFromWMSWMTS(capabilties_url, layer, wmts=False):