eodash_catalog 0.3.7__tar.gz → 0.3.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.bumpversion.cfg +1 -1
  2. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/Dockerfile +1 -1
  3. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/PKG-INFO +1 -1
  4. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/__about__.py +1 -1
  5. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/endpoints.py +121 -30
  6. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/generate_indicators.py +17 -7
  7. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/sh_endpoint.py +3 -1
  8. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/stac_handling.py +89 -21
  9. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/thumbnails.py +4 -1
  10. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/utils.py +6 -0
  11. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_sh_wms.json +2 -2
  12. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.dockerignore +0 -0
  13. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.github/workflows/ci.yml +0 -0
  14. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.github/workflows/python-publish.yml +0 -0
  15. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.gitignore +0 -0
  16. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.vscode/extensions.json +0 -0
  17. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.vscode/settings.json +0 -0
  18. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/LICENSE.txt +0 -0
  19. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/README.md +0 -0
  20. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/pyproject.toml +0 -0
  21. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/requirements.txt +0 -0
  22. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/ruff.toml +0 -0
  23. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/__init__.py +0 -0
  24. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/duration.py +0 -0
  25. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/__init__.py +0 -0
  26. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/test-data/regional_forecast.json +0 -0
  27. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/test_generate.py +0 -0
  28. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/test_geoparquet.py +0 -0
  29. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-catalogs/testing-json.json +0 -0
  30. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-catalogs/testing.yaml +0 -0
  31. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_CROPOMAT1.yaml +0 -0
  32. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_cmems.json +0 -0
  33. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_cog.json +0 -0
  34. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geodb.yaml +0 -0
  35. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geodb_locations.yaml +0 -0
  36. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geojson.yaml +0 -0
  37. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_locations_processing.json +0 -0
  38. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_process.yaml +0 -0
  39. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_projection.json +0 -0
  40. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_see_solar_energy.yaml +0 -0
  41. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_sh_wms_locations.json +0 -0
  42. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_1.yaml +0 -0
  43. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_1_json.json +0 -0
  44. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_2.yaml +0 -0
  45. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_veda.json +0 -0
  46. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_veda_tiles.json +0 -0
  47. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_wms_no_time.yaml +0 -0
  48. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-indicators/test_indicator.yaml +0 -0
  49. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-layers/baselayers.yaml +0 -0
  50. {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-layers/overlays.yaml +0 -0
@@ -1,5 +1,5 @@
1
1
  [bumpversion]
2
- current_version = 0.3.7
2
+ current_version = 0.3.18
3
3
  commit = True
4
4
  tag = True
5
5
  parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)\.(?P<build>\d+))?
@@ -29,4 +29,4 @@ RUN eodash_catalog --help
29
29
 
30
30
  CMD ["eodash_catalog"]
31
31
 
32
- LABEL version="0.3.7"
32
+ LABEL version="0.3.18"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: eodash_catalog
3
- Version: 0.3.7
3
+ Version: 0.3.18
4
4
  Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
5
5
  Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
6
6
  Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
@@ -1,4 +1,4 @@
1
1
  # SPDX-FileCopyrightText: 2024-present Daniel Santillan <daniel.santillan@eox.at>
2
2
  #
3
3
  # SPDX-License-Identifier: MIT
4
- __version__ = "0.3.7"
4
+ __version__ = "0.3.18"
@@ -9,6 +9,7 @@ from collections.abc import Callable
9
9
  from datetime import datetime, timedelta
10
10
  from itertools import groupby
11
11
  from operator import itemgetter
12
+ from urllib.parse import urlparse
12
13
 
13
14
  import pyarrow.parquet as pq
14
15
  import requests
@@ -20,6 +21,8 @@ from structlog import get_logger
20
21
 
21
22
  from eodash_catalog.sh_endpoint import get_SH_token
22
23
  from eodash_catalog.stac_handling import (
24
+ add_authentication,
25
+ add_base_overlay_info,
23
26
  add_collection_information,
24
27
  add_example_info,
25
28
  add_process_info_child_collection,
@@ -173,7 +176,7 @@ def handle_STAC_based_endpoint(
173
176
  catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
174
177
  )
175
178
  for location in collection_config["Locations"]:
176
- identifier = location.get("Identifier", uuid.uuid4())
179
+ identifier = location.get("Identifier", str(uuid.uuid4()))
177
180
  collection = process_STACAPI_Endpoint(
178
181
  catalog_config=catalog_config,
179
182
  endpoint_config=endpoint_config,
@@ -213,11 +216,13 @@ def handle_STAC_based_endpoint(
213
216
  location["OverwriteBBox"],
214
217
  ]
215
218
  )
219
+ add_collection_information(catalog_config, collection, collection_config)
220
+ add_base_overlay_info(collection, catalog_config, collection_config)
216
221
  update_extents_from_collection_children(root_collection)
217
222
  else:
218
223
  bbox = None
219
- if endpoint_config.get("Bbox"):
220
- bbox = ",".join(map(str, endpoint_config["Bbox"]))
224
+ if endpoint_config.get("OverwriteBBox"):
225
+ bbox = ",".join(map(str, endpoint_config["OverwriteBBox"]))
221
226
  root_collection = process_STACAPI_Endpoint(
222
227
  catalog_config=catalog_config,
223
228
  endpoint_config=endpoint_config,
@@ -392,6 +397,7 @@ def handle_collection_only(
392
397
  properties={},
393
398
  geometry=None,
394
399
  datetime=dt,
400
+ assets={"dummy_asset": Asset(href="")},
395
401
  )
396
402
  link = collection.add_item(item)
397
403
  link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
@@ -464,12 +470,14 @@ def handle_SH_WMS_endpoint(
464
470
  LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
465
471
  add_visualization_info(collection, collection_config, endpoint_config)
466
472
  add_process_info_child_collection(collection, catalog_config, collection_config)
473
+ add_collection_information(catalog_config, collection, collection_config)
474
+ add_base_overlay_info(collection, catalog_config, collection_config)
467
475
  update_extents_from_collection_children(root_collection)
468
476
  else:
469
477
  # if locations are not provided, treat the collection as a
470
478
  # general proxy to the sentinel hub layer
471
479
  datetimes = get_collection_datetimes_from_config(endpoint_config)
472
- bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
480
+ bbox = endpoint_config.get("OverwriteBBox", [-180, -85, 180, 85])
473
481
  items = []
474
482
  for dt in datetimes:
475
483
  item = Item(
@@ -910,6 +918,7 @@ def handle_GeoDB_endpoint(
910
918
  link.extra_fields["country"] = country
911
919
  link.extra_fields["name"] = city
912
920
  add_collection_information(catalog_config, locations_collection, collection_config)
921
+ add_base_overlay_info(locations_collection, catalog_config, collection_config)
913
922
 
914
923
  if "yAxis" not in collection_config:
915
924
  # fetch yAxis and store it to data, preventing need to save it per dataset in yml
@@ -948,7 +957,8 @@ def handle_SH_endpoint(
948
957
  ) -> Collection:
949
958
  token = get_SH_token(endpoint_config)
950
959
  headers = {"Authorization": f"Bearer {token}"}
951
- endpoint_config["EndPoint"] = "https://services.sentinel-hub.com/api/v1/catalog/1.0.0/"
960
+ endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
961
+ endpoint_config["EndPoint"] = f"https://{endpoint_url_parts.netloc}/api/v1/catalog/1.0.0/"
952
962
  # Overwrite collection id with type, such as ZARR or BYOC
953
963
  if endpoint_config.get("Type"):
954
964
  endpoint_config["CollectionId"] = (
@@ -985,12 +995,14 @@ def handle_WMS_endpoint(
985
995
  # some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
986
996
  # have to process full service capabilities XML
987
997
  capabilities_url = endpoint_config["EndPoint"]
988
- spatial_extent, datetimes = retrieveExtentFromWMSWMTS(
998
+ spatial_extent, datetimes_retrieved = retrieveExtentFromWMSWMTS(
989
999
  capabilities_url,
990
1000
  endpoint_config["LayerId"],
991
1001
  version=endpoint_config.get("Version", "1.1.1"),
992
1002
  wmts=wmts,
993
1003
  )
1004
+ if datetimes_retrieved:
1005
+ datetimes = datetimes_retrieved
994
1006
  # optionally filter time results
995
1007
  if query := endpoint_config.get("Query"):
996
1008
  datetimes = filter_time_entries(datetimes, query)
@@ -1054,6 +1066,10 @@ def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
1054
1066
  color_formula = ""
1055
1067
  if endpoint_config.get("ColorFormula"):
1056
1068
  color_formula = "&color_formula={}".format(endpoint_config["ColorFormula"])
1069
+ rescale = ""
1070
+ if endpoint_config.get("Rescale"):
1071
+ for rescale in endpoint_config["Rescale"]:
1072
+ rescale += f"&rescale={rescale}"
1057
1073
  no_data = ""
1058
1074
  if endpoint_config.get("NoData"):
1059
1075
  no_data = "&no_data={}".format(endpoint_config["NoData"])
@@ -1061,7 +1077,7 @@ def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
1061
1077
  target_url_base = endpoint_config["EndPoint"].replace("/stac/", "")
1062
1078
  target_url = (
1063
1079
  f"{target_url_base}/raster/collections/{collection}/items/{item}"
1064
- f"/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}"
1080
+ f"/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}{rescale}"
1065
1081
  )
1066
1082
  return target_url
1067
1083
 
@@ -1111,9 +1127,10 @@ def add_visualization_info(
1111
1127
 
1112
1128
  if dimensions != {}:
1113
1129
  extra_fields["wms:dimensions"] = dimensions
1130
+ endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
1114
1131
  link = Link(
1115
1132
  rel="wms",
1116
- target=f"https://services.sentinel-hub.com/ogc/wms/{instanceId}",
1133
+ target=f"https://{endpoint_url_parts.netloc}/ogc/wms/{instanceId}",
1117
1134
  media_type=(endpoint_config.get("MimeType", "image/png")),
1118
1135
  title=collection_config["Name"],
1119
1136
  extra_fields=extra_fields,
@@ -1130,6 +1147,14 @@ def add_visualization_info(
1130
1147
  "role": ["data"],
1131
1148
  }
1132
1149
  )
1150
+ if collection_config.get("EodashIdentifier") == "FNF":
1151
+ extra_fields.update(
1152
+ {
1153
+ "wms:layers": endpoint_config.get("LayerId", "").replace(
1154
+ "{time}", (datetimes is not None and str(datetimes[0].year)) or "2020"
1155
+ ),
1156
+ }
1157
+ )
1133
1158
  dimensions = {}
1134
1159
  if dimensions_config := endpoint_config.get("Dimensions", {}):
1135
1160
  for key, value in dimensions_config.items():
@@ -1203,28 +1228,6 @@ def add_visualization_info(
1203
1228
  link,
1204
1229
  )
1205
1230
  stac_object.add_link(link)
1206
- elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
1207
- target_url = "{}".format(endpoint_config.get("EndPoint"))
1208
- # custom time just for this special case as a default for collection wmts
1209
- time = None
1210
- if datetimes is not None:
1211
- time = datetimes[0]
1212
- extra_fields.update(
1213
- {
1214
- "wmts:layer": endpoint_config.get("LayerId", "").replace(
1215
- "{time}", (time and str(time.year)) or "2017"
1216
- )
1217
- }
1218
- )
1219
- stac_object.add_link(
1220
- Link(
1221
- rel="wmts",
1222
- target=target_url,
1223
- media_type="image/png",
1224
- title="wmts capabilities",
1225
- extra_fields=extra_fields,
1226
- )
1227
- )
1228
1231
  elif endpoint_config["Name"] == "xcube":
1229
1232
  if endpoint_config["Type"] == "zarr":
1230
1233
  # either preset ColormapName of left as a template
@@ -1483,3 +1486,91 @@ def handle_raw_source(
1483
1486
 
1484
1487
  add_collection_information(catalog_config, collection, collection_config)
1485
1488
  return collection
1489
+
1490
+
1491
+ def handle_vector_tile_source(
1492
+ catalog_config: dict,
1493
+ endpoint_config: dict,
1494
+ collection_config: dict,
1495
+ coll_path_rel_to_root_catalog: str,
1496
+ catalog: Catalog,
1497
+ options: Options,
1498
+ ) -> Collection:
1499
+ collection = get_or_create_collection(
1500
+ catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
1501
+ )
1502
+ coll_path_rel_to_root_catalog = f'{coll_path_rel_to_root_catalog}/{collection_config["Name"]}'
1503
+ if len(endpoint_config.get("TimeEntries", [])) > 0:
1504
+ items = []
1505
+ style_link = None
1506
+ for time_entry in endpoint_config["TimeEntries"]:
1507
+ # create Item for each time entry
1508
+ media_type = "application/vnd.mapbox-vector-tile"
1509
+ style_type = "text/vector-styles"
1510
+ bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
1511
+ dt = parse_datestring_to_tz_aware_datetime(time_entry["Time"])
1512
+
1513
+ item = Item(
1514
+ id=format_datetime_to_isostring_zulu(dt),
1515
+ bbox=bbox,
1516
+ properties={},
1517
+ geometry=create_geometry_from_bbox(bbox),
1518
+ datetime=dt,
1519
+ extra_fields={},
1520
+ assets={"dummy_asset": Asset(href="")},
1521
+ )
1522
+ extra_fields_link = {}
1523
+ add_authentication(item, time_entry["Url"], extra_fields_link)
1524
+ # add mapbox vector tile link
1525
+ identifier = str(uuid.uuid4())
1526
+ extra_fields_link["key"] = identifier
1527
+ if vector_tile_id_property := endpoint_config.get("idProperty"):
1528
+ extra_fields_link["idProperty"] = vector_tile_id_property
1529
+ link = Link(
1530
+ rel="vector-tile",
1531
+ target=time_entry["Url"],
1532
+ media_type=media_type,
1533
+ title=collection_config["Name"],
1534
+ extra_fields=extra_fields_link,
1535
+ )
1536
+ add_projection_info(
1537
+ endpoint_config,
1538
+ link,
1539
+ )
1540
+ item.add_link(link)
1541
+ add_projection_info(
1542
+ endpoint_config,
1543
+ item,
1544
+ )
1545
+ if endpoint_config.get("Attribution"):
1546
+ item.stac_extensions.append(
1547
+ "https://stac-extensions.github.io/attribution/v0.1.0/schema.json"
1548
+ )
1549
+ item.extra_fields["attribution"] = endpoint_config["Attribution"]
1550
+ # add style
1551
+ if ep_st := endpoint_config.get("Style"):
1552
+ style_link = Link(
1553
+ rel="style",
1554
+ target=ep_st
1555
+ if ep_st.startswith("http")
1556
+ else f"{catalog_config['assets_endpoint']}/{ep_st}",
1557
+ media_type=style_type,
1558
+ extra_fields={"links:keys": [identifier]},
1559
+ )
1560
+ item.add_link(style_link)
1561
+ items.append(item)
1562
+
1563
+ save_items(
1564
+ collection,
1565
+ items,
1566
+ options.outputpath,
1567
+ catalog_config["id"],
1568
+ coll_path_rel_to_root_catalog,
1569
+ options.gp,
1570
+ )
1571
+
1572
+ else:
1573
+ LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
1574
+
1575
+ add_collection_information(catalog_config, collection, collection_config)
1576
+ return collection
@@ -24,6 +24,7 @@ from eodash_catalog.endpoints import (
24
24
  handle_raw_source,
25
25
  handle_SH_endpoint,
26
26
  handle_SH_WMS_endpoint,
27
+ handle_vector_tile_source,
27
28
  handle_VEDA_endpoint,
28
29
  handle_WMS_endpoint,
29
30
  handle_xcube_endpoint,
@@ -292,16 +293,14 @@ def process_collection_file(
292
293
  catalog,
293
294
  options,
294
295
  )
295
- elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
296
- # somewhat one off creation of individual WMTS layers as individual items
297
- collection = handle_WMS_endpoint(
296
+ elif endpoint_config["Name"] == "VectorTile source":
297
+ collection = handle_vector_tile_source(
298
298
  catalog_config,
299
299
  endpoint_config,
300
300
  collection_config,
301
301
  coll_path_rel_to_root_catalog,
302
302
  catalog,
303
303
  options,
304
- wmts=True,
305
304
  )
306
305
  elif endpoint_config["Name"] == "Collection-only":
307
306
  collection = handle_collection_only(
@@ -360,9 +359,11 @@ def process_collection_file(
360
359
  countries.extend(sub_coll_def["Country"])
361
360
  else:
362
361
  countries.append(sub_coll_def["Country"])
363
- coll_path_rel_to_root_catalog = (
364
- f"{coll_path_rel_to_root_catalog}/{sub_coll_def['Collection']}"
365
- )
362
+ # commented out intentionally, because otherwise paths further down did
363
+ # not match, parquet file was one level deeper
364
+ # coll_path_rel_to_root_catalog = (
365
+ # f"{coll_path_rel_to_root_catalog}/{sub_coll_def['Collection']}"
366
+ # )
366
367
  process_collection_file(
367
368
  catalog_config,
368
369
  "{}/{}".format(options.collectionspath, sub_coll_def["Collection"]),
@@ -421,6 +422,7 @@ def process_collection_file(
421
422
  add_collection_information(catalog_config, parent_collection, collection_config, True)
422
423
  add_process_info(parent_collection, catalog_config, collection_config)
423
424
  update_extents_from_collection_children(parent_collection)
425
+ add_base_overlay_info(parent_collection, catalog_config, collection_config)
424
426
  # Fill summaries for locations
425
427
  parent_collection.summaries = Summaries(
426
428
  {
@@ -456,12 +458,20 @@ def add_to_catalog(
456
458
  link.extra_fields["endpointtype"] = endpoint["Name"]
457
459
  if collection_config.get("Subtitle"):
458
460
  link.extra_fields["subtitle"] = collection_config["Subtitle"]
461
+ if collection_config.get("ShortDescription"):
462
+ link.extra_fields["shortdescription"] = collection_config["ShortDescription"]
459
463
  link.extra_fields["title"] = collection.title
460
464
  if collection_config.get("EodashIdentifier"):
461
465
  link.extra_fields["code"] = collection_config["EodashIdentifier"]
462
466
  link.extra_fields["id"] = collection_config["Name"]
463
467
  if collection_config.get("Themes"):
464
468
  link.extra_fields["themes"] = collection_config["Themes"]
469
+ if collection_config.get("Provider"):
470
+ # get all provider names
471
+ link.extra_fields["providers"] = [
472
+ provider.get("Name") or provider.get("Url")
473
+ for provider in collection_config["Provider"]
474
+ ]
465
475
  # Check for summaries and bubble up info
466
476
  if disable:
467
477
  link.extra_fields["roles"] = ["disable"]
@@ -1,10 +1,10 @@
1
1
  import os
2
2
  import time
3
+ from urllib.parse import urlparse
3
4
 
4
5
  from oauthlib.oauth2 import BackendApplicationClient
5
6
  from requests_oauthlib import OAuth2Session
6
7
 
7
- SH_TOKEN_URL = "https://services.sentinel-hub.com/oauth/token"
8
8
  _token_cache: dict[str, dict] = {}
9
9
 
10
10
 
@@ -24,6 +24,8 @@ def get_SH_token(endpoint_config: dict) -> str:
24
24
  client = BackendApplicationClient(client_id=client_id)
25
25
  oauth = OAuth2Session(client=client)
26
26
  # Get token for the session
27
+ endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
28
+ SH_TOKEN_URL = f"https://{endpoint_url_parts.netloc}/oauth/token"
27
29
  token = oauth.fetch_token(
28
30
  token_url=SH_TOKEN_URL,
29
31
  client_secret=client_secret,
@@ -1,4 +1,6 @@
1
+ import uuid
1
2
  from datetime import datetime, timezone
3
+ from urllib.parse import parse_qs, urlparse
2
4
 
3
5
  import requests
4
6
  import spdx_lookup as lookup
@@ -122,11 +124,14 @@ def create_service_link(endpoint_config: dict, catalog_config: dict) -> Link:
122
124
  return sl
123
125
 
124
126
 
125
- def create_web_map_link(layer_config: dict, role: str) -> Link:
127
+ def create_web_map_link(
128
+ collection: Collection, catalog_config: dict, layer_config: dict, role: str
129
+ ) -> Link:
126
130
  extra_fields = {
127
131
  "roles": [role],
128
132
  "id": layer_config["id"],
129
133
  }
134
+ media_type = (layer_config.get("media_type", "image/png"),)
130
135
  if layer_config.get("default"):
131
136
  extra_fields["roles"].append("default")
132
137
  if layer_config.get("visible"):
@@ -146,12 +151,32 @@ def create_web_map_link(layer_config: dict, role: str) -> Link:
146
151
  extra_fields["wmts:layer"] = layer_config["layer"]
147
152
  if layer_config.get("dimensions"):
148
153
  extra_fields["wmts:dimensions"] = layer_config["dimensions"]
154
+ case "vector-tile":
155
+ identifier = str(uuid.uuid4())
156
+ extra_fields["key"] = identifier
157
+ media_type = "application/vnd.mapbox-vector-tile"
158
+ if vector_tile_id_property := layer_config.get("idProperty"):
159
+ extra_fields["idProperty"] = vector_tile_id_property
160
+ if ep_st := layer_config.get("Style"):
161
+ style_link = Link(
162
+ rel="style",
163
+ target=ep_st
164
+ if ep_st.startswith("http")
165
+ else f"{catalog_config['assets_endpoint']}/{ep_st}",
166
+ media_type="text/vector-styles",
167
+ extra_fields={"links:keys": [identifier]},
168
+ )
169
+ collection.add_link(style_link)
170
+ add_authentication(collection, layer_config["url"], extra_fields)
171
+
149
172
  if layer_config.get("Attribution"):
150
173
  extra_fields["attribution"] = layer_config["Attribution"]
174
+ if layer_config.get("Colorlegend"):
175
+ extra_fields["eox:colorlegend"] = layer_config["Colorlegend"]
151
176
  wml = Link(
152
177
  rel=layer_config["protocol"],
153
178
  target=layer_config["url"],
154
- media_type=layer_config.get("media_type", "image/png"),
179
+ media_type=media_type,
155
180
  title=layer_config["name"],
156
181
  extra_fields=extra_fields,
157
182
  )
@@ -288,6 +313,10 @@ def add_collection_information(
288
313
 
289
314
  if collection_config.get("Subtitle"):
290
315
  collection.extra_fields["subtitle"] = collection_config["Subtitle"]
316
+
317
+ if collection_config.get("ShortDescription"):
318
+ collection.extra_fields["shortdescription"] = collection_config["ShortDescription"]
319
+
291
320
  if collection_config.get("Legend"):
292
321
  collection.add_asset(
293
322
  "legend",
@@ -297,28 +326,42 @@ def add_collection_information(
297
326
  roles=["metadata"],
298
327
  ),
299
328
  )
300
- if collection_config.get("Story"):
301
- collection.add_asset(
302
- "story",
303
- Asset(
304
- href=f'{catalog_config["assets_endpoint"]}/{collection_config["Story"]}',
305
- media_type="text/markdown",
306
- roles=["metadata"],
307
- ),
308
- )
329
+ if stories := collection_config.get("Stories"):
330
+ for story in stories:
331
+ story_url = story.get("Url")
332
+ if not story_url.startswith("http"):
333
+ story_url = f'{catalog_config.get("stories_endpoint")}/{story_url}'
334
+ parsed_url = urlparse(story_url)
335
+ # check if it is URL with a query parameter id=story-identifier
336
+ if parsed_url.query and len(parse_qs(parsed_url.query).get("id")) > 0:
337
+ story_id = parse_qs(parsed_url.query).get("id")[0]
338
+ else:
339
+ story_id = parsed_url.path.rsplit("/")[-1].replace(".md", "").replace(".MD", "")
340
+ collection.add_asset(
341
+ story_id,
342
+ Asset(
343
+ title=story.get("Name"),
344
+ href=story_url,
345
+ media_type="text/markdown",
346
+ roles=["metadata", "story"],
347
+ ),
348
+ )
309
349
  if collection_config.get("Image"):
350
+ # Check if absolute URL or relative path
351
+ if collection_config["Image"].startswith("http"):
352
+ image_url = collection_config["Image"]
353
+ else:
354
+ image_url = f'{catalog_config["assets_endpoint"]}/{collection_config["Image"]}'
310
355
  collection.add_asset(
311
356
  "thumbnail",
312
357
  Asset(
313
- href=f'{catalog_config["assets_endpoint"]}/{collection_config["Image"]}',
358
+ href=image_url,
314
359
  media_type="image/png",
315
360
  roles=["thumbnail"],
316
361
  ),
317
362
  )
318
363
  # Bubble up thumbnail to extra fields
319
- collection.extra_fields["thumbnail"] = (
320
- f'{catalog_config["assets_endpoint"]}/' f'{collection_config["Image"]}'
321
- )
364
+ collection.extra_fields["thumbnail"] = image_url
322
365
  # Add extra fields to collection if available
323
366
  add_extra_fields(collection, collection_config, is_root_collection)
324
367
 
@@ -473,23 +516,31 @@ def add_base_overlay_info(
473
516
  collection: Collection, catalog_config: dict, collection_config: dict
474
517
  ) -> None:
475
518
  # add custom baselayers specially for this indicator
476
- if collection_config.get("BaseLayers"):
519
+ if "BaseLayers" in collection_config:
477
520
  for layer in collection_config["BaseLayers"]:
478
- collection.add_link(create_web_map_link(layer, role="baselayer"))
521
+ collection.add_link(
522
+ create_web_map_link(collection, catalog_config, layer, role="baselayer")
523
+ )
479
524
  # alternatively use default base layers defined
480
525
  elif catalog_config.get("default_base_layers"):
481
526
  base_layers = read_config_file(catalog_config["default_base_layers"])
482
527
  for layer in base_layers:
483
- collection.add_link(create_web_map_link(layer, role="baselayer"))
528
+ collection.add_link(
529
+ create_web_map_link(collection, catalog_config, layer, role="baselayer")
530
+ )
484
531
  # add custom overlays just for this indicator
485
- if collection_config.get("OverlayLayers"):
532
+ if "OverlayLayers" in collection_config:
486
533
  for layer in collection_config["OverlayLayers"]:
487
- collection.add_link(create_web_map_link(layer, role="overlay"))
534
+ collection.add_link(
535
+ create_web_map_link(collection, catalog_config, layer, role="overlay")
536
+ )
488
537
  # check if default overlay layers defined
489
538
  elif catalog_config.get("default_overlay_layers"):
490
539
  overlay_layers = read_config_file(catalog_config["default_overlay_layers"])
491
540
  for layer in overlay_layers:
492
- collection.add_link(create_web_map_link(layer, role="overlay"))
541
+ collection.add_link(
542
+ create_web_map_link(collection, catalog_config, layer, role="overlay")
543
+ )
493
544
 
494
545
 
495
546
  def add_extra_fields(
@@ -579,3 +630,20 @@ def add_projection_info(
579
630
  stac_object.extra_fields["eodash:proj4_def"] = proj
580
631
  else:
581
632
  raise Exception(f"Incorrect type of proj definition {proj}")
633
+
634
+
635
+ def add_authentication(stac_object: Item | Collection | Catalog, url: str, extra_fields_link: dict):
636
+ if "api.mapbox" in url:
637
+ # add authentication info
638
+ auth_extension = "https://stac-extensions.github.io/authentication/v1.1.0/schema.json"
639
+ if auth_extension not in stac_object.stac_extensions:
640
+ stac_object.stac_extensions.append(auth_extension)
641
+ stac_object.extra_fields["auth:schemes"] = {
642
+ "mapboxauth": {
643
+ "type": "apiKey",
644
+ "name": "access_token",
645
+ "in": "query",
646
+ }
647
+ }
648
+ extra_fields_link["auth:refs"] = ["mapboxauth"]
649
+ pass
@@ -1,6 +1,7 @@
1
1
  import os
2
2
  import re
3
3
  from pathlib import Path
4
+ from urllib.parse import urlparse
4
5
 
5
6
  import requests
6
7
  from pystac import (
@@ -46,7 +47,9 @@ def generate_thumbnail(
46
47
  # if it is start and end datetime have to exist
47
48
  if item_datetime:
48
49
  time = format_datetime_to_isostring_zulu(item_datetime)
49
- url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format(
50
+ endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
51
+ url = "https://{}/ogc/wms/{}?{}&layers={}&time={}&{}".format(
52
+ endpoint_url_parts,
50
53
  instanceId,
51
54
  wms_config,
52
55
  endpoint_config["LayerId"],
@@ -328,6 +328,7 @@ def add_single_item_if_collection_empty(endpoint_config: dict, collection: Colle
328
328
  datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
329
329
  start_datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
330
330
  end_datetime=datetime.now(tz=pytztimezone("UTC")),
331
+ assets={"dummy_asset": Asset(href="")},
331
332
  )
332
333
  collection.add_item(item)
333
334
  if not endpoint_config.get("OverwriteBBox"):
@@ -468,6 +469,11 @@ def extract_extent_from_geoparquet(table) -> tuple[TemporalExtent, SpatialExtent
468
469
  # fallback to start_datetime
469
470
  min_datetime = pc.min(table["start_datetime"]).as_py()
470
471
  max_datetime = pc.max(table["start_datetime"]).as_py()
472
+ # Making sure time extent is timezone aware
473
+ if min_datetime and min_datetime.tzinfo is None:
474
+ min_datetime = min_datetime.replace(tzinfo=timezone.utc)
475
+ if max_datetime and max_datetime.tzinfo is None:
476
+ max_datetime = max_datetime.replace(tzinfo=timezone.utc)
471
477
  temporal = TemporalExtent([min_datetime, max_datetime])
472
478
  geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
473
479
  bbox = sgeom.MultiPolygon(geoms).bounds
@@ -33,7 +33,7 @@
33
33
  "Name": "Sentinel Hub WMS",
34
34
  "CollectionId": "sentinel-1-grd",
35
35
  "LayerId": "SENTINEL_1_IW_VV",
36
- "Bbox": [
36
+ "OverwriteBBox": [
37
37
  101.938,
38
38
  11.945,
39
39
  106.37,
@@ -65,4 +65,4 @@
65
65
  "Url": "https://www.copernicus.eu/en/access-data/conventional-data-access-hubs"
66
66
  }
67
67
  ]
68
- }
68
+ }