eodash_catalog 0.3.7__tar.gz → 0.3.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.bumpversion.cfg +1 -1
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/Dockerfile +1 -1
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/PKG-INFO +1 -1
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/__about__.py +1 -1
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/endpoints.py +121 -30
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/generate_indicators.py +17 -7
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/sh_endpoint.py +3 -1
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/stac_handling.py +89 -21
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/thumbnails.py +4 -1
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/utils.py +6 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_sh_wms.json +2 -2
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.dockerignore +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.github/workflows/ci.yml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.github/workflows/python-publish.yml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.gitignore +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.vscode/extensions.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/.vscode/settings.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/LICENSE.txt +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/README.md +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/pyproject.toml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/requirements.txt +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/ruff.toml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/__init__.py +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/src/eodash_catalog/duration.py +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/__init__.py +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/test-data/regional_forecast.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/test_generate.py +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/test_geoparquet.py +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-catalogs/testing-json.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-catalogs/testing.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_CROPOMAT1.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_cmems.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_cog.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geodb.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geodb_locations.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geojson.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_locations_processing.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_process.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_projection.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_see_solar_energy.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_sh_wms_locations.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_1.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_1_json.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_2.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_veda.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_veda_tiles.json +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_wms_no_time.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-indicators/test_indicator.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-layers/baselayers.yaml +0 -0
- {eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-layers/overlays.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: eodash_catalog
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.18
|
|
4
4
|
Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
|
|
5
5
|
Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
|
|
@@ -9,6 +9,7 @@ from collections.abc import Callable
|
|
|
9
9
|
from datetime import datetime, timedelta
|
|
10
10
|
from itertools import groupby
|
|
11
11
|
from operator import itemgetter
|
|
12
|
+
from urllib.parse import urlparse
|
|
12
13
|
|
|
13
14
|
import pyarrow.parquet as pq
|
|
14
15
|
import requests
|
|
@@ -20,6 +21,8 @@ from structlog import get_logger
|
|
|
20
21
|
|
|
21
22
|
from eodash_catalog.sh_endpoint import get_SH_token
|
|
22
23
|
from eodash_catalog.stac_handling import (
|
|
24
|
+
add_authentication,
|
|
25
|
+
add_base_overlay_info,
|
|
23
26
|
add_collection_information,
|
|
24
27
|
add_example_info,
|
|
25
28
|
add_process_info_child_collection,
|
|
@@ -173,7 +176,7 @@ def handle_STAC_based_endpoint(
|
|
|
173
176
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
174
177
|
)
|
|
175
178
|
for location in collection_config["Locations"]:
|
|
176
|
-
identifier = location.get("Identifier", uuid.uuid4())
|
|
179
|
+
identifier = location.get("Identifier", str(uuid.uuid4()))
|
|
177
180
|
collection = process_STACAPI_Endpoint(
|
|
178
181
|
catalog_config=catalog_config,
|
|
179
182
|
endpoint_config=endpoint_config,
|
|
@@ -213,11 +216,13 @@ def handle_STAC_based_endpoint(
|
|
|
213
216
|
location["OverwriteBBox"],
|
|
214
217
|
]
|
|
215
218
|
)
|
|
219
|
+
add_collection_information(catalog_config, collection, collection_config)
|
|
220
|
+
add_base_overlay_info(collection, catalog_config, collection_config)
|
|
216
221
|
update_extents_from_collection_children(root_collection)
|
|
217
222
|
else:
|
|
218
223
|
bbox = None
|
|
219
|
-
if endpoint_config.get("
|
|
220
|
-
bbox = ",".join(map(str, endpoint_config["
|
|
224
|
+
if endpoint_config.get("OverwriteBBox"):
|
|
225
|
+
bbox = ",".join(map(str, endpoint_config["OverwriteBBox"]))
|
|
221
226
|
root_collection = process_STACAPI_Endpoint(
|
|
222
227
|
catalog_config=catalog_config,
|
|
223
228
|
endpoint_config=endpoint_config,
|
|
@@ -392,6 +397,7 @@ def handle_collection_only(
|
|
|
392
397
|
properties={},
|
|
393
398
|
geometry=None,
|
|
394
399
|
datetime=dt,
|
|
400
|
+
assets={"dummy_asset": Asset(href="")},
|
|
395
401
|
)
|
|
396
402
|
link = collection.add_item(item)
|
|
397
403
|
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
@@ -464,12 +470,14 @@ def handle_SH_WMS_endpoint(
|
|
|
464
470
|
LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
|
|
465
471
|
add_visualization_info(collection, collection_config, endpoint_config)
|
|
466
472
|
add_process_info_child_collection(collection, catalog_config, collection_config)
|
|
473
|
+
add_collection_information(catalog_config, collection, collection_config)
|
|
474
|
+
add_base_overlay_info(collection, catalog_config, collection_config)
|
|
467
475
|
update_extents_from_collection_children(root_collection)
|
|
468
476
|
else:
|
|
469
477
|
# if locations are not provided, treat the collection as a
|
|
470
478
|
# general proxy to the sentinel hub layer
|
|
471
479
|
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
472
|
-
bbox = endpoint_config.get("
|
|
480
|
+
bbox = endpoint_config.get("OverwriteBBox", [-180, -85, 180, 85])
|
|
473
481
|
items = []
|
|
474
482
|
for dt in datetimes:
|
|
475
483
|
item = Item(
|
|
@@ -910,6 +918,7 @@ def handle_GeoDB_endpoint(
|
|
|
910
918
|
link.extra_fields["country"] = country
|
|
911
919
|
link.extra_fields["name"] = city
|
|
912
920
|
add_collection_information(catalog_config, locations_collection, collection_config)
|
|
921
|
+
add_base_overlay_info(locations_collection, catalog_config, collection_config)
|
|
913
922
|
|
|
914
923
|
if "yAxis" not in collection_config:
|
|
915
924
|
# fetch yAxis and store it to data, preventing need to save it per dataset in yml
|
|
@@ -948,7 +957,8 @@ def handle_SH_endpoint(
|
|
|
948
957
|
) -> Collection:
|
|
949
958
|
token = get_SH_token(endpoint_config)
|
|
950
959
|
headers = {"Authorization": f"Bearer {token}"}
|
|
951
|
-
endpoint_config["EndPoint"]
|
|
960
|
+
endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
|
|
961
|
+
endpoint_config["EndPoint"] = f"https://{endpoint_url_parts.netloc}/api/v1/catalog/1.0.0/"
|
|
952
962
|
# Overwrite collection id with type, such as ZARR or BYOC
|
|
953
963
|
if endpoint_config.get("Type"):
|
|
954
964
|
endpoint_config["CollectionId"] = (
|
|
@@ -985,12 +995,14 @@ def handle_WMS_endpoint(
|
|
|
985
995
|
# some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
|
|
986
996
|
# have to process full service capabilities XML
|
|
987
997
|
capabilities_url = endpoint_config["EndPoint"]
|
|
988
|
-
spatial_extent,
|
|
998
|
+
spatial_extent, datetimes_retrieved = retrieveExtentFromWMSWMTS(
|
|
989
999
|
capabilities_url,
|
|
990
1000
|
endpoint_config["LayerId"],
|
|
991
1001
|
version=endpoint_config.get("Version", "1.1.1"),
|
|
992
1002
|
wmts=wmts,
|
|
993
1003
|
)
|
|
1004
|
+
if datetimes_retrieved:
|
|
1005
|
+
datetimes = datetimes_retrieved
|
|
994
1006
|
# optionally filter time results
|
|
995
1007
|
if query := endpoint_config.get("Query"):
|
|
996
1008
|
datetimes = filter_time_entries(datetimes, query)
|
|
@@ -1054,6 +1066,10 @@ def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
|
|
|
1054
1066
|
color_formula = ""
|
|
1055
1067
|
if endpoint_config.get("ColorFormula"):
|
|
1056
1068
|
color_formula = "&color_formula={}".format(endpoint_config["ColorFormula"])
|
|
1069
|
+
rescale = ""
|
|
1070
|
+
if endpoint_config.get("Rescale"):
|
|
1071
|
+
for rescale in endpoint_config["Rescale"]:
|
|
1072
|
+
rescale += f"&rescale={rescale}"
|
|
1057
1073
|
no_data = ""
|
|
1058
1074
|
if endpoint_config.get("NoData"):
|
|
1059
1075
|
no_data = "&no_data={}".format(endpoint_config["NoData"])
|
|
@@ -1061,7 +1077,7 @@ def generate_veda_tiles_link(endpoint_config: dict, item: str | None) -> str:
|
|
|
1061
1077
|
target_url_base = endpoint_config["EndPoint"].replace("/stac/", "")
|
|
1062
1078
|
target_url = (
|
|
1063
1079
|
f"{target_url_base}/raster/collections/{collection}/items/{item}"
|
|
1064
|
-
f"/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}"
|
|
1080
|
+
f"/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{assets}{color_formula}{no_data}{rescale}"
|
|
1065
1081
|
)
|
|
1066
1082
|
return target_url
|
|
1067
1083
|
|
|
@@ -1111,9 +1127,10 @@ def add_visualization_info(
|
|
|
1111
1127
|
|
|
1112
1128
|
if dimensions != {}:
|
|
1113
1129
|
extra_fields["wms:dimensions"] = dimensions
|
|
1130
|
+
endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
|
|
1114
1131
|
link = Link(
|
|
1115
1132
|
rel="wms",
|
|
1116
|
-
target=f"https://
|
|
1133
|
+
target=f"https://{endpoint_url_parts.netloc}/ogc/wms/{instanceId}",
|
|
1117
1134
|
media_type=(endpoint_config.get("MimeType", "image/png")),
|
|
1118
1135
|
title=collection_config["Name"],
|
|
1119
1136
|
extra_fields=extra_fields,
|
|
@@ -1130,6 +1147,14 @@ def add_visualization_info(
|
|
|
1130
1147
|
"role": ["data"],
|
|
1131
1148
|
}
|
|
1132
1149
|
)
|
|
1150
|
+
if collection_config.get("EodashIdentifier") == "FNF":
|
|
1151
|
+
extra_fields.update(
|
|
1152
|
+
{
|
|
1153
|
+
"wms:layers": endpoint_config.get("LayerId", "").replace(
|
|
1154
|
+
"{time}", (datetimes is not None and str(datetimes[0].year)) or "2020"
|
|
1155
|
+
),
|
|
1156
|
+
}
|
|
1157
|
+
)
|
|
1133
1158
|
dimensions = {}
|
|
1134
1159
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
1135
1160
|
for key, value in dimensions_config.items():
|
|
@@ -1203,28 +1228,6 @@ def add_visualization_info(
|
|
|
1203
1228
|
link,
|
|
1204
1229
|
)
|
|
1205
1230
|
stac_object.add_link(link)
|
|
1206
|
-
elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
|
|
1207
|
-
target_url = "{}".format(endpoint_config.get("EndPoint"))
|
|
1208
|
-
# custom time just for this special case as a default for collection wmts
|
|
1209
|
-
time = None
|
|
1210
|
-
if datetimes is not None:
|
|
1211
|
-
time = datetimes[0]
|
|
1212
|
-
extra_fields.update(
|
|
1213
|
-
{
|
|
1214
|
-
"wmts:layer": endpoint_config.get("LayerId", "").replace(
|
|
1215
|
-
"{time}", (time and str(time.year)) or "2017"
|
|
1216
|
-
)
|
|
1217
|
-
}
|
|
1218
|
-
)
|
|
1219
|
-
stac_object.add_link(
|
|
1220
|
-
Link(
|
|
1221
|
-
rel="wmts",
|
|
1222
|
-
target=target_url,
|
|
1223
|
-
media_type="image/png",
|
|
1224
|
-
title="wmts capabilities",
|
|
1225
|
-
extra_fields=extra_fields,
|
|
1226
|
-
)
|
|
1227
|
-
)
|
|
1228
1231
|
elif endpoint_config["Name"] == "xcube":
|
|
1229
1232
|
if endpoint_config["Type"] == "zarr":
|
|
1230
1233
|
# either preset ColormapName of left as a template
|
|
@@ -1483,3 +1486,91 @@ def handle_raw_source(
|
|
|
1483
1486
|
|
|
1484
1487
|
add_collection_information(catalog_config, collection, collection_config)
|
|
1485
1488
|
return collection
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
def handle_vector_tile_source(
|
|
1492
|
+
catalog_config: dict,
|
|
1493
|
+
endpoint_config: dict,
|
|
1494
|
+
collection_config: dict,
|
|
1495
|
+
coll_path_rel_to_root_catalog: str,
|
|
1496
|
+
catalog: Catalog,
|
|
1497
|
+
options: Options,
|
|
1498
|
+
) -> Collection:
|
|
1499
|
+
collection = get_or_create_collection(
|
|
1500
|
+
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
1501
|
+
)
|
|
1502
|
+
coll_path_rel_to_root_catalog = f'{coll_path_rel_to_root_catalog}/{collection_config["Name"]}'
|
|
1503
|
+
if len(endpoint_config.get("TimeEntries", [])) > 0:
|
|
1504
|
+
items = []
|
|
1505
|
+
style_link = None
|
|
1506
|
+
for time_entry in endpoint_config["TimeEntries"]:
|
|
1507
|
+
# create Item for each time entry
|
|
1508
|
+
media_type = "application/vnd.mapbox-vector-tile"
|
|
1509
|
+
style_type = "text/vector-styles"
|
|
1510
|
+
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
1511
|
+
dt = parse_datestring_to_tz_aware_datetime(time_entry["Time"])
|
|
1512
|
+
|
|
1513
|
+
item = Item(
|
|
1514
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
1515
|
+
bbox=bbox,
|
|
1516
|
+
properties={},
|
|
1517
|
+
geometry=create_geometry_from_bbox(bbox),
|
|
1518
|
+
datetime=dt,
|
|
1519
|
+
extra_fields={},
|
|
1520
|
+
assets={"dummy_asset": Asset(href="")},
|
|
1521
|
+
)
|
|
1522
|
+
extra_fields_link = {}
|
|
1523
|
+
add_authentication(item, time_entry["Url"], extra_fields_link)
|
|
1524
|
+
# add mapbox vector tile link
|
|
1525
|
+
identifier = str(uuid.uuid4())
|
|
1526
|
+
extra_fields_link["key"] = identifier
|
|
1527
|
+
if vector_tile_id_property := endpoint_config.get("idProperty"):
|
|
1528
|
+
extra_fields_link["idProperty"] = vector_tile_id_property
|
|
1529
|
+
link = Link(
|
|
1530
|
+
rel="vector-tile",
|
|
1531
|
+
target=time_entry["Url"],
|
|
1532
|
+
media_type=media_type,
|
|
1533
|
+
title=collection_config["Name"],
|
|
1534
|
+
extra_fields=extra_fields_link,
|
|
1535
|
+
)
|
|
1536
|
+
add_projection_info(
|
|
1537
|
+
endpoint_config,
|
|
1538
|
+
link,
|
|
1539
|
+
)
|
|
1540
|
+
item.add_link(link)
|
|
1541
|
+
add_projection_info(
|
|
1542
|
+
endpoint_config,
|
|
1543
|
+
item,
|
|
1544
|
+
)
|
|
1545
|
+
if endpoint_config.get("Attribution"):
|
|
1546
|
+
item.stac_extensions.append(
|
|
1547
|
+
"https://stac-extensions.github.io/attribution/v0.1.0/schema.json"
|
|
1548
|
+
)
|
|
1549
|
+
item.extra_fields["attribution"] = endpoint_config["Attribution"]
|
|
1550
|
+
# add style
|
|
1551
|
+
if ep_st := endpoint_config.get("Style"):
|
|
1552
|
+
style_link = Link(
|
|
1553
|
+
rel="style",
|
|
1554
|
+
target=ep_st
|
|
1555
|
+
if ep_st.startswith("http")
|
|
1556
|
+
else f"{catalog_config['assets_endpoint']}/{ep_st}",
|
|
1557
|
+
media_type=style_type,
|
|
1558
|
+
extra_fields={"links:keys": [identifier]},
|
|
1559
|
+
)
|
|
1560
|
+
item.add_link(style_link)
|
|
1561
|
+
items.append(item)
|
|
1562
|
+
|
|
1563
|
+
save_items(
|
|
1564
|
+
collection,
|
|
1565
|
+
items,
|
|
1566
|
+
options.outputpath,
|
|
1567
|
+
catalog_config["id"],
|
|
1568
|
+
coll_path_rel_to_root_catalog,
|
|
1569
|
+
options.gp,
|
|
1570
|
+
)
|
|
1571
|
+
|
|
1572
|
+
else:
|
|
1573
|
+
LOGGER.warn(f"NO datetimes configured for collection: {collection_config['Name']}!")
|
|
1574
|
+
|
|
1575
|
+
add_collection_information(catalog_config, collection, collection_config)
|
|
1576
|
+
return collection
|
|
@@ -24,6 +24,7 @@ from eodash_catalog.endpoints import (
|
|
|
24
24
|
handle_raw_source,
|
|
25
25
|
handle_SH_endpoint,
|
|
26
26
|
handle_SH_WMS_endpoint,
|
|
27
|
+
handle_vector_tile_source,
|
|
27
28
|
handle_VEDA_endpoint,
|
|
28
29
|
handle_WMS_endpoint,
|
|
29
30
|
handle_xcube_endpoint,
|
|
@@ -292,16 +293,14 @@ def process_collection_file(
|
|
|
292
293
|
catalog,
|
|
293
294
|
options,
|
|
294
295
|
)
|
|
295
|
-
elif endpoint_config["Name"] == "
|
|
296
|
-
|
|
297
|
-
collection = handle_WMS_endpoint(
|
|
296
|
+
elif endpoint_config["Name"] == "VectorTile source":
|
|
297
|
+
collection = handle_vector_tile_source(
|
|
298
298
|
catalog_config,
|
|
299
299
|
endpoint_config,
|
|
300
300
|
collection_config,
|
|
301
301
|
coll_path_rel_to_root_catalog,
|
|
302
302
|
catalog,
|
|
303
303
|
options,
|
|
304
|
-
wmts=True,
|
|
305
304
|
)
|
|
306
305
|
elif endpoint_config["Name"] == "Collection-only":
|
|
307
306
|
collection = handle_collection_only(
|
|
@@ -360,9 +359,11 @@ def process_collection_file(
|
|
|
360
359
|
countries.extend(sub_coll_def["Country"])
|
|
361
360
|
else:
|
|
362
361
|
countries.append(sub_coll_def["Country"])
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
362
|
+
# commented out intentionally, because otherwise paths further down did
|
|
363
|
+
# not match, parquet file was one level deeper
|
|
364
|
+
# coll_path_rel_to_root_catalog = (
|
|
365
|
+
# f"{coll_path_rel_to_root_catalog}/{sub_coll_def['Collection']}"
|
|
366
|
+
# )
|
|
366
367
|
process_collection_file(
|
|
367
368
|
catalog_config,
|
|
368
369
|
"{}/{}".format(options.collectionspath, sub_coll_def["Collection"]),
|
|
@@ -421,6 +422,7 @@ def process_collection_file(
|
|
|
421
422
|
add_collection_information(catalog_config, parent_collection, collection_config, True)
|
|
422
423
|
add_process_info(parent_collection, catalog_config, collection_config)
|
|
423
424
|
update_extents_from_collection_children(parent_collection)
|
|
425
|
+
add_base_overlay_info(parent_collection, catalog_config, collection_config)
|
|
424
426
|
# Fill summaries for locations
|
|
425
427
|
parent_collection.summaries = Summaries(
|
|
426
428
|
{
|
|
@@ -456,12 +458,20 @@ def add_to_catalog(
|
|
|
456
458
|
link.extra_fields["endpointtype"] = endpoint["Name"]
|
|
457
459
|
if collection_config.get("Subtitle"):
|
|
458
460
|
link.extra_fields["subtitle"] = collection_config["Subtitle"]
|
|
461
|
+
if collection_config.get("ShortDescription"):
|
|
462
|
+
link.extra_fields["shortdescription"] = collection_config["ShortDescription"]
|
|
459
463
|
link.extra_fields["title"] = collection.title
|
|
460
464
|
if collection_config.get("EodashIdentifier"):
|
|
461
465
|
link.extra_fields["code"] = collection_config["EodashIdentifier"]
|
|
462
466
|
link.extra_fields["id"] = collection_config["Name"]
|
|
463
467
|
if collection_config.get("Themes"):
|
|
464
468
|
link.extra_fields["themes"] = collection_config["Themes"]
|
|
469
|
+
if collection_config.get("Provider"):
|
|
470
|
+
# get all provider names
|
|
471
|
+
link.extra_fields["providers"] = [
|
|
472
|
+
provider.get("Name") or provider.get("Url")
|
|
473
|
+
for provider in collection_config["Provider"]
|
|
474
|
+
]
|
|
465
475
|
# Check for summaries and bubble up info
|
|
466
476
|
if disable:
|
|
467
477
|
link.extra_fields["roles"] = ["disable"]
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import time
|
|
3
|
+
from urllib.parse import urlparse
|
|
3
4
|
|
|
4
5
|
from oauthlib.oauth2 import BackendApplicationClient
|
|
5
6
|
from requests_oauthlib import OAuth2Session
|
|
6
7
|
|
|
7
|
-
SH_TOKEN_URL = "https://services.sentinel-hub.com/oauth/token"
|
|
8
8
|
_token_cache: dict[str, dict] = {}
|
|
9
9
|
|
|
10
10
|
|
|
@@ -24,6 +24,8 @@ def get_SH_token(endpoint_config: dict) -> str:
|
|
|
24
24
|
client = BackendApplicationClient(client_id=client_id)
|
|
25
25
|
oauth = OAuth2Session(client=client)
|
|
26
26
|
# Get token for the session
|
|
27
|
+
endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
|
|
28
|
+
SH_TOKEN_URL = f"https://{endpoint_url_parts.netloc}/oauth/token"
|
|
27
29
|
token = oauth.fetch_token(
|
|
28
30
|
token_url=SH_TOKEN_URL,
|
|
29
31
|
client_secret=client_secret,
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import uuid
|
|
1
2
|
from datetime import datetime, timezone
|
|
3
|
+
from urllib.parse import parse_qs, urlparse
|
|
2
4
|
|
|
3
5
|
import requests
|
|
4
6
|
import spdx_lookup as lookup
|
|
@@ -122,11 +124,14 @@ def create_service_link(endpoint_config: dict, catalog_config: dict) -> Link:
|
|
|
122
124
|
return sl
|
|
123
125
|
|
|
124
126
|
|
|
125
|
-
def create_web_map_link(
|
|
127
|
+
def create_web_map_link(
|
|
128
|
+
collection: Collection, catalog_config: dict, layer_config: dict, role: str
|
|
129
|
+
) -> Link:
|
|
126
130
|
extra_fields = {
|
|
127
131
|
"roles": [role],
|
|
128
132
|
"id": layer_config["id"],
|
|
129
133
|
}
|
|
134
|
+
media_type = (layer_config.get("media_type", "image/png"),)
|
|
130
135
|
if layer_config.get("default"):
|
|
131
136
|
extra_fields["roles"].append("default")
|
|
132
137
|
if layer_config.get("visible"):
|
|
@@ -146,12 +151,32 @@ def create_web_map_link(layer_config: dict, role: str) -> Link:
|
|
|
146
151
|
extra_fields["wmts:layer"] = layer_config["layer"]
|
|
147
152
|
if layer_config.get("dimensions"):
|
|
148
153
|
extra_fields["wmts:dimensions"] = layer_config["dimensions"]
|
|
154
|
+
case "vector-tile":
|
|
155
|
+
identifier = str(uuid.uuid4())
|
|
156
|
+
extra_fields["key"] = identifier
|
|
157
|
+
media_type = "application/vnd.mapbox-vector-tile"
|
|
158
|
+
if vector_tile_id_property := layer_config.get("idProperty"):
|
|
159
|
+
extra_fields["idProperty"] = vector_tile_id_property
|
|
160
|
+
if ep_st := layer_config.get("Style"):
|
|
161
|
+
style_link = Link(
|
|
162
|
+
rel="style",
|
|
163
|
+
target=ep_st
|
|
164
|
+
if ep_st.startswith("http")
|
|
165
|
+
else f"{catalog_config['assets_endpoint']}/{ep_st}",
|
|
166
|
+
media_type="text/vector-styles",
|
|
167
|
+
extra_fields={"links:keys": [identifier]},
|
|
168
|
+
)
|
|
169
|
+
collection.add_link(style_link)
|
|
170
|
+
add_authentication(collection, layer_config["url"], extra_fields)
|
|
171
|
+
|
|
149
172
|
if layer_config.get("Attribution"):
|
|
150
173
|
extra_fields["attribution"] = layer_config["Attribution"]
|
|
174
|
+
if layer_config.get("Colorlegend"):
|
|
175
|
+
extra_fields["eox:colorlegend"] = layer_config["Colorlegend"]
|
|
151
176
|
wml = Link(
|
|
152
177
|
rel=layer_config["protocol"],
|
|
153
178
|
target=layer_config["url"],
|
|
154
|
-
media_type=
|
|
179
|
+
media_type=media_type,
|
|
155
180
|
title=layer_config["name"],
|
|
156
181
|
extra_fields=extra_fields,
|
|
157
182
|
)
|
|
@@ -288,6 +313,10 @@ def add_collection_information(
|
|
|
288
313
|
|
|
289
314
|
if collection_config.get("Subtitle"):
|
|
290
315
|
collection.extra_fields["subtitle"] = collection_config["Subtitle"]
|
|
316
|
+
|
|
317
|
+
if collection_config.get("ShortDescription"):
|
|
318
|
+
collection.extra_fields["shortdescription"] = collection_config["ShortDescription"]
|
|
319
|
+
|
|
291
320
|
if collection_config.get("Legend"):
|
|
292
321
|
collection.add_asset(
|
|
293
322
|
"legend",
|
|
@@ -297,28 +326,42 @@ def add_collection_information(
|
|
|
297
326
|
roles=["metadata"],
|
|
298
327
|
),
|
|
299
328
|
)
|
|
300
|
-
if collection_config.get("
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
)
|
|
308
|
-
|
|
329
|
+
if stories := collection_config.get("Stories"):
|
|
330
|
+
for story in stories:
|
|
331
|
+
story_url = story.get("Url")
|
|
332
|
+
if not story_url.startswith("http"):
|
|
333
|
+
story_url = f'{catalog_config.get("stories_endpoint")}/{story_url}'
|
|
334
|
+
parsed_url = urlparse(story_url)
|
|
335
|
+
# check if it is URL with a query parameter id=story-identifier
|
|
336
|
+
if parsed_url.query and len(parse_qs(parsed_url.query).get("id")) > 0:
|
|
337
|
+
story_id = parse_qs(parsed_url.query).get("id")[0]
|
|
338
|
+
else:
|
|
339
|
+
story_id = parsed_url.path.rsplit("/")[-1].replace(".md", "").replace(".MD", "")
|
|
340
|
+
collection.add_asset(
|
|
341
|
+
story_id,
|
|
342
|
+
Asset(
|
|
343
|
+
title=story.get("Name"),
|
|
344
|
+
href=story_url,
|
|
345
|
+
media_type="text/markdown",
|
|
346
|
+
roles=["metadata", "story"],
|
|
347
|
+
),
|
|
348
|
+
)
|
|
309
349
|
if collection_config.get("Image"):
|
|
350
|
+
# Check if absolute URL or relative path
|
|
351
|
+
if collection_config["Image"].startswith("http"):
|
|
352
|
+
image_url = collection_config["Image"]
|
|
353
|
+
else:
|
|
354
|
+
image_url = f'{catalog_config["assets_endpoint"]}/{collection_config["Image"]}'
|
|
310
355
|
collection.add_asset(
|
|
311
356
|
"thumbnail",
|
|
312
357
|
Asset(
|
|
313
|
-
href=
|
|
358
|
+
href=image_url,
|
|
314
359
|
media_type="image/png",
|
|
315
360
|
roles=["thumbnail"],
|
|
316
361
|
),
|
|
317
362
|
)
|
|
318
363
|
# Bubble up thumbnail to extra fields
|
|
319
|
-
collection.extra_fields["thumbnail"] =
|
|
320
|
-
f'{catalog_config["assets_endpoint"]}/' f'{collection_config["Image"]}'
|
|
321
|
-
)
|
|
364
|
+
collection.extra_fields["thumbnail"] = image_url
|
|
322
365
|
# Add extra fields to collection if available
|
|
323
366
|
add_extra_fields(collection, collection_config, is_root_collection)
|
|
324
367
|
|
|
@@ -473,23 +516,31 @@ def add_base_overlay_info(
|
|
|
473
516
|
collection: Collection, catalog_config: dict, collection_config: dict
|
|
474
517
|
) -> None:
|
|
475
518
|
# add custom baselayers specially for this indicator
|
|
476
|
-
if
|
|
519
|
+
if "BaseLayers" in collection_config:
|
|
477
520
|
for layer in collection_config["BaseLayers"]:
|
|
478
|
-
collection.add_link(
|
|
521
|
+
collection.add_link(
|
|
522
|
+
create_web_map_link(collection, catalog_config, layer, role="baselayer")
|
|
523
|
+
)
|
|
479
524
|
# alternatively use default base layers defined
|
|
480
525
|
elif catalog_config.get("default_base_layers"):
|
|
481
526
|
base_layers = read_config_file(catalog_config["default_base_layers"])
|
|
482
527
|
for layer in base_layers:
|
|
483
|
-
collection.add_link(
|
|
528
|
+
collection.add_link(
|
|
529
|
+
create_web_map_link(collection, catalog_config, layer, role="baselayer")
|
|
530
|
+
)
|
|
484
531
|
# add custom overlays just for this indicator
|
|
485
|
-
if
|
|
532
|
+
if "OverlayLayers" in collection_config:
|
|
486
533
|
for layer in collection_config["OverlayLayers"]:
|
|
487
|
-
collection.add_link(
|
|
534
|
+
collection.add_link(
|
|
535
|
+
create_web_map_link(collection, catalog_config, layer, role="overlay")
|
|
536
|
+
)
|
|
488
537
|
# check if default overlay layers defined
|
|
489
538
|
elif catalog_config.get("default_overlay_layers"):
|
|
490
539
|
overlay_layers = read_config_file(catalog_config["default_overlay_layers"])
|
|
491
540
|
for layer in overlay_layers:
|
|
492
|
-
collection.add_link(
|
|
541
|
+
collection.add_link(
|
|
542
|
+
create_web_map_link(collection, catalog_config, layer, role="overlay")
|
|
543
|
+
)
|
|
493
544
|
|
|
494
545
|
|
|
495
546
|
def add_extra_fields(
|
|
@@ -579,3 +630,20 @@ def add_projection_info(
|
|
|
579
630
|
stac_object.extra_fields["eodash:proj4_def"] = proj
|
|
580
631
|
else:
|
|
581
632
|
raise Exception(f"Incorrect type of proj definition {proj}")
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def add_authentication(stac_object: Item | Collection | Catalog, url: str, extra_fields_link: dict):
|
|
636
|
+
if "api.mapbox" in url:
|
|
637
|
+
# add authentication info
|
|
638
|
+
auth_extension = "https://stac-extensions.github.io/authentication/v1.1.0/schema.json"
|
|
639
|
+
if auth_extension not in stac_object.stac_extensions:
|
|
640
|
+
stac_object.stac_extensions.append(auth_extension)
|
|
641
|
+
stac_object.extra_fields["auth:schemes"] = {
|
|
642
|
+
"mapboxauth": {
|
|
643
|
+
"type": "apiKey",
|
|
644
|
+
"name": "access_token",
|
|
645
|
+
"in": "query",
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
extra_fields_link["auth:refs"] = ["mapboxauth"]
|
|
649
|
+
pass
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import re
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
from urllib.parse import urlparse
|
|
4
5
|
|
|
5
6
|
import requests
|
|
6
7
|
from pystac import (
|
|
@@ -46,7 +47,9 @@ def generate_thumbnail(
|
|
|
46
47
|
# if it is start and end datetime have to exist
|
|
47
48
|
if item_datetime:
|
|
48
49
|
time = format_datetime_to_isostring_zulu(item_datetime)
|
|
49
|
-
|
|
50
|
+
endpoint_url_parts = urlparse(endpoint_config["EndPoint"])
|
|
51
|
+
url = "https://{}/ogc/wms/{}?{}&layers={}&time={}&{}".format(
|
|
52
|
+
endpoint_url_parts,
|
|
50
53
|
instanceId,
|
|
51
54
|
wms_config,
|
|
52
55
|
endpoint_config["LayerId"],
|
|
@@ -328,6 +328,7 @@ def add_single_item_if_collection_empty(endpoint_config: dict, collection: Colle
|
|
|
328
328
|
datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
329
329
|
start_datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
330
330
|
end_datetime=datetime.now(tz=pytztimezone("UTC")),
|
|
331
|
+
assets={"dummy_asset": Asset(href="")},
|
|
331
332
|
)
|
|
332
333
|
collection.add_item(item)
|
|
333
334
|
if not endpoint_config.get("OverwriteBBox"):
|
|
@@ -468,6 +469,11 @@ def extract_extent_from_geoparquet(table) -> tuple[TemporalExtent, SpatialExtent
|
|
|
468
469
|
# fallback to start_datetime
|
|
469
470
|
min_datetime = pc.min(table["start_datetime"]).as_py()
|
|
470
471
|
max_datetime = pc.max(table["start_datetime"]).as_py()
|
|
472
|
+
# Making sure time extent is timezone aware
|
|
473
|
+
if min_datetime and min_datetime.tzinfo is None:
|
|
474
|
+
min_datetime = min_datetime.replace(tzinfo=timezone.utc)
|
|
475
|
+
if max_datetime and max_datetime.tzinfo is None:
|
|
476
|
+
max_datetime = max_datetime.replace(tzinfo=timezone.utc)
|
|
471
477
|
temporal = TemporalExtent([min_datetime, max_datetime])
|
|
472
478
|
geoms = [wkb.loads(g.as_py()) for g in table["geometry"] if g is not None]
|
|
473
479
|
bbox = sgeom.MultiPolygon(geoms).bounds
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"Name": "Sentinel Hub WMS",
|
|
34
34
|
"CollectionId": "sentinel-1-grd",
|
|
35
35
|
"LayerId": "SENTINEL_1_IW_VV",
|
|
36
|
-
"
|
|
36
|
+
"OverwriteBBox": [
|
|
37
37
|
101.938,
|
|
38
38
|
11.945,
|
|
39
39
|
106.37,
|
|
@@ -65,4 +65,4 @@
|
|
|
65
65
|
"Url": "https://www.copernicus.eu/en/access-data/conventional-data-access-hubs"
|
|
66
66
|
}
|
|
67
67
|
]
|
|
68
|
-
}
|
|
68
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_CROPOMAT1.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_geodb_locations.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_projection.json
RENAMED
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_see_solar_energy.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_sh_wms_locations.json
RENAMED
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_1_json.json
RENAMED
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_tif_demo_2.yaml
RENAMED
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_veda_tiles.json
RENAMED
|
File without changes
|
{eodash_catalog-0.3.7 → eodash_catalog-0.3.18}/tests/testing-collections/test_wms_no_time.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|