eodash_catalog 0.0.30__tar.gz → 0.0.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of eodash_catalog might be problematic. Click here for more details.
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/.bumpversion.cfg +1 -1
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/PKG-INFO +1 -1
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/__about__.py +1 -1
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/endpoints.py +110 -97
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/stac_handling.py +16 -19
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/thumbnails.py +2 -2
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/utils.py +34 -21
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/.github/workflows/ci.yml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/.github/workflows/python-publish.yml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/.gitignore +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/.vscode/extensions.json +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/.vscode/settings.json +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/LICENSE.txt +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/README.md +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/pyproject.toml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/requirements.txt +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/ruff.toml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/__init__.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/duration.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/generate_indicators.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/src/eodash_catalog/sh_endpoint.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/__init__.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/test-data/regional_forecast.json +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/test_generate.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-catalogs/testing.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_CROPOMAT1.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_see_solar_energy.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_1.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_2.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_wms_no_time.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-indicators/test_indicator.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-layers/baselayers.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-layers/overlays.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: eodash_catalog
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.31
|
|
4
4
|
Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
|
|
5
5
|
Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
|
|
@@ -9,7 +9,6 @@ from itertools import groupby
|
|
|
9
9
|
from operator import itemgetter
|
|
10
10
|
|
|
11
11
|
import requests
|
|
12
|
-
from dateutil import parser
|
|
13
12
|
from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries
|
|
14
13
|
from pystac_client import Client
|
|
15
14
|
from structlog import get_logger
|
|
@@ -19,7 +18,7 @@ from eodash_catalog.stac_handling import (
|
|
|
19
18
|
add_collection_information,
|
|
20
19
|
add_example_info,
|
|
21
20
|
add_projection_info,
|
|
22
|
-
|
|
21
|
+
get_collection_datetimes_from_config,
|
|
23
22
|
get_or_create_collection,
|
|
24
23
|
)
|
|
25
24
|
from eodash_catalog.thumbnails import generate_thumbnail
|
|
@@ -28,7 +27,9 @@ from eodash_catalog.utils import (
|
|
|
28
27
|
create_geojson_from_bbox,
|
|
29
28
|
create_geojson_point,
|
|
30
29
|
filter_time_entries,
|
|
30
|
+
format_datetime_to_isostring_zulu,
|
|
31
31
|
generate_veda_cog_link,
|
|
32
|
+
parse_datestring_to_tz_aware_datetime,
|
|
32
33
|
replace_with_env_variables,
|
|
33
34
|
retrieveExtentFromWMSWMTS,
|
|
34
35
|
)
|
|
@@ -69,29 +70,26 @@ def process_STAC_Datacube_Endpoint(
|
|
|
69
70
|
if v.get("type") == "temporal":
|
|
70
71
|
time_dimension = k
|
|
71
72
|
break
|
|
72
|
-
|
|
73
|
+
datetimes = [
|
|
74
|
+
parse_datestring_to_tz_aware_datetime(time_string)
|
|
75
|
+
for time_string in dimensions.get(time_dimension).get("values")
|
|
76
|
+
]
|
|
73
77
|
# optionally subset time results based on config
|
|
74
78
|
if query := endpoint_config.get("Query"):
|
|
75
|
-
|
|
79
|
+
datetimes = filter_time_entries(datetimes, query)
|
|
76
80
|
|
|
77
|
-
for
|
|
78
|
-
|
|
79
|
-
id=
|
|
81
|
+
for dt in datetimes:
|
|
82
|
+
new_item = Item(
|
|
83
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
80
84
|
bbox=item.bbox,
|
|
81
85
|
properties={},
|
|
82
86
|
geometry=item.geometry,
|
|
83
|
-
datetime=
|
|
87
|
+
datetime=dt,
|
|
84
88
|
)
|
|
85
|
-
link = collection.add_item(
|
|
86
|
-
link.extra_fields["datetime"] = t
|
|
89
|
+
link = collection.add_item(new_item)
|
|
87
90
|
# bubble up information we want to the link
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
if item_datetime:
|
|
91
|
-
link.extra_fields["datetime"] = item_datetime.isoformat()[:-6] + "Z"
|
|
92
|
-
else:
|
|
93
|
-
link.extra_fields["start_datetime"] = item.properties["start_datetime"]
|
|
94
|
-
link.extra_fields["end_datetime"] = item.properties["end_datetime"]
|
|
91
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
92
|
+
|
|
95
93
|
unit = variables.get(endpoint_config.get("Variable")).get("unit")
|
|
96
94
|
if unit and "yAxis" not in collection_config:
|
|
97
95
|
collection_config["yAxis"] = unit
|
|
@@ -238,16 +236,15 @@ def process_STACAPI_Endpoint(
|
|
|
238
236
|
)
|
|
239
237
|
link.extra_fields["cog_href"] = item.assets["cog_default"].href
|
|
240
238
|
elif item_datetime:
|
|
241
|
-
|
|
242
|
-
|
|
239
|
+
add_visualization_info(
|
|
240
|
+
item, collection_config, endpoint_config, datetimes=[item_datetime]
|
|
241
|
+
)
|
|
243
242
|
elif "start_datetime" in item.properties and "end_datetime" in item.properties:
|
|
244
243
|
add_visualization_info(
|
|
245
244
|
item,
|
|
246
245
|
collection_config,
|
|
247
246
|
endpoint_config,
|
|
248
|
-
|
|
249
|
-
item.properties["start_datetime"], item.properties["end_datetime"]
|
|
250
|
-
),
|
|
247
|
+
datetimes=[item.properties["start_datetime"], item.properties["end_datetime"]],
|
|
251
248
|
)
|
|
252
249
|
# If a root collection exists we point back to it from the item
|
|
253
250
|
if root_collection:
|
|
@@ -256,12 +253,8 @@ def process_STACAPI_Endpoint(
|
|
|
256
253
|
# bubble up information we want to the link
|
|
257
254
|
# it is possible for datetime to be null, if it is start and end datetime have to exist
|
|
258
255
|
if item_datetime:
|
|
259
|
-
iso_time = item_datetime
|
|
260
|
-
|
|
261
|
-
# for SH WMS we only save the date (no time)
|
|
262
|
-
link.extra_fields["datetime"] = iso_date
|
|
263
|
-
else:
|
|
264
|
-
link.extra_fields["datetime"] = iso_time
|
|
256
|
+
iso_time = format_datetime_to_isostring_zulu(item_datetime)
|
|
257
|
+
link.extra_fields["datetime"] = iso_time
|
|
265
258
|
else:
|
|
266
259
|
link.extra_fields["start_datetime"] = item.properties["start_datetime"]
|
|
267
260
|
link.extra_fields["end_datetime"] = item.properties["end_datetime"]
|
|
@@ -305,18 +298,18 @@ def handle_collection_only(
|
|
|
305
298
|
collection = get_or_create_collection(
|
|
306
299
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
307
300
|
)
|
|
308
|
-
|
|
309
|
-
if len(
|
|
310
|
-
for
|
|
301
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
302
|
+
if len(datetimes) > 0:
|
|
303
|
+
for dt in datetimes:
|
|
311
304
|
item = Item(
|
|
312
|
-
id=
|
|
305
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
313
306
|
bbox=endpoint_config.get("OverwriteBBox"),
|
|
314
307
|
properties={},
|
|
315
308
|
geometry=None,
|
|
316
|
-
datetime=
|
|
309
|
+
datetime=dt,
|
|
317
310
|
)
|
|
318
311
|
link = collection.add_item(item)
|
|
319
|
-
link.extra_fields["datetime"] =
|
|
312
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
320
313
|
add_collection_information(catalog_config, collection, collection_config)
|
|
321
314
|
# eodash v4 compatibility
|
|
322
315
|
add_visualization_info(collection, collection_config, endpoint_config)
|
|
@@ -342,21 +335,22 @@ def handle_SH_WMS_endpoint(
|
|
|
342
335
|
catalog, location["Identifier"], location_config, catalog_config, endpoint_config
|
|
343
336
|
)
|
|
344
337
|
collection.extra_fields["endpointtype"] = endpoint_config["Name"]
|
|
345
|
-
for
|
|
338
|
+
for time_string in location["Times"]:
|
|
339
|
+
dt = parse_datestring_to_tz_aware_datetime(time_string)
|
|
346
340
|
item = Item(
|
|
347
|
-
id=
|
|
341
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
348
342
|
bbox=location["Bbox"],
|
|
349
343
|
properties={},
|
|
350
344
|
geometry=None,
|
|
351
|
-
datetime=
|
|
345
|
+
datetime=dt,
|
|
352
346
|
stac_extensions=[
|
|
353
347
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
354
348
|
],
|
|
355
349
|
)
|
|
356
350
|
add_projection_info(endpoint_config, item)
|
|
357
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
351
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
358
352
|
item_link = collection.add_item(item)
|
|
359
|
-
item_link.extra_fields["datetime"] =
|
|
353
|
+
item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
360
354
|
|
|
361
355
|
link = root_collection.add_child(collection)
|
|
362
356
|
# bubble up information we want to the link
|
|
@@ -376,23 +370,23 @@ def handle_SH_WMS_endpoint(
|
|
|
376
370
|
else:
|
|
377
371
|
# if locations are not provided, treat the collection as a
|
|
378
372
|
# general proxy to the sentinel hub layer
|
|
379
|
-
|
|
373
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
380
374
|
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
381
|
-
for
|
|
375
|
+
for dt in datetimes:
|
|
382
376
|
item = Item(
|
|
383
|
-
id=
|
|
377
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
384
378
|
bbox=bbox,
|
|
385
379
|
properties={},
|
|
386
380
|
geometry=None,
|
|
387
|
-
datetime=
|
|
381
|
+
datetime=dt,
|
|
388
382
|
stac_extensions=[
|
|
389
383
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
390
384
|
],
|
|
391
385
|
)
|
|
392
386
|
add_projection_info(endpoint_config, item)
|
|
393
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
387
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
394
388
|
item_link = root_collection.add_item(item)
|
|
395
|
-
item_link.extra_fields["datetime"] =
|
|
389
|
+
item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
396
390
|
# eodash v4 compatibility
|
|
397
391
|
add_collection_information(catalog_config, root_collection, collection_config)
|
|
398
392
|
add_visualization_info(root_collection, collection_config, endpoint_config)
|
|
@@ -538,13 +532,13 @@ def handle_WMS_endpoint(
|
|
|
538
532
|
collection = get_or_create_collection(
|
|
539
533
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
540
534
|
)
|
|
541
|
-
|
|
535
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
542
536
|
spatial_extent = collection.extent.spatial.to_dict().get("bbox", [-180, -90, 180, 90])[0]
|
|
543
537
|
if endpoint_config.get("Type") != "OverwriteTimes" or not endpoint_config.get("OverwriteBBox"):
|
|
544
538
|
# some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
|
|
545
539
|
# have to process full service capabilities XML
|
|
546
540
|
capabilities_url = endpoint_config["EndPoint"]
|
|
547
|
-
spatial_extent,
|
|
541
|
+
spatial_extent, datetimes = retrieveExtentFromWMSWMTS(
|
|
548
542
|
capabilities_url,
|
|
549
543
|
endpoint_config["LayerId"],
|
|
550
544
|
version=endpoint_config.get("Version", "1.1.1"),
|
|
@@ -552,24 +546,24 @@ def handle_WMS_endpoint(
|
|
|
552
546
|
)
|
|
553
547
|
# optionally filter time results
|
|
554
548
|
if query := endpoint_config.get("Query"):
|
|
555
|
-
|
|
549
|
+
datetimes = filter_time_entries(datetimes, query)
|
|
556
550
|
# Create an item per time to allow visualization in stac clients
|
|
557
|
-
if len(
|
|
558
|
-
for
|
|
551
|
+
if len(datetimes) > 0:
|
|
552
|
+
for dt in datetimes:
|
|
559
553
|
item = Item(
|
|
560
|
-
id=
|
|
554
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
561
555
|
bbox=spatial_extent,
|
|
562
556
|
properties={},
|
|
563
557
|
geometry=None,
|
|
564
|
-
datetime=
|
|
558
|
+
datetime=dt,
|
|
565
559
|
stac_extensions=[
|
|
566
560
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
567
561
|
],
|
|
568
562
|
)
|
|
569
563
|
add_projection_info(endpoint_config, item)
|
|
570
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
564
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
571
565
|
link = collection.add_item(item)
|
|
572
|
-
link.extra_fields["datetime"] =
|
|
566
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
573
567
|
collection.update_extent_from_items()
|
|
574
568
|
|
|
575
569
|
# Check if we should overwrite bbox
|
|
@@ -606,7 +600,7 @@ def add_visualization_info(
|
|
|
606
600
|
collection_config: dict,
|
|
607
601
|
endpoint_config: dict,
|
|
608
602
|
file_url: str | None = None,
|
|
609
|
-
|
|
603
|
+
datetimes: list[datetime] | None = None,
|
|
610
604
|
) -> None:
|
|
611
605
|
extra_fields: dict[str, list[str] | dict[str, str]] = {}
|
|
612
606
|
if "Attribution" in endpoint_config:
|
|
@@ -630,27 +624,32 @@ def add_visualization_info(
|
|
|
630
624
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
631
625
|
for key, value in dimensions_config.items():
|
|
632
626
|
dimensions[key] = value
|
|
633
|
-
if
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
627
|
+
if datetimes is not None:
|
|
628
|
+
dt = datetimes[0]
|
|
629
|
+
start_isostring = format_datetime_to_isostring_zulu(dt)
|
|
630
|
+
# SH WMS for public collections needs time interval, we use full day here
|
|
631
|
+
end = dt + timedelta(days=1) - timedelta(milliseconds=1)
|
|
632
|
+
# we have start_datetime and end_datetime
|
|
633
|
+
if len(datetimes) == 2:
|
|
634
|
+
end = datetimes[1]
|
|
635
|
+
end_isostring = format_datetime_to_isostring_zulu(end)
|
|
636
|
+
time_interval = f"{start_isostring}/{end_isostring}"
|
|
637
|
+
dimensions["TIME"] = time_interval
|
|
638
|
+
|
|
643
639
|
if dimensions != {}:
|
|
644
640
|
extra_fields["wms:dimensions"] = dimensions
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
extra_fields=extra_fields,
|
|
652
|
-
)
|
|
641
|
+
link = Link(
|
|
642
|
+
rel="wms",
|
|
643
|
+
target=f"https://services.sentinel-hub.com/ogc/wms/{instanceId}",
|
|
644
|
+
media_type=(endpoint_config.get("MimeType", "image/png")),
|
|
645
|
+
title=collection_config["Name"],
|
|
646
|
+
extra_fields=extra_fields,
|
|
653
647
|
)
|
|
648
|
+
add_projection_info(
|
|
649
|
+
endpoint_config,
|
|
650
|
+
link,
|
|
651
|
+
)
|
|
652
|
+
stac_object.add_link(link)
|
|
654
653
|
elif endpoint_config["Name"] == "WMS":
|
|
655
654
|
extra_fields.update(
|
|
656
655
|
{
|
|
@@ -662,8 +661,8 @@ def add_visualization_info(
|
|
|
662
661
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
663
662
|
for key, value in dimensions_config.items():
|
|
664
663
|
dimensions[key] = value
|
|
665
|
-
if
|
|
666
|
-
dimensions["TIME"] =
|
|
664
|
+
if datetimes is not None:
|
|
665
|
+
dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
|
|
667
666
|
if dimensions != {}:
|
|
668
667
|
extra_fields["wms:dimensions"] = dimensions
|
|
669
668
|
if "Styles" in endpoint_config:
|
|
@@ -672,20 +671,30 @@ def add_visualization_info(
|
|
|
672
671
|
endpoint_url = endpoint_config["EndPoint"]
|
|
673
672
|
# custom replacing of all ENV VARS present as template in URL as {VAR}
|
|
674
673
|
endpoint_url = replace_with_env_variables(endpoint_url)
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
674
|
+
link = Link(
|
|
675
|
+
rel="wms",
|
|
676
|
+
target=endpoint_url,
|
|
677
|
+
media_type=media_type,
|
|
678
|
+
title=collection_config["Name"],
|
|
679
|
+
extra_fields=extra_fields,
|
|
680
|
+
)
|
|
681
|
+
add_projection_info(
|
|
682
|
+
endpoint_config,
|
|
683
|
+
link,
|
|
683
684
|
)
|
|
685
|
+
stac_object.add_link(link)
|
|
684
686
|
elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
|
|
685
687
|
target_url = "{}".format(endpoint_config.get("EndPoint"))
|
|
686
688
|
# custom time just for this special case as a default for collection wmts
|
|
689
|
+
time = None
|
|
690
|
+
if datetimes is not None:
|
|
691
|
+
time = datetimes[0]
|
|
687
692
|
extra_fields.update(
|
|
688
|
-
{
|
|
693
|
+
{
|
|
694
|
+
"wmts:layer": endpoint_config.get("LayerId", "").replace(
|
|
695
|
+
"{time}", (time and str(time.year)) or "2017"
|
|
696
|
+
)
|
|
697
|
+
}
|
|
689
698
|
)
|
|
690
699
|
stac_object.add_link(
|
|
691
700
|
Link(
|
|
@@ -739,8 +748,8 @@ def add_visualization_info(
|
|
|
739
748
|
}
|
|
740
749
|
)
|
|
741
750
|
dimensions = {}
|
|
742
|
-
if
|
|
743
|
-
dimensions["time"] =
|
|
751
|
+
if datetimes is not None:
|
|
752
|
+
dimensions["time"] = format_datetime_to_isostring_zulu(datetimes[0])
|
|
744
753
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
745
754
|
for key, value in dimensions_config.items():
|
|
746
755
|
dimensions[key] = value
|
|
@@ -761,15 +770,18 @@ def add_visualization_info(
|
|
|
761
770
|
elif endpoint_config["Type"] == "tiles":
|
|
762
771
|
target_url = generate_veda_tiles_link(endpoint_config, file_url)
|
|
763
772
|
if target_url:
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
773
|
+
link = Link(
|
|
774
|
+
rel="xyz",
|
|
775
|
+
target=target_url,
|
|
776
|
+
media_type="image/png",
|
|
777
|
+
title=collection_config["Name"],
|
|
778
|
+
extra_fields=extra_fields,
|
|
779
|
+
)
|
|
780
|
+
add_projection_info(
|
|
781
|
+
endpoint_config,
|
|
782
|
+
link,
|
|
772
783
|
)
|
|
784
|
+
stac_object.add_link(link)
|
|
773
785
|
elif endpoint_config["Name"] == "GeoDB Vector Tiles":
|
|
774
786
|
# `${geoserverUrl}${config.layerName}@EPSG%3A${projString}@pbf/{z}/{x}/{-y}.pbf`,
|
|
775
787
|
# 'geodb_debd884d-92f9-4979-87b6-eadef1139394:GTIF_AT_Gemeinden_3857'
|
|
@@ -863,12 +875,13 @@ def handle_raw_source(
|
|
|
863
875
|
add_projection_info(endpoint_config, asset)
|
|
864
876
|
assets[a["Identifier"]] = asset
|
|
865
877
|
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
878
|
+
dt = parse_datestring_to_tz_aware_datetime(time_entry["Time"])
|
|
866
879
|
item = Item(
|
|
867
|
-
id=
|
|
880
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
868
881
|
bbox=bbox,
|
|
869
882
|
properties={},
|
|
870
883
|
geometry=create_geojson_from_bbox(bbox),
|
|
871
|
-
datetime=
|
|
884
|
+
datetime=dt,
|
|
872
885
|
assets=assets,
|
|
873
886
|
extra_fields={},
|
|
874
887
|
)
|
|
@@ -889,7 +902,7 @@ def handle_raw_source(
|
|
|
889
902
|
)
|
|
890
903
|
item.add_link(style_link)
|
|
891
904
|
link = collection.add_item(item)
|
|
892
|
-
link.extra_fields["datetime"] =
|
|
905
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
893
906
|
link.extra_fields["assets"] = [a["File"] for a in time_entry["Assets"]]
|
|
894
907
|
# eodash v4 compatibility, adding last referenced style to collection
|
|
895
908
|
if style_link:
|
|
@@ -3,7 +3,6 @@ from datetime import datetime
|
|
|
3
3
|
import requests
|
|
4
4
|
import spdx_lookup as lookup
|
|
5
5
|
import yaml
|
|
6
|
-
from dateutil import parser
|
|
7
6
|
from pystac import (
|
|
8
7
|
Asset,
|
|
9
8
|
Catalog,
|
|
@@ -18,7 +17,10 @@ from pystac import (
|
|
|
18
17
|
from structlog import get_logger
|
|
19
18
|
from yaml.loader import SafeLoader
|
|
20
19
|
|
|
21
|
-
from eodash_catalog.utils import
|
|
20
|
+
from eodash_catalog.utils import (
|
|
21
|
+
generateDatetimesFromInterval,
|
|
22
|
+
parse_datestring_to_tz_aware_datetime,
|
|
23
|
+
)
|
|
22
24
|
|
|
23
25
|
LOGGER = get_logger(__name__)
|
|
24
26
|
|
|
@@ -42,20 +44,12 @@ def get_or_create_collection(
|
|
|
42
44
|
spatial_extent,
|
|
43
45
|
]
|
|
44
46
|
)
|
|
45
|
-
times: list[str] = []
|
|
46
47
|
temporal_extent = TemporalExtent([[datetime.now(), None]])
|
|
47
48
|
if endpoint_config:
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
51
|
-
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
52
|
-
elif endpoint_config.get("DateTimeInterval"):
|
|
53
|
-
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00")
|
|
54
|
-
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00")
|
|
55
|
-
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
56
|
-
times = generateDateIsostringsFromInterval(start, end, timedelta_config)
|
|
57
|
-
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
49
|
+
times_datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
50
|
+
if len(times_datetimes) > 0:
|
|
58
51
|
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
52
|
+
|
|
59
53
|
extent = Extent(spatial=spatial_extent, temporal=temporal_extent)
|
|
60
54
|
|
|
61
55
|
# Check if description is link to markdown file
|
|
@@ -387,17 +381,20 @@ def add_extra_fields(stac_object: Collection | Link, collection_config: dict) ->
|
|
|
387
381
|
stac_object.extra_fields["eodash:mapProjection"] = collection_config["MapProjection"]
|
|
388
382
|
|
|
389
383
|
|
|
390
|
-
def
|
|
391
|
-
|
|
384
|
+
def get_collection_datetimes_from_config(endpoint_config: dict) -> list[datetime]:
|
|
385
|
+
times_datetimes: list[datetime] = []
|
|
392
386
|
if endpoint_config:
|
|
393
387
|
if endpoint_config.get("Times"):
|
|
394
388
|
times = list(endpoint_config.get("Times", []))
|
|
389
|
+
times_datetimes = sorted(
|
|
390
|
+
[parse_datestring_to_tz_aware_datetime(time) for time in times]
|
|
391
|
+
)
|
|
395
392
|
elif endpoint_config.get("DateTimeInterval"):
|
|
396
|
-
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:
|
|
397
|
-
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:
|
|
393
|
+
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00Z")
|
|
394
|
+
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00Z")
|
|
398
395
|
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
399
|
-
|
|
400
|
-
return
|
|
396
|
+
times_datetimes = generateDatetimesFromInterval(start, end, timedelta_config)
|
|
397
|
+
return times_datetimes
|
|
401
398
|
|
|
402
399
|
|
|
403
400
|
def add_projection_info(
|
|
@@ -7,7 +7,7 @@ from pystac import (
|
|
|
7
7
|
Item,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
|
-
from eodash_catalog.utils import generate_veda_cog_link
|
|
10
|
+
from eodash_catalog.utils import format_datetime_to_isostring_zulu, generate_veda_cog_link
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def fetch_and_save_thumbnail(collection_config: dict, url: str) -> None:
|
|
@@ -45,7 +45,7 @@ def generate_thumbnail(
|
|
|
45
45
|
# it is possible for datetime to be null,
|
|
46
46
|
# if it is start and end datetime have to exist
|
|
47
47
|
if item_datetime:
|
|
48
|
-
time = item_datetime
|
|
48
|
+
time = format_datetime_to_isostring_zulu(item_datetime)
|
|
49
49
|
url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format(
|
|
50
50
|
instanceId,
|
|
51
51
|
wms_config,
|
|
@@ -14,6 +14,7 @@ from dateutil import parser
|
|
|
14
14
|
from owslib.wms import WebMapService
|
|
15
15
|
from owslib.wmts import WebMapTileService
|
|
16
16
|
from pystac import Catalog, Collection, Item, RelType
|
|
17
|
+
from pytz import timezone as pytztimezone
|
|
17
18
|
from six import string_types
|
|
18
19
|
from structlog import get_logger
|
|
19
20
|
|
|
@@ -57,7 +58,7 @@ def create_geojson_from_bbox(bbox: list[float | int]) -> dict:
|
|
|
57
58
|
|
|
58
59
|
def retrieveExtentFromWMSWMTS(
|
|
59
60
|
capabilities_url: str, layer: str, version: str = "1.1.1", wmts: bool = False
|
|
60
|
-
) -> tuple[list[float], list[
|
|
61
|
+
) -> tuple[list[float], list[datetime]]:
|
|
61
62
|
times = []
|
|
62
63
|
try:
|
|
63
64
|
if not wmts:
|
|
@@ -96,7 +97,9 @@ def retrieveExtentFromWMSWMTS(
|
|
|
96
97
|
bbox = [-180.0, -90.0, 180.0, 90.0]
|
|
97
98
|
if service and service[layer].boundingBoxWGS84:
|
|
98
99
|
bbox = [float(x) for x in service[layer].boundingBoxWGS84]
|
|
99
|
-
|
|
100
|
+
|
|
101
|
+
datetimes = [parse_datestring_to_tz_aware_datetime(time_str) for time_str in times]
|
|
102
|
+
return bbox, datetimes
|
|
100
103
|
|
|
101
104
|
|
|
102
105
|
def interval(start: datetime, stop: datetime, delta: timedelta) -> Iterator[datetime]:
|
|
@@ -151,19 +154,20 @@ def parse_duration(datestring):
|
|
|
151
154
|
return ret
|
|
152
155
|
|
|
153
156
|
|
|
154
|
-
def
|
|
157
|
+
def generateDatetimesFromInterval(
|
|
155
158
|
start: str, end: str, timedelta_config: dict | None = None
|
|
156
|
-
) -> list[
|
|
159
|
+
) -> list[datetime]:
|
|
157
160
|
if timedelta_config is None:
|
|
158
161
|
timedelta_config = {}
|
|
159
|
-
start_dt =
|
|
162
|
+
start_dt = parse_datestring_to_tz_aware_datetime(start)
|
|
160
163
|
if end == "today":
|
|
161
|
-
|
|
162
|
-
|
|
164
|
+
end_dt = datetime.now(tz=timezone.utc)
|
|
165
|
+
else:
|
|
166
|
+
end_dt = parse_datestring_to_tz_aware_datetime(end)
|
|
163
167
|
delta = timedelta(**timedelta_config)
|
|
164
168
|
dates = []
|
|
165
169
|
while start_dt <= end_dt:
|
|
166
|
-
dates.append(start_dt
|
|
170
|
+
dates.append(start_dt)
|
|
167
171
|
start_dt += delta
|
|
168
172
|
return dates
|
|
169
173
|
|
|
@@ -251,9 +255,9 @@ def add_single_item_if_collection_empty(collection: Collection) -> None:
|
|
|
251
255
|
bbox=[-180, -85, 180, 85],
|
|
252
256
|
properties={},
|
|
253
257
|
geometry=None,
|
|
254
|
-
datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
255
|
-
start_datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
256
|
-
end_datetime=datetime.now(),
|
|
258
|
+
datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
259
|
+
start_datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
260
|
+
end_datetime=datetime.now(tz=pytztimezone("UTC")),
|
|
257
261
|
)
|
|
258
262
|
collection.add_item(item)
|
|
259
263
|
|
|
@@ -309,19 +313,28 @@ def retry(exceptions, tries=3, delay=2, backoff=1, logger=None):
|
|
|
309
313
|
return decorator
|
|
310
314
|
|
|
311
315
|
|
|
312
|
-
def filter_time_entries(time_entries: list[
|
|
316
|
+
def filter_time_entries(time_entries: list[datetime], query: dict[str, str]) -> list[datetime]:
|
|
313
317
|
datetime_query = [
|
|
314
|
-
|
|
315
|
-
|
|
318
|
+
time_entries[0],
|
|
319
|
+
time_entries[-1],
|
|
316
320
|
]
|
|
317
321
|
if start := query.get("Start"):
|
|
318
|
-
datetime_query[0] =
|
|
322
|
+
datetime_query[0] = parse_datestring_to_tz_aware_datetime(start)
|
|
319
323
|
if end := query.get("End"):
|
|
320
|
-
datetime_query[1] =
|
|
324
|
+
datetime_query[1] = parse_datestring_to_tz_aware_datetime(end)
|
|
321
325
|
# filter times based on query Start/End
|
|
322
|
-
time_entries = [
|
|
323
|
-
datetime_str
|
|
324
|
-
for datetime_str in time_entries
|
|
325
|
-
if datetime_query[0] <= parser.isoparse(datetime_str) < datetime_query[1]
|
|
326
|
-
]
|
|
326
|
+
time_entries = [dt for dt in time_entries if datetime_query[0] <= dt < datetime_query[1]]
|
|
327
327
|
return time_entries
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def parse_datestring_to_tz_aware_datetime(datestring: str) -> datetime:
|
|
331
|
+
dt = parser.isoparse(datestring)
|
|
332
|
+
dt = pytztimezone("UTC").localize(dt) if dt.tzinfo is None else dt
|
|
333
|
+
return dt
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def format_datetime_to_isostring_zulu(datetime_obj: datetime) -> str:
|
|
337
|
+
# although "+00:00" is a valid ISO 8601 timezone designation for UTC,
|
|
338
|
+
# we rather convert it to Zulu based string in order for various clients
|
|
339
|
+
# to understand it better (WMS)
|
|
340
|
+
return (datetime_obj.replace(microsecond=0).isoformat()).replace("+00:00", "Z")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_CROPOMAT1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_see_solar_energy.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_2.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-collections/test_wms_no_time.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.31}/tests/testing-indicators/test_indicator.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|