eodash_catalog 0.0.30__tar.gz → 0.0.32__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of eodash_catalog might be problematic. Click here for more details.
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/.bumpversion.cfg +1 -1
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/PKG-INFO +1 -1
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/__about__.py +1 -1
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/endpoints.py +118 -99
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/stac_handling.py +16 -19
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/thumbnails.py +2 -2
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/utils.py +34 -21
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/.github/workflows/ci.yml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/.github/workflows/python-publish.yml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/.gitignore +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/.vscode/extensions.json +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/.vscode/settings.json +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/LICENSE.txt +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/README.md +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/pyproject.toml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/requirements.txt +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/ruff.toml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/__init__.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/duration.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/generate_indicators.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/src/eodash_catalog/sh_endpoint.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/__init__.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/test-data/regional_forecast.json +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/test_generate.py +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-catalogs/testing.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_CROPOMAT1.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_see_solar_energy.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_tif_demo_1.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_tif_demo_2.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_wms_no_time.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-indicators/test_indicator.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-layers/baselayers.yaml +0 -0
- {eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-layers/overlays.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: eodash_catalog
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.32
|
|
4
4
|
Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
|
|
5
5
|
Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
|
|
@@ -9,7 +9,6 @@ from itertools import groupby
|
|
|
9
9
|
from operator import itemgetter
|
|
10
10
|
|
|
11
11
|
import requests
|
|
12
|
-
from dateutil import parser
|
|
13
12
|
from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries
|
|
14
13
|
from pystac_client import Client
|
|
15
14
|
from structlog import get_logger
|
|
@@ -19,7 +18,7 @@ from eodash_catalog.stac_handling import (
|
|
|
19
18
|
add_collection_information,
|
|
20
19
|
add_example_info,
|
|
21
20
|
add_projection_info,
|
|
22
|
-
|
|
21
|
+
get_collection_datetimes_from_config,
|
|
23
22
|
get_or_create_collection,
|
|
24
23
|
)
|
|
25
24
|
from eodash_catalog.thumbnails import generate_thumbnail
|
|
@@ -28,7 +27,9 @@ from eodash_catalog.utils import (
|
|
|
28
27
|
create_geojson_from_bbox,
|
|
29
28
|
create_geojson_point,
|
|
30
29
|
filter_time_entries,
|
|
30
|
+
format_datetime_to_isostring_zulu,
|
|
31
31
|
generate_veda_cog_link,
|
|
32
|
+
parse_datestring_to_tz_aware_datetime,
|
|
32
33
|
replace_with_env_variables,
|
|
33
34
|
retrieveExtentFromWMSWMTS,
|
|
34
35
|
)
|
|
@@ -69,29 +70,26 @@ def process_STAC_Datacube_Endpoint(
|
|
|
69
70
|
if v.get("type") == "temporal":
|
|
70
71
|
time_dimension = k
|
|
71
72
|
break
|
|
72
|
-
|
|
73
|
+
datetimes = [
|
|
74
|
+
parse_datestring_to_tz_aware_datetime(time_string)
|
|
75
|
+
for time_string in dimensions.get(time_dimension).get("values")
|
|
76
|
+
]
|
|
73
77
|
# optionally subset time results based on config
|
|
74
78
|
if query := endpoint_config.get("Query"):
|
|
75
|
-
|
|
79
|
+
datetimes = filter_time_entries(datetimes, query)
|
|
76
80
|
|
|
77
|
-
for
|
|
78
|
-
|
|
79
|
-
id=
|
|
81
|
+
for dt in datetimes:
|
|
82
|
+
new_item = Item(
|
|
83
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
80
84
|
bbox=item.bbox,
|
|
81
85
|
properties={},
|
|
82
86
|
geometry=item.geometry,
|
|
83
|
-
datetime=
|
|
87
|
+
datetime=dt,
|
|
84
88
|
)
|
|
85
|
-
link = collection.add_item(
|
|
86
|
-
link.extra_fields["datetime"] = t
|
|
89
|
+
link = collection.add_item(new_item)
|
|
87
90
|
# bubble up information we want to the link
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
if item_datetime:
|
|
91
|
-
link.extra_fields["datetime"] = item_datetime.isoformat()[:-6] + "Z"
|
|
92
|
-
else:
|
|
93
|
-
link.extra_fields["start_datetime"] = item.properties["start_datetime"]
|
|
94
|
-
link.extra_fields["end_datetime"] = item.properties["end_datetime"]
|
|
91
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
92
|
+
|
|
95
93
|
unit = variables.get(endpoint_config.get("Variable")).get("unit")
|
|
96
94
|
if unit and "yAxis" not in collection_config:
|
|
97
95
|
collection_config["yAxis"] = unit
|
|
@@ -238,16 +236,18 @@ def process_STACAPI_Endpoint(
|
|
|
238
236
|
)
|
|
239
237
|
link.extra_fields["cog_href"] = item.assets["cog_default"].href
|
|
240
238
|
elif item_datetime:
|
|
241
|
-
|
|
242
|
-
|
|
239
|
+
add_visualization_info(
|
|
240
|
+
item, collection_config, endpoint_config, datetimes=[item_datetime]
|
|
241
|
+
)
|
|
243
242
|
elif "start_datetime" in item.properties and "end_datetime" in item.properties:
|
|
244
243
|
add_visualization_info(
|
|
245
244
|
item,
|
|
246
245
|
collection_config,
|
|
247
246
|
endpoint_config,
|
|
248
|
-
|
|
249
|
-
item.properties["start_datetime"],
|
|
250
|
-
|
|
247
|
+
datetimes=[
|
|
248
|
+
parse_datestring_to_tz_aware_datetime(item.properties["start_datetime"]),
|
|
249
|
+
parse_datestring_to_tz_aware_datetime(item.properties["end_datetime"]),
|
|
250
|
+
],
|
|
251
251
|
)
|
|
252
252
|
# If a root collection exists we point back to it from the item
|
|
253
253
|
if root_collection:
|
|
@@ -256,15 +256,14 @@ def process_STACAPI_Endpoint(
|
|
|
256
256
|
# bubble up information we want to the link
|
|
257
257
|
# it is possible for datetime to be null, if it is start and end datetime have to exist
|
|
258
258
|
if item_datetime:
|
|
259
|
-
|
|
260
|
-
if endpoint_config["Name"] == "Sentinel Hub":
|
|
261
|
-
# for SH WMS we only save the date (no time)
|
|
262
|
-
link.extra_fields["datetime"] = iso_date
|
|
263
|
-
else:
|
|
264
|
-
link.extra_fields["datetime"] = iso_time
|
|
259
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(item_datetime)
|
|
265
260
|
else:
|
|
266
|
-
link.extra_fields["start_datetime"] =
|
|
267
|
-
|
|
261
|
+
link.extra_fields["start_datetime"] = format_datetime_to_isostring_zulu(
|
|
262
|
+
parse_datestring_to_tz_aware_datetime(item.properties["start_datetime"])
|
|
263
|
+
)
|
|
264
|
+
link.extra_fields["end_datetime"] = format_datetime_to_isostring_zulu(
|
|
265
|
+
parse_datestring_to_tz_aware_datetime(item.properties["end_datetime"])
|
|
266
|
+
)
|
|
268
267
|
add_projection_info(
|
|
269
268
|
endpoint_config,
|
|
270
269
|
item,
|
|
@@ -305,18 +304,18 @@ def handle_collection_only(
|
|
|
305
304
|
collection = get_or_create_collection(
|
|
306
305
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
307
306
|
)
|
|
308
|
-
|
|
309
|
-
if len(
|
|
310
|
-
for
|
|
307
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
308
|
+
if len(datetimes) > 0:
|
|
309
|
+
for dt in datetimes:
|
|
311
310
|
item = Item(
|
|
312
|
-
id=
|
|
311
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
313
312
|
bbox=endpoint_config.get("OverwriteBBox"),
|
|
314
313
|
properties={},
|
|
315
314
|
geometry=None,
|
|
316
|
-
datetime=
|
|
315
|
+
datetime=dt,
|
|
317
316
|
)
|
|
318
317
|
link = collection.add_item(item)
|
|
319
|
-
link.extra_fields["datetime"] =
|
|
318
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
320
319
|
add_collection_information(catalog_config, collection, collection_config)
|
|
321
320
|
# eodash v4 compatibility
|
|
322
321
|
add_visualization_info(collection, collection_config, endpoint_config)
|
|
@@ -342,21 +341,22 @@ def handle_SH_WMS_endpoint(
|
|
|
342
341
|
catalog, location["Identifier"], location_config, catalog_config, endpoint_config
|
|
343
342
|
)
|
|
344
343
|
collection.extra_fields["endpointtype"] = endpoint_config["Name"]
|
|
345
|
-
for
|
|
344
|
+
for time_string in location["Times"]:
|
|
345
|
+
dt = parse_datestring_to_tz_aware_datetime(time_string)
|
|
346
346
|
item = Item(
|
|
347
|
-
id=
|
|
347
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
348
348
|
bbox=location["Bbox"],
|
|
349
349
|
properties={},
|
|
350
350
|
geometry=None,
|
|
351
|
-
datetime=
|
|
351
|
+
datetime=dt,
|
|
352
352
|
stac_extensions=[
|
|
353
353
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
354
354
|
],
|
|
355
355
|
)
|
|
356
356
|
add_projection_info(endpoint_config, item)
|
|
357
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
357
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
358
358
|
item_link = collection.add_item(item)
|
|
359
|
-
item_link.extra_fields["datetime"] =
|
|
359
|
+
item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
360
360
|
|
|
361
361
|
link = root_collection.add_child(collection)
|
|
362
362
|
# bubble up information we want to the link
|
|
@@ -376,23 +376,23 @@ def handle_SH_WMS_endpoint(
|
|
|
376
376
|
else:
|
|
377
377
|
# if locations are not provided, treat the collection as a
|
|
378
378
|
# general proxy to the sentinel hub layer
|
|
379
|
-
|
|
379
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
380
380
|
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
381
|
-
for
|
|
381
|
+
for dt in datetimes:
|
|
382
382
|
item = Item(
|
|
383
|
-
id=
|
|
383
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
384
384
|
bbox=bbox,
|
|
385
385
|
properties={},
|
|
386
386
|
geometry=None,
|
|
387
|
-
datetime=
|
|
387
|
+
datetime=dt,
|
|
388
388
|
stac_extensions=[
|
|
389
389
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
390
390
|
],
|
|
391
391
|
)
|
|
392
392
|
add_projection_info(endpoint_config, item)
|
|
393
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
393
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
394
394
|
item_link = root_collection.add_item(item)
|
|
395
|
-
item_link.extra_fields["datetime"] =
|
|
395
|
+
item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
396
396
|
# eodash v4 compatibility
|
|
397
397
|
add_collection_information(catalog_config, root_collection, collection_config)
|
|
398
398
|
add_visualization_info(root_collection, collection_config, endpoint_config)
|
|
@@ -538,13 +538,13 @@ def handle_WMS_endpoint(
|
|
|
538
538
|
collection = get_or_create_collection(
|
|
539
539
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
540
540
|
)
|
|
541
|
-
|
|
541
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
542
542
|
spatial_extent = collection.extent.spatial.to_dict().get("bbox", [-180, -90, 180, 90])[0]
|
|
543
543
|
if endpoint_config.get("Type") != "OverwriteTimes" or not endpoint_config.get("OverwriteBBox"):
|
|
544
544
|
# some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
|
|
545
545
|
# have to process full service capabilities XML
|
|
546
546
|
capabilities_url = endpoint_config["EndPoint"]
|
|
547
|
-
spatial_extent,
|
|
547
|
+
spatial_extent, datetimes = retrieveExtentFromWMSWMTS(
|
|
548
548
|
capabilities_url,
|
|
549
549
|
endpoint_config["LayerId"],
|
|
550
550
|
version=endpoint_config.get("Version", "1.1.1"),
|
|
@@ -552,24 +552,24 @@ def handle_WMS_endpoint(
|
|
|
552
552
|
)
|
|
553
553
|
# optionally filter time results
|
|
554
554
|
if query := endpoint_config.get("Query"):
|
|
555
|
-
|
|
555
|
+
datetimes = filter_time_entries(datetimes, query)
|
|
556
556
|
# Create an item per time to allow visualization in stac clients
|
|
557
|
-
if len(
|
|
558
|
-
for
|
|
557
|
+
if len(datetimes) > 0:
|
|
558
|
+
for dt in datetimes:
|
|
559
559
|
item = Item(
|
|
560
|
-
id=
|
|
560
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
561
561
|
bbox=spatial_extent,
|
|
562
562
|
properties={},
|
|
563
563
|
geometry=None,
|
|
564
|
-
datetime=
|
|
564
|
+
datetime=dt,
|
|
565
565
|
stac_extensions=[
|
|
566
566
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
567
567
|
],
|
|
568
568
|
)
|
|
569
569
|
add_projection_info(endpoint_config, item)
|
|
570
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
570
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
571
571
|
link = collection.add_item(item)
|
|
572
|
-
link.extra_fields["datetime"] =
|
|
572
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
573
573
|
collection.update_extent_from_items()
|
|
574
574
|
|
|
575
575
|
# Check if we should overwrite bbox
|
|
@@ -606,7 +606,7 @@ def add_visualization_info(
|
|
|
606
606
|
collection_config: dict,
|
|
607
607
|
endpoint_config: dict,
|
|
608
608
|
file_url: str | None = None,
|
|
609
|
-
|
|
609
|
+
datetimes: list[datetime] | None = None,
|
|
610
610
|
) -> None:
|
|
611
611
|
extra_fields: dict[str, list[str] | dict[str, str]] = {}
|
|
612
612
|
if "Attribution" in endpoint_config:
|
|
@@ -630,27 +630,32 @@ def add_visualization_info(
|
|
|
630
630
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
631
631
|
for key, value in dimensions_config.items():
|
|
632
632
|
dimensions[key] = value
|
|
633
|
-
if
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
633
|
+
if datetimes is not None:
|
|
634
|
+
dt = datetimes[0]
|
|
635
|
+
start_isostring = format_datetime_to_isostring_zulu(dt)
|
|
636
|
+
# SH WMS for public collections needs time interval, we use full day here
|
|
637
|
+
end = dt + timedelta(days=1) - timedelta(milliseconds=1)
|
|
638
|
+
# we have start_datetime and end_datetime
|
|
639
|
+
if len(datetimes) == 2:
|
|
640
|
+
end = datetimes[1]
|
|
641
|
+
end_isostring = format_datetime_to_isostring_zulu(end)
|
|
642
|
+
time_interval = f"{start_isostring}/{end_isostring}"
|
|
643
|
+
dimensions["TIME"] = time_interval
|
|
644
|
+
|
|
643
645
|
if dimensions != {}:
|
|
644
646
|
extra_fields["wms:dimensions"] = dimensions
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
647
|
+
link = Link(
|
|
648
|
+
rel="wms",
|
|
649
|
+
target=f"https://services.sentinel-hub.com/ogc/wms/{instanceId}",
|
|
650
|
+
media_type=(endpoint_config.get("MimeType", "image/png")),
|
|
651
|
+
title=collection_config["Name"],
|
|
652
|
+
extra_fields=extra_fields,
|
|
653
|
+
)
|
|
654
|
+
add_projection_info(
|
|
655
|
+
endpoint_config,
|
|
656
|
+
link,
|
|
653
657
|
)
|
|
658
|
+
stac_object.add_link(link)
|
|
654
659
|
elif endpoint_config["Name"] == "WMS":
|
|
655
660
|
extra_fields.update(
|
|
656
661
|
{
|
|
@@ -662,8 +667,8 @@ def add_visualization_info(
|
|
|
662
667
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
663
668
|
for key, value in dimensions_config.items():
|
|
664
669
|
dimensions[key] = value
|
|
665
|
-
if
|
|
666
|
-
dimensions["TIME"] =
|
|
670
|
+
if datetimes is not None:
|
|
671
|
+
dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
|
|
667
672
|
if dimensions != {}:
|
|
668
673
|
extra_fields["wms:dimensions"] = dimensions
|
|
669
674
|
if "Styles" in endpoint_config:
|
|
@@ -672,20 +677,30 @@ def add_visualization_info(
|
|
|
672
677
|
endpoint_url = endpoint_config["EndPoint"]
|
|
673
678
|
# custom replacing of all ENV VARS present as template in URL as {VAR}
|
|
674
679
|
endpoint_url = replace_with_env_variables(endpoint_url)
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
680
|
+
link = Link(
|
|
681
|
+
rel="wms",
|
|
682
|
+
target=endpoint_url,
|
|
683
|
+
media_type=media_type,
|
|
684
|
+
title=collection_config["Name"],
|
|
685
|
+
extra_fields=extra_fields,
|
|
686
|
+
)
|
|
687
|
+
add_projection_info(
|
|
688
|
+
endpoint_config,
|
|
689
|
+
link,
|
|
683
690
|
)
|
|
691
|
+
stac_object.add_link(link)
|
|
684
692
|
elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
|
|
685
693
|
target_url = "{}".format(endpoint_config.get("EndPoint"))
|
|
686
694
|
# custom time just for this special case as a default for collection wmts
|
|
695
|
+
time = None
|
|
696
|
+
if datetimes is not None:
|
|
697
|
+
time = datetimes[0]
|
|
687
698
|
extra_fields.update(
|
|
688
|
-
{
|
|
699
|
+
{
|
|
700
|
+
"wmts:layer": endpoint_config.get("LayerId", "").replace(
|
|
701
|
+
"{time}", (time and str(time.year)) or "2017"
|
|
702
|
+
)
|
|
703
|
+
}
|
|
689
704
|
)
|
|
690
705
|
stac_object.add_link(
|
|
691
706
|
Link(
|
|
@@ -739,8 +754,8 @@ def add_visualization_info(
|
|
|
739
754
|
}
|
|
740
755
|
)
|
|
741
756
|
dimensions = {}
|
|
742
|
-
if
|
|
743
|
-
dimensions["time"] =
|
|
757
|
+
if datetimes is not None:
|
|
758
|
+
dimensions["time"] = format_datetime_to_isostring_zulu(datetimes[0])
|
|
744
759
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
745
760
|
for key, value in dimensions_config.items():
|
|
746
761
|
dimensions[key] = value
|
|
@@ -761,15 +776,18 @@ def add_visualization_info(
|
|
|
761
776
|
elif endpoint_config["Type"] == "tiles":
|
|
762
777
|
target_url = generate_veda_tiles_link(endpoint_config, file_url)
|
|
763
778
|
if target_url:
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
779
|
+
link = Link(
|
|
780
|
+
rel="xyz",
|
|
781
|
+
target=target_url,
|
|
782
|
+
media_type="image/png",
|
|
783
|
+
title=collection_config["Name"],
|
|
784
|
+
extra_fields=extra_fields,
|
|
785
|
+
)
|
|
786
|
+
add_projection_info(
|
|
787
|
+
endpoint_config,
|
|
788
|
+
link,
|
|
772
789
|
)
|
|
790
|
+
stac_object.add_link(link)
|
|
773
791
|
elif endpoint_config["Name"] == "GeoDB Vector Tiles":
|
|
774
792
|
# `${geoserverUrl}${config.layerName}@EPSG%3A${projString}@pbf/{z}/{x}/{-y}.pbf`,
|
|
775
793
|
# 'geodb_debd884d-92f9-4979-87b6-eadef1139394:GTIF_AT_Gemeinden_3857'
|
|
@@ -863,12 +881,13 @@ def handle_raw_source(
|
|
|
863
881
|
add_projection_info(endpoint_config, asset)
|
|
864
882
|
assets[a["Identifier"]] = asset
|
|
865
883
|
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
884
|
+
dt = parse_datestring_to_tz_aware_datetime(time_entry["Time"])
|
|
866
885
|
item = Item(
|
|
867
|
-
id=
|
|
886
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
868
887
|
bbox=bbox,
|
|
869
888
|
properties={},
|
|
870
889
|
geometry=create_geojson_from_bbox(bbox),
|
|
871
|
-
datetime=
|
|
890
|
+
datetime=dt,
|
|
872
891
|
assets=assets,
|
|
873
892
|
extra_fields={},
|
|
874
893
|
)
|
|
@@ -889,7 +908,7 @@ def handle_raw_source(
|
|
|
889
908
|
)
|
|
890
909
|
item.add_link(style_link)
|
|
891
910
|
link = collection.add_item(item)
|
|
892
|
-
link.extra_fields["datetime"] =
|
|
911
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
893
912
|
link.extra_fields["assets"] = [a["File"] for a in time_entry["Assets"]]
|
|
894
913
|
# eodash v4 compatibility, adding last referenced style to collection
|
|
895
914
|
if style_link:
|
|
@@ -3,7 +3,6 @@ from datetime import datetime
|
|
|
3
3
|
import requests
|
|
4
4
|
import spdx_lookup as lookup
|
|
5
5
|
import yaml
|
|
6
|
-
from dateutil import parser
|
|
7
6
|
from pystac import (
|
|
8
7
|
Asset,
|
|
9
8
|
Catalog,
|
|
@@ -18,7 +17,10 @@ from pystac import (
|
|
|
18
17
|
from structlog import get_logger
|
|
19
18
|
from yaml.loader import SafeLoader
|
|
20
19
|
|
|
21
|
-
from eodash_catalog.utils import
|
|
20
|
+
from eodash_catalog.utils import (
|
|
21
|
+
generateDatetimesFromInterval,
|
|
22
|
+
parse_datestring_to_tz_aware_datetime,
|
|
23
|
+
)
|
|
22
24
|
|
|
23
25
|
LOGGER = get_logger(__name__)
|
|
24
26
|
|
|
@@ -42,20 +44,12 @@ def get_or_create_collection(
|
|
|
42
44
|
spatial_extent,
|
|
43
45
|
]
|
|
44
46
|
)
|
|
45
|
-
times: list[str] = []
|
|
46
47
|
temporal_extent = TemporalExtent([[datetime.now(), None]])
|
|
47
48
|
if endpoint_config:
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
51
|
-
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
52
|
-
elif endpoint_config.get("DateTimeInterval"):
|
|
53
|
-
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00")
|
|
54
|
-
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00")
|
|
55
|
-
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
56
|
-
times = generateDateIsostringsFromInterval(start, end, timedelta_config)
|
|
57
|
-
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
49
|
+
times_datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
50
|
+
if len(times_datetimes) > 0:
|
|
58
51
|
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
52
|
+
|
|
59
53
|
extent = Extent(spatial=spatial_extent, temporal=temporal_extent)
|
|
60
54
|
|
|
61
55
|
# Check if description is link to markdown file
|
|
@@ -387,17 +381,20 @@ def add_extra_fields(stac_object: Collection | Link, collection_config: dict) ->
|
|
|
387
381
|
stac_object.extra_fields["eodash:mapProjection"] = collection_config["MapProjection"]
|
|
388
382
|
|
|
389
383
|
|
|
390
|
-
def
|
|
391
|
-
|
|
384
|
+
def get_collection_datetimes_from_config(endpoint_config: dict) -> list[datetime]:
|
|
385
|
+
times_datetimes: list[datetime] = []
|
|
392
386
|
if endpoint_config:
|
|
393
387
|
if endpoint_config.get("Times"):
|
|
394
388
|
times = list(endpoint_config.get("Times", []))
|
|
389
|
+
times_datetimes = sorted(
|
|
390
|
+
[parse_datestring_to_tz_aware_datetime(time) for time in times]
|
|
391
|
+
)
|
|
395
392
|
elif endpoint_config.get("DateTimeInterval"):
|
|
396
|
-
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:
|
|
397
|
-
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:
|
|
393
|
+
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00Z")
|
|
394
|
+
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00Z")
|
|
398
395
|
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
399
|
-
|
|
400
|
-
return
|
|
396
|
+
times_datetimes = generateDatetimesFromInterval(start, end, timedelta_config)
|
|
397
|
+
return times_datetimes
|
|
401
398
|
|
|
402
399
|
|
|
403
400
|
def add_projection_info(
|
|
@@ -7,7 +7,7 @@ from pystac import (
|
|
|
7
7
|
Item,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
|
-
from eodash_catalog.utils import generate_veda_cog_link
|
|
10
|
+
from eodash_catalog.utils import format_datetime_to_isostring_zulu, generate_veda_cog_link
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def fetch_and_save_thumbnail(collection_config: dict, url: str) -> None:
|
|
@@ -45,7 +45,7 @@ def generate_thumbnail(
|
|
|
45
45
|
# it is possible for datetime to be null,
|
|
46
46
|
# if it is start and end datetime have to exist
|
|
47
47
|
if item_datetime:
|
|
48
|
-
time = item_datetime
|
|
48
|
+
time = format_datetime_to_isostring_zulu(item_datetime)
|
|
49
49
|
url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format(
|
|
50
50
|
instanceId,
|
|
51
51
|
wms_config,
|
|
@@ -14,6 +14,7 @@ from dateutil import parser
|
|
|
14
14
|
from owslib.wms import WebMapService
|
|
15
15
|
from owslib.wmts import WebMapTileService
|
|
16
16
|
from pystac import Catalog, Collection, Item, RelType
|
|
17
|
+
from pytz import timezone as pytztimezone
|
|
17
18
|
from six import string_types
|
|
18
19
|
from structlog import get_logger
|
|
19
20
|
|
|
@@ -57,7 +58,7 @@ def create_geojson_from_bbox(bbox: list[float | int]) -> dict:
|
|
|
57
58
|
|
|
58
59
|
def retrieveExtentFromWMSWMTS(
|
|
59
60
|
capabilities_url: str, layer: str, version: str = "1.1.1", wmts: bool = False
|
|
60
|
-
) -> tuple[list[float], list[
|
|
61
|
+
) -> tuple[list[float], list[datetime]]:
|
|
61
62
|
times = []
|
|
62
63
|
try:
|
|
63
64
|
if not wmts:
|
|
@@ -96,7 +97,9 @@ def retrieveExtentFromWMSWMTS(
|
|
|
96
97
|
bbox = [-180.0, -90.0, 180.0, 90.0]
|
|
97
98
|
if service and service[layer].boundingBoxWGS84:
|
|
98
99
|
bbox = [float(x) for x in service[layer].boundingBoxWGS84]
|
|
99
|
-
|
|
100
|
+
|
|
101
|
+
datetimes = [parse_datestring_to_tz_aware_datetime(time_str) for time_str in times]
|
|
102
|
+
return bbox, datetimes
|
|
100
103
|
|
|
101
104
|
|
|
102
105
|
def interval(start: datetime, stop: datetime, delta: timedelta) -> Iterator[datetime]:
|
|
@@ -151,19 +154,20 @@ def parse_duration(datestring):
|
|
|
151
154
|
return ret
|
|
152
155
|
|
|
153
156
|
|
|
154
|
-
def
|
|
157
|
+
def generateDatetimesFromInterval(
|
|
155
158
|
start: str, end: str, timedelta_config: dict | None = None
|
|
156
|
-
) -> list[
|
|
159
|
+
) -> list[datetime]:
|
|
157
160
|
if timedelta_config is None:
|
|
158
161
|
timedelta_config = {}
|
|
159
|
-
start_dt =
|
|
162
|
+
start_dt = parse_datestring_to_tz_aware_datetime(start)
|
|
160
163
|
if end == "today":
|
|
161
|
-
|
|
162
|
-
|
|
164
|
+
end_dt = datetime.now(tz=timezone.utc)
|
|
165
|
+
else:
|
|
166
|
+
end_dt = parse_datestring_to_tz_aware_datetime(end)
|
|
163
167
|
delta = timedelta(**timedelta_config)
|
|
164
168
|
dates = []
|
|
165
169
|
while start_dt <= end_dt:
|
|
166
|
-
dates.append(start_dt
|
|
170
|
+
dates.append(start_dt)
|
|
167
171
|
start_dt += delta
|
|
168
172
|
return dates
|
|
169
173
|
|
|
@@ -251,9 +255,9 @@ def add_single_item_if_collection_empty(collection: Collection) -> None:
|
|
|
251
255
|
bbox=[-180, -85, 180, 85],
|
|
252
256
|
properties={},
|
|
253
257
|
geometry=None,
|
|
254
|
-
datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
255
|
-
start_datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
256
|
-
end_datetime=datetime.now(),
|
|
258
|
+
datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
259
|
+
start_datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
260
|
+
end_datetime=datetime.now(tz=pytztimezone("UTC")),
|
|
257
261
|
)
|
|
258
262
|
collection.add_item(item)
|
|
259
263
|
|
|
@@ -309,19 +313,28 @@ def retry(exceptions, tries=3, delay=2, backoff=1, logger=None):
|
|
|
309
313
|
return decorator
|
|
310
314
|
|
|
311
315
|
|
|
312
|
-
def filter_time_entries(time_entries: list[
|
|
316
|
+
def filter_time_entries(time_entries: list[datetime], query: dict[str, str]) -> list[datetime]:
|
|
313
317
|
datetime_query = [
|
|
314
|
-
|
|
315
|
-
|
|
318
|
+
time_entries[0],
|
|
319
|
+
time_entries[-1],
|
|
316
320
|
]
|
|
317
321
|
if start := query.get("Start"):
|
|
318
|
-
datetime_query[0] =
|
|
322
|
+
datetime_query[0] = parse_datestring_to_tz_aware_datetime(start)
|
|
319
323
|
if end := query.get("End"):
|
|
320
|
-
datetime_query[1] =
|
|
324
|
+
datetime_query[1] = parse_datestring_to_tz_aware_datetime(end)
|
|
321
325
|
# filter times based on query Start/End
|
|
322
|
-
time_entries = [
|
|
323
|
-
datetime_str
|
|
324
|
-
for datetime_str in time_entries
|
|
325
|
-
if datetime_query[0] <= parser.isoparse(datetime_str) < datetime_query[1]
|
|
326
|
-
]
|
|
326
|
+
time_entries = [dt for dt in time_entries if datetime_query[0] <= dt < datetime_query[1]]
|
|
327
327
|
return time_entries
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def parse_datestring_to_tz_aware_datetime(datestring: str) -> datetime:
|
|
331
|
+
dt = parser.isoparse(datestring)
|
|
332
|
+
dt = pytztimezone("UTC").localize(dt) if dt.tzinfo is None else dt
|
|
333
|
+
return dt
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def format_datetime_to_isostring_zulu(datetime_obj: datetime) -> str:
|
|
337
|
+
# although "+00:00" is a valid ISO 8601 timezone designation for UTC,
|
|
338
|
+
# we rather convert it to Zulu based string in order for various clients
|
|
339
|
+
# to understand it better (WMS)
|
|
340
|
+
return (datetime_obj.replace(microsecond=0).isoformat()).replace("+00:00", "Z")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_CROPOMAT1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_see_solar_energy.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_tif_demo_1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_tif_demo_2.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-collections/test_wms_no_time.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.30 → eodash_catalog-0.0.32}/tests/testing-indicators/test_indicator.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|