eodash_catalog 0.0.29__tar.gz → 0.0.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of eodash_catalog might be problematic. Click here for more details.
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/.bumpversion.cfg +1 -1
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/PKG-INFO +1 -1
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/__about__.py +1 -1
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/endpoints.py +139 -96
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/stac_handling.py +18 -21
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/thumbnails.py +2 -2
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/utils.py +34 -21
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/.github/workflows/ci.yml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/.github/workflows/python-publish.yml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/.gitignore +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/.vscode/extensions.json +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/.vscode/settings.json +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/LICENSE.txt +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/README.md +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/pyproject.toml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/requirements.txt +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/ruff.toml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/__init__.py +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/duration.py +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/generate_indicators.py +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/src/eodash_catalog/sh_endpoint.py +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/__init__.py +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/test-data/regional_forecast.json +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/test_generate.py +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-catalogs/testing.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_CROPOMAT1.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_see_solar_energy.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_1.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_2.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_wms_no_time.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-indicators/test_indicator.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-layers/baselayers.yaml +0 -0
- {eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-layers/overlays.yaml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: eodash_catalog
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.31
|
|
4
4
|
Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
|
|
5
5
|
Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
|
|
@@ -9,7 +9,6 @@ from itertools import groupby
|
|
|
9
9
|
from operator import itemgetter
|
|
10
10
|
|
|
11
11
|
import requests
|
|
12
|
-
from dateutil import parser
|
|
13
12
|
from pystac import Asset, Catalog, Collection, Item, Link, SpatialExtent, Summaries
|
|
14
13
|
from pystac_client import Client
|
|
15
14
|
from structlog import get_logger
|
|
@@ -19,7 +18,7 @@ from eodash_catalog.stac_handling import (
|
|
|
19
18
|
add_collection_information,
|
|
20
19
|
add_example_info,
|
|
21
20
|
add_projection_info,
|
|
22
|
-
|
|
21
|
+
get_collection_datetimes_from_config,
|
|
23
22
|
get_or_create_collection,
|
|
24
23
|
)
|
|
25
24
|
from eodash_catalog.thumbnails import generate_thumbnail
|
|
@@ -28,7 +27,9 @@ from eodash_catalog.utils import (
|
|
|
28
27
|
create_geojson_from_bbox,
|
|
29
28
|
create_geojson_point,
|
|
30
29
|
filter_time_entries,
|
|
30
|
+
format_datetime_to_isostring_zulu,
|
|
31
31
|
generate_veda_cog_link,
|
|
32
|
+
parse_datestring_to_tz_aware_datetime,
|
|
32
33
|
replace_with_env_variables,
|
|
33
34
|
retrieveExtentFromWMSWMTS,
|
|
34
35
|
)
|
|
@@ -69,29 +70,26 @@ def process_STAC_Datacube_Endpoint(
|
|
|
69
70
|
if v.get("type") == "temporal":
|
|
70
71
|
time_dimension = k
|
|
71
72
|
break
|
|
72
|
-
|
|
73
|
+
datetimes = [
|
|
74
|
+
parse_datestring_to_tz_aware_datetime(time_string)
|
|
75
|
+
for time_string in dimensions.get(time_dimension).get("values")
|
|
76
|
+
]
|
|
73
77
|
# optionally subset time results based on config
|
|
74
78
|
if query := endpoint_config.get("Query"):
|
|
75
|
-
|
|
79
|
+
datetimes = filter_time_entries(datetimes, query)
|
|
76
80
|
|
|
77
|
-
for
|
|
78
|
-
|
|
79
|
-
id=
|
|
81
|
+
for dt in datetimes:
|
|
82
|
+
new_item = Item(
|
|
83
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
80
84
|
bbox=item.bbox,
|
|
81
85
|
properties={},
|
|
82
86
|
geometry=item.geometry,
|
|
83
|
-
datetime=
|
|
87
|
+
datetime=dt,
|
|
84
88
|
)
|
|
85
|
-
link = collection.add_item(
|
|
86
|
-
link.extra_fields["datetime"] = t
|
|
89
|
+
link = collection.add_item(new_item)
|
|
87
90
|
# bubble up information we want to the link
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
if item_datetime:
|
|
91
|
-
link.extra_fields["datetime"] = item_datetime.isoformat()[:-6] + "Z"
|
|
92
|
-
else:
|
|
93
|
-
link.extra_fields["start_datetime"] = item.properties["start_datetime"]
|
|
94
|
-
link.extra_fields["end_datetime"] = item.properties["end_datetime"]
|
|
91
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
92
|
+
|
|
95
93
|
unit = variables.get(endpoint_config.get("Variable")).get("unit")
|
|
96
94
|
if unit and "yAxis" not in collection_config:
|
|
97
95
|
collection_config["yAxis"] = unit
|
|
@@ -238,16 +236,15 @@ def process_STACAPI_Endpoint(
|
|
|
238
236
|
)
|
|
239
237
|
link.extra_fields["cog_href"] = item.assets["cog_default"].href
|
|
240
238
|
elif item_datetime:
|
|
241
|
-
|
|
242
|
-
|
|
239
|
+
add_visualization_info(
|
|
240
|
+
item, collection_config, endpoint_config, datetimes=[item_datetime]
|
|
241
|
+
)
|
|
243
242
|
elif "start_datetime" in item.properties and "end_datetime" in item.properties:
|
|
244
243
|
add_visualization_info(
|
|
245
244
|
item,
|
|
246
245
|
collection_config,
|
|
247
246
|
endpoint_config,
|
|
248
|
-
|
|
249
|
-
item.properties["start_datetime"], item.properties["end_datetime"]
|
|
250
|
-
),
|
|
247
|
+
datetimes=[item.properties["start_datetime"], item.properties["end_datetime"]],
|
|
251
248
|
)
|
|
252
249
|
# If a root collection exists we point back to it from the item
|
|
253
250
|
if root_collection:
|
|
@@ -256,12 +253,8 @@ def process_STACAPI_Endpoint(
|
|
|
256
253
|
# bubble up information we want to the link
|
|
257
254
|
# it is possible for datetime to be null, if it is start and end datetime have to exist
|
|
258
255
|
if item_datetime:
|
|
259
|
-
iso_time = item_datetime
|
|
260
|
-
|
|
261
|
-
# for SH WMS we only save the date (no time)
|
|
262
|
-
link.extra_fields["datetime"] = iso_date
|
|
263
|
-
else:
|
|
264
|
-
link.extra_fields["datetime"] = iso_time
|
|
256
|
+
iso_time = format_datetime_to_isostring_zulu(item_datetime)
|
|
257
|
+
link.extra_fields["datetime"] = iso_time
|
|
265
258
|
else:
|
|
266
259
|
link.extra_fields["start_datetime"] = item.properties["start_datetime"]
|
|
267
260
|
link.extra_fields["end_datetime"] = item.properties["end_datetime"]
|
|
@@ -305,18 +298,18 @@ def handle_collection_only(
|
|
|
305
298
|
collection = get_or_create_collection(
|
|
306
299
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
307
300
|
)
|
|
308
|
-
|
|
309
|
-
if len(
|
|
310
|
-
for
|
|
301
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
302
|
+
if len(datetimes) > 0:
|
|
303
|
+
for dt in datetimes:
|
|
311
304
|
item = Item(
|
|
312
|
-
id=
|
|
305
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
313
306
|
bbox=endpoint_config.get("OverwriteBBox"),
|
|
314
307
|
properties={},
|
|
315
308
|
geometry=None,
|
|
316
|
-
datetime=
|
|
309
|
+
datetime=dt,
|
|
317
310
|
)
|
|
318
311
|
link = collection.add_item(item)
|
|
319
|
-
link.extra_fields["datetime"] =
|
|
312
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
320
313
|
add_collection_information(catalog_config, collection, collection_config)
|
|
321
314
|
# eodash v4 compatibility
|
|
322
315
|
add_visualization_info(collection, collection_config, endpoint_config)
|
|
@@ -327,10 +320,10 @@ def handle_SH_WMS_endpoint(
|
|
|
327
320
|
catalog_config: dict, endpoint_config: dict, collection_config: dict, catalog: Catalog
|
|
328
321
|
) -> Collection:
|
|
329
322
|
# create collection and subcollections (based on locations)
|
|
323
|
+
root_collection = get_or_create_collection(
|
|
324
|
+
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
325
|
+
)
|
|
330
326
|
if "Locations" in collection_config:
|
|
331
|
-
root_collection = get_or_create_collection(
|
|
332
|
-
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
333
|
-
)
|
|
334
327
|
for location in collection_config["Locations"]:
|
|
335
328
|
# create and populate location collections based on times
|
|
336
329
|
# TODO: Should we add some new description per location?
|
|
@@ -342,21 +335,22 @@ def handle_SH_WMS_endpoint(
|
|
|
342
335
|
catalog, location["Identifier"], location_config, catalog_config, endpoint_config
|
|
343
336
|
)
|
|
344
337
|
collection.extra_fields["endpointtype"] = endpoint_config["Name"]
|
|
345
|
-
for
|
|
338
|
+
for time_string in location["Times"]:
|
|
339
|
+
dt = parse_datestring_to_tz_aware_datetime(time_string)
|
|
346
340
|
item = Item(
|
|
347
|
-
id=
|
|
341
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
348
342
|
bbox=location["Bbox"],
|
|
349
343
|
properties={},
|
|
350
344
|
geometry=None,
|
|
351
|
-
datetime=
|
|
345
|
+
datetime=dt,
|
|
352
346
|
stac_extensions=[
|
|
353
347
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
354
348
|
],
|
|
355
349
|
)
|
|
356
350
|
add_projection_info(endpoint_config, item)
|
|
357
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
351
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
358
352
|
item_link = collection.add_item(item)
|
|
359
|
-
item_link.extra_fields["datetime"] =
|
|
353
|
+
item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
360
354
|
|
|
361
355
|
link = root_collection.add_child(collection)
|
|
362
356
|
# bubble up information we want to the link
|
|
@@ -373,6 +367,26 @@ def handle_SH_WMS_endpoint(
|
|
|
373
367
|
for c_child in root_collection.get_children():
|
|
374
368
|
if isinstance(c_child, Collection):
|
|
375
369
|
root_collection.extent.spatial.bboxes.append(c_child.extent.spatial.bboxes[0])
|
|
370
|
+
else:
|
|
371
|
+
# if locations are not provided, treat the collection as a
|
|
372
|
+
# general proxy to the sentinel hub layer
|
|
373
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
374
|
+
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
375
|
+
for dt in datetimes:
|
|
376
|
+
item = Item(
|
|
377
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
378
|
+
bbox=bbox,
|
|
379
|
+
properties={},
|
|
380
|
+
geometry=None,
|
|
381
|
+
datetime=dt,
|
|
382
|
+
stac_extensions=[
|
|
383
|
+
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
384
|
+
],
|
|
385
|
+
)
|
|
386
|
+
add_projection_info(endpoint_config, item)
|
|
387
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
388
|
+
item_link = root_collection.add_item(item)
|
|
389
|
+
item_link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
376
390
|
# eodash v4 compatibility
|
|
377
391
|
add_collection_information(catalog_config, root_collection, collection_config)
|
|
378
392
|
add_visualization_info(root_collection, collection_config, endpoint_config)
|
|
@@ -518,13 +532,13 @@ def handle_WMS_endpoint(
|
|
|
518
532
|
collection = get_or_create_collection(
|
|
519
533
|
catalog, collection_config["Name"], collection_config, catalog_config, endpoint_config
|
|
520
534
|
)
|
|
521
|
-
|
|
535
|
+
datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
522
536
|
spatial_extent = collection.extent.spatial.to_dict().get("bbox", [-180, -90, 180, 90])[0]
|
|
523
537
|
if endpoint_config.get("Type") != "OverwriteTimes" or not endpoint_config.get("OverwriteBBox"):
|
|
524
538
|
# some endpoints allow "narrowed-down" capabilities per-layer, which we utilize to not
|
|
525
539
|
# have to process full service capabilities XML
|
|
526
540
|
capabilities_url = endpoint_config["EndPoint"]
|
|
527
|
-
spatial_extent,
|
|
541
|
+
spatial_extent, datetimes = retrieveExtentFromWMSWMTS(
|
|
528
542
|
capabilities_url,
|
|
529
543
|
endpoint_config["LayerId"],
|
|
530
544
|
version=endpoint_config.get("Version", "1.1.1"),
|
|
@@ -532,24 +546,24 @@ def handle_WMS_endpoint(
|
|
|
532
546
|
)
|
|
533
547
|
# optionally filter time results
|
|
534
548
|
if query := endpoint_config.get("Query"):
|
|
535
|
-
|
|
549
|
+
datetimes = filter_time_entries(datetimes, query)
|
|
536
550
|
# Create an item per time to allow visualization in stac clients
|
|
537
|
-
if len(
|
|
538
|
-
for
|
|
551
|
+
if len(datetimes) > 0:
|
|
552
|
+
for dt in datetimes:
|
|
539
553
|
item = Item(
|
|
540
|
-
id=
|
|
554
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
541
555
|
bbox=spatial_extent,
|
|
542
556
|
properties={},
|
|
543
557
|
geometry=None,
|
|
544
|
-
datetime=
|
|
558
|
+
datetime=dt,
|
|
545
559
|
stac_extensions=[
|
|
546
560
|
"https://stac-extensions.github.io/web-map-links/v1.1.0/schema.json",
|
|
547
561
|
],
|
|
548
562
|
)
|
|
549
563
|
add_projection_info(endpoint_config, item)
|
|
550
|
-
add_visualization_info(item, collection_config, endpoint_config,
|
|
564
|
+
add_visualization_info(item, collection_config, endpoint_config, datetimes=[dt])
|
|
551
565
|
link = collection.add_item(item)
|
|
552
|
-
link.extra_fields["datetime"] =
|
|
566
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
553
567
|
collection.update_extent_from_items()
|
|
554
568
|
|
|
555
569
|
# Check if we should overwrite bbox
|
|
@@ -586,7 +600,7 @@ def add_visualization_info(
|
|
|
586
600
|
collection_config: dict,
|
|
587
601
|
endpoint_config: dict,
|
|
588
602
|
file_url: str | None = None,
|
|
589
|
-
|
|
603
|
+
datetimes: list[datetime] | None = None,
|
|
590
604
|
) -> None:
|
|
591
605
|
extra_fields: dict[str, list[str] | dict[str, str]] = {}
|
|
592
606
|
if "Attribution" in endpoint_config:
|
|
@@ -606,25 +620,36 @@ def add_visualization_info(
|
|
|
606
620
|
"role": ["data"],
|
|
607
621
|
}
|
|
608
622
|
)
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
623
|
+
dimensions = {}
|
|
624
|
+
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
625
|
+
for key, value in dimensions_config.items():
|
|
626
|
+
dimensions[key] = value
|
|
627
|
+
if datetimes is not None:
|
|
628
|
+
dt = datetimes[0]
|
|
629
|
+
start_isostring = format_datetime_to_isostring_zulu(dt)
|
|
630
|
+
# SH WMS for public collections needs time interval, we use full day here
|
|
631
|
+
end = dt + timedelta(days=1) - timedelta(milliseconds=1)
|
|
632
|
+
# we have start_datetime and end_datetime
|
|
633
|
+
if len(datetimes) == 2:
|
|
634
|
+
end = datetimes[1]
|
|
635
|
+
end_isostring = format_datetime_to_isostring_zulu(end)
|
|
636
|
+
time_interval = f"{start_isostring}/{end_isostring}"
|
|
637
|
+
dimensions["TIME"] = time_interval
|
|
638
|
+
|
|
639
|
+
if dimensions != {}:
|
|
640
|
+
extra_fields["wms:dimensions"] = dimensions
|
|
641
|
+
link = Link(
|
|
642
|
+
rel="wms",
|
|
643
|
+
target=f"https://services.sentinel-hub.com/ogc/wms/{instanceId}",
|
|
644
|
+
media_type=(endpoint_config.get("MimeType", "image/png")),
|
|
645
|
+
title=collection_config["Name"],
|
|
646
|
+
extra_fields=extra_fields,
|
|
627
647
|
)
|
|
648
|
+
add_projection_info(
|
|
649
|
+
endpoint_config,
|
|
650
|
+
link,
|
|
651
|
+
)
|
|
652
|
+
stac_object.add_link(link)
|
|
628
653
|
elif endpoint_config["Name"] == "WMS":
|
|
629
654
|
extra_fields.update(
|
|
630
655
|
{
|
|
@@ -632,30 +657,44 @@ def add_visualization_info(
|
|
|
632
657
|
"role": ["data"],
|
|
633
658
|
}
|
|
634
659
|
)
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
660
|
+
dimensions = {}
|
|
661
|
+
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
662
|
+
for key, value in dimensions_config.items():
|
|
663
|
+
dimensions[key] = value
|
|
664
|
+
if datetimes is not None:
|
|
665
|
+
dimensions["TIME"] = format_datetime_to_isostring_zulu(datetimes[0])
|
|
666
|
+
if dimensions != {}:
|
|
667
|
+
extra_fields["wms:dimensions"] = dimensions
|
|
639
668
|
if "Styles" in endpoint_config:
|
|
640
669
|
extra_fields["wms:styles"] = endpoint_config["Styles"]
|
|
641
670
|
media_type = endpoint_config.get("MediaType", "image/jpeg")
|
|
642
671
|
endpoint_url = endpoint_config["EndPoint"]
|
|
643
672
|
# custom replacing of all ENV VARS present as template in URL as {VAR}
|
|
644
673
|
endpoint_url = replace_with_env_variables(endpoint_url)
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
674
|
+
link = Link(
|
|
675
|
+
rel="wms",
|
|
676
|
+
target=endpoint_url,
|
|
677
|
+
media_type=media_type,
|
|
678
|
+
title=collection_config["Name"],
|
|
679
|
+
extra_fields=extra_fields,
|
|
680
|
+
)
|
|
681
|
+
add_projection_info(
|
|
682
|
+
endpoint_config,
|
|
683
|
+
link,
|
|
653
684
|
)
|
|
685
|
+
stac_object.add_link(link)
|
|
654
686
|
elif endpoint_config["Name"] == "JAXA_WMTS_PALSAR":
|
|
655
687
|
target_url = "{}".format(endpoint_config.get("EndPoint"))
|
|
656
688
|
# custom time just for this special case as a default for collection wmts
|
|
689
|
+
time = None
|
|
690
|
+
if datetimes is not None:
|
|
691
|
+
time = datetimes[0]
|
|
657
692
|
extra_fields.update(
|
|
658
|
-
{
|
|
693
|
+
{
|
|
694
|
+
"wmts:layer": endpoint_config.get("LayerId", "").replace(
|
|
695
|
+
"{time}", (time and str(time.year)) or "2017"
|
|
696
|
+
)
|
|
697
|
+
}
|
|
659
698
|
)
|
|
660
699
|
stac_object.add_link(
|
|
661
700
|
Link(
|
|
@@ -709,8 +748,8 @@ def add_visualization_info(
|
|
|
709
748
|
}
|
|
710
749
|
)
|
|
711
750
|
dimensions = {}
|
|
712
|
-
if
|
|
713
|
-
dimensions["time"] =
|
|
751
|
+
if datetimes is not None:
|
|
752
|
+
dimensions["time"] = format_datetime_to_isostring_zulu(datetimes[0])
|
|
714
753
|
if dimensions_config := endpoint_config.get("Dimensions", {}):
|
|
715
754
|
for key, value in dimensions_config.items():
|
|
716
755
|
dimensions[key] = value
|
|
@@ -731,15 +770,18 @@ def add_visualization_info(
|
|
|
731
770
|
elif endpoint_config["Type"] == "tiles":
|
|
732
771
|
target_url = generate_veda_tiles_link(endpoint_config, file_url)
|
|
733
772
|
if target_url:
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
773
|
+
link = Link(
|
|
774
|
+
rel="xyz",
|
|
775
|
+
target=target_url,
|
|
776
|
+
media_type="image/png",
|
|
777
|
+
title=collection_config["Name"],
|
|
778
|
+
extra_fields=extra_fields,
|
|
779
|
+
)
|
|
780
|
+
add_projection_info(
|
|
781
|
+
endpoint_config,
|
|
782
|
+
link,
|
|
742
783
|
)
|
|
784
|
+
stac_object.add_link(link)
|
|
743
785
|
elif endpoint_config["Name"] == "GeoDB Vector Tiles":
|
|
744
786
|
# `${geoserverUrl}${config.layerName}@EPSG%3A${projString}@pbf/{z}/{x}/{-y}.pbf`,
|
|
745
787
|
# 'geodb_debd884d-92f9-4979-87b6-eadef1139394:GTIF_AT_Gemeinden_3857'
|
|
@@ -833,12 +875,13 @@ def handle_raw_source(
|
|
|
833
875
|
add_projection_info(endpoint_config, asset)
|
|
834
876
|
assets[a["Identifier"]] = asset
|
|
835
877
|
bbox = endpoint_config.get("Bbox", [-180, -85, 180, 85])
|
|
878
|
+
dt = parse_datestring_to_tz_aware_datetime(time_entry["Time"])
|
|
836
879
|
item = Item(
|
|
837
|
-
id=
|
|
880
|
+
id=format_datetime_to_isostring_zulu(dt),
|
|
838
881
|
bbox=bbox,
|
|
839
882
|
properties={},
|
|
840
883
|
geometry=create_geojson_from_bbox(bbox),
|
|
841
|
-
datetime=
|
|
884
|
+
datetime=dt,
|
|
842
885
|
assets=assets,
|
|
843
886
|
extra_fields={},
|
|
844
887
|
)
|
|
@@ -859,7 +902,7 @@ def handle_raw_source(
|
|
|
859
902
|
)
|
|
860
903
|
item.add_link(style_link)
|
|
861
904
|
link = collection.add_item(item)
|
|
862
|
-
link.extra_fields["datetime"] =
|
|
905
|
+
link.extra_fields["datetime"] = format_datetime_to_isostring_zulu(dt)
|
|
863
906
|
link.extra_fields["assets"] = [a["File"] for a in time_entry["Assets"]]
|
|
864
907
|
# eodash v4 compatibility, adding last referenced style to collection
|
|
865
908
|
if style_link:
|
|
@@ -3,7 +3,6 @@ from datetime import datetime
|
|
|
3
3
|
import requests
|
|
4
4
|
import spdx_lookup as lookup
|
|
5
5
|
import yaml
|
|
6
|
-
from dateutil import parser
|
|
7
6
|
from pystac import (
|
|
8
7
|
Asset,
|
|
9
8
|
Catalog,
|
|
@@ -18,7 +17,10 @@ from pystac import (
|
|
|
18
17
|
from structlog import get_logger
|
|
19
18
|
from yaml.loader import SafeLoader
|
|
20
19
|
|
|
21
|
-
from eodash_catalog.utils import
|
|
20
|
+
from eodash_catalog.utils import (
|
|
21
|
+
generateDatetimesFromInterval,
|
|
22
|
+
parse_datestring_to_tz_aware_datetime,
|
|
23
|
+
)
|
|
22
24
|
|
|
23
25
|
LOGGER = get_logger(__name__)
|
|
24
26
|
|
|
@@ -42,20 +44,12 @@ def get_or_create_collection(
|
|
|
42
44
|
spatial_extent,
|
|
43
45
|
]
|
|
44
46
|
)
|
|
45
|
-
times: list[str] = []
|
|
46
47
|
temporal_extent = TemporalExtent([[datetime.now(), None]])
|
|
47
|
-
if endpoint_config
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
51
|
-
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
52
|
-
elif endpoint_config.get("DateTimeInterval"):
|
|
53
|
-
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00")
|
|
54
|
-
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00")
|
|
55
|
-
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
56
|
-
times = generateDateIsostringsFromInterval(start, end, timedelta_config)
|
|
57
|
-
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
48
|
+
if endpoint_config:
|
|
49
|
+
times_datetimes = get_collection_datetimes_from_config(endpoint_config)
|
|
50
|
+
if len(times_datetimes) > 0:
|
|
58
51
|
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
52
|
+
|
|
59
53
|
extent = Extent(spatial=spatial_extent, temporal=temporal_extent)
|
|
60
54
|
|
|
61
55
|
# Check if description is link to markdown file
|
|
@@ -387,17 +381,20 @@ def add_extra_fields(stac_object: Collection | Link, collection_config: dict) ->
|
|
|
387
381
|
stac_object.extra_fields["eodash:mapProjection"] = collection_config["MapProjection"]
|
|
388
382
|
|
|
389
383
|
|
|
390
|
-
def
|
|
391
|
-
|
|
392
|
-
if endpoint_config
|
|
384
|
+
def get_collection_datetimes_from_config(endpoint_config: dict) -> list[datetime]:
|
|
385
|
+
times_datetimes: list[datetime] = []
|
|
386
|
+
if endpoint_config:
|
|
393
387
|
if endpoint_config.get("Times"):
|
|
394
388
|
times = list(endpoint_config.get("Times", []))
|
|
389
|
+
times_datetimes = sorted(
|
|
390
|
+
[parse_datestring_to_tz_aware_datetime(time) for time in times]
|
|
391
|
+
)
|
|
395
392
|
elif endpoint_config.get("DateTimeInterval"):
|
|
396
|
-
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:
|
|
397
|
-
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:
|
|
393
|
+
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00Z")
|
|
394
|
+
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00Z")
|
|
398
395
|
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
399
|
-
|
|
400
|
-
return
|
|
396
|
+
times_datetimes = generateDatetimesFromInterval(start, end, timedelta_config)
|
|
397
|
+
return times_datetimes
|
|
401
398
|
|
|
402
399
|
|
|
403
400
|
def add_projection_info(
|
|
@@ -7,7 +7,7 @@ from pystac import (
|
|
|
7
7
|
Item,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
|
-
from eodash_catalog.utils import generate_veda_cog_link
|
|
10
|
+
from eodash_catalog.utils import format_datetime_to_isostring_zulu, generate_veda_cog_link
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def fetch_and_save_thumbnail(collection_config: dict, url: str) -> None:
|
|
@@ -45,7 +45,7 @@ def generate_thumbnail(
|
|
|
45
45
|
# it is possible for datetime to be null,
|
|
46
46
|
# if it is start and end datetime have to exist
|
|
47
47
|
if item_datetime:
|
|
48
|
-
time = item_datetime
|
|
48
|
+
time = format_datetime_to_isostring_zulu(item_datetime)
|
|
49
49
|
url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format(
|
|
50
50
|
instanceId,
|
|
51
51
|
wms_config,
|
|
@@ -14,6 +14,7 @@ from dateutil import parser
|
|
|
14
14
|
from owslib.wms import WebMapService
|
|
15
15
|
from owslib.wmts import WebMapTileService
|
|
16
16
|
from pystac import Catalog, Collection, Item, RelType
|
|
17
|
+
from pytz import timezone as pytztimezone
|
|
17
18
|
from six import string_types
|
|
18
19
|
from structlog import get_logger
|
|
19
20
|
|
|
@@ -57,7 +58,7 @@ def create_geojson_from_bbox(bbox: list[float | int]) -> dict:
|
|
|
57
58
|
|
|
58
59
|
def retrieveExtentFromWMSWMTS(
|
|
59
60
|
capabilities_url: str, layer: str, version: str = "1.1.1", wmts: bool = False
|
|
60
|
-
) -> tuple[list[float], list[
|
|
61
|
+
) -> tuple[list[float], list[datetime]]:
|
|
61
62
|
times = []
|
|
62
63
|
try:
|
|
63
64
|
if not wmts:
|
|
@@ -96,7 +97,9 @@ def retrieveExtentFromWMSWMTS(
|
|
|
96
97
|
bbox = [-180.0, -90.0, 180.0, 90.0]
|
|
97
98
|
if service and service[layer].boundingBoxWGS84:
|
|
98
99
|
bbox = [float(x) for x in service[layer].boundingBoxWGS84]
|
|
99
|
-
|
|
100
|
+
|
|
101
|
+
datetimes = [parse_datestring_to_tz_aware_datetime(time_str) for time_str in times]
|
|
102
|
+
return bbox, datetimes
|
|
100
103
|
|
|
101
104
|
|
|
102
105
|
def interval(start: datetime, stop: datetime, delta: timedelta) -> Iterator[datetime]:
|
|
@@ -151,19 +154,20 @@ def parse_duration(datestring):
|
|
|
151
154
|
return ret
|
|
152
155
|
|
|
153
156
|
|
|
154
|
-
def
|
|
157
|
+
def generateDatetimesFromInterval(
|
|
155
158
|
start: str, end: str, timedelta_config: dict | None = None
|
|
156
|
-
) -> list[
|
|
159
|
+
) -> list[datetime]:
|
|
157
160
|
if timedelta_config is None:
|
|
158
161
|
timedelta_config = {}
|
|
159
|
-
start_dt =
|
|
162
|
+
start_dt = parse_datestring_to_tz_aware_datetime(start)
|
|
160
163
|
if end == "today":
|
|
161
|
-
|
|
162
|
-
|
|
164
|
+
end_dt = datetime.now(tz=timezone.utc)
|
|
165
|
+
else:
|
|
166
|
+
end_dt = parse_datestring_to_tz_aware_datetime(end)
|
|
163
167
|
delta = timedelta(**timedelta_config)
|
|
164
168
|
dates = []
|
|
165
169
|
while start_dt <= end_dt:
|
|
166
|
-
dates.append(start_dt
|
|
170
|
+
dates.append(start_dt)
|
|
167
171
|
start_dt += delta
|
|
168
172
|
return dates
|
|
169
173
|
|
|
@@ -251,9 +255,9 @@ def add_single_item_if_collection_empty(collection: Collection) -> None:
|
|
|
251
255
|
bbox=[-180, -85, 180, 85],
|
|
252
256
|
properties={},
|
|
253
257
|
geometry=None,
|
|
254
|
-
datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
255
|
-
start_datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
256
|
-
end_datetime=datetime.now(),
|
|
258
|
+
datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
259
|
+
start_datetime=datetime(1970, 1, 1, 0, 0, 0, tzinfo=pytztimezone("UTC")),
|
|
260
|
+
end_datetime=datetime.now(tz=pytztimezone("UTC")),
|
|
257
261
|
)
|
|
258
262
|
collection.add_item(item)
|
|
259
263
|
|
|
@@ -309,19 +313,28 @@ def retry(exceptions, tries=3, delay=2, backoff=1, logger=None):
|
|
|
309
313
|
return decorator
|
|
310
314
|
|
|
311
315
|
|
|
312
|
-
def filter_time_entries(time_entries: list[
|
|
316
|
+
def filter_time_entries(time_entries: list[datetime], query: dict[str, str]) -> list[datetime]:
|
|
313
317
|
datetime_query = [
|
|
314
|
-
|
|
315
|
-
|
|
318
|
+
time_entries[0],
|
|
319
|
+
time_entries[-1],
|
|
316
320
|
]
|
|
317
321
|
if start := query.get("Start"):
|
|
318
|
-
datetime_query[0] =
|
|
322
|
+
datetime_query[0] = parse_datestring_to_tz_aware_datetime(start)
|
|
319
323
|
if end := query.get("End"):
|
|
320
|
-
datetime_query[1] =
|
|
324
|
+
datetime_query[1] = parse_datestring_to_tz_aware_datetime(end)
|
|
321
325
|
# filter times based on query Start/End
|
|
322
|
-
time_entries = [
|
|
323
|
-
datetime_str
|
|
324
|
-
for datetime_str in time_entries
|
|
325
|
-
if datetime_query[0] <= parser.isoparse(datetime_str) < datetime_query[1]
|
|
326
|
-
]
|
|
326
|
+
time_entries = [dt for dt in time_entries if datetime_query[0] <= dt < datetime_query[1]]
|
|
327
327
|
return time_entries
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def parse_datestring_to_tz_aware_datetime(datestring: str) -> datetime:
|
|
331
|
+
dt = parser.isoparse(datestring)
|
|
332
|
+
dt = pytztimezone("UTC").localize(dt) if dt.tzinfo is None else dt
|
|
333
|
+
return dt
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def format_datetime_to_isostring_zulu(datetime_obj: datetime) -> str:
|
|
337
|
+
# although "+00:00" is a valid ISO 8601 timezone designation for UTC,
|
|
338
|
+
# we rather convert it to Zulu based string in order for various clients
|
|
339
|
+
# to understand it better (WMS)
|
|
340
|
+
return (datetime_obj.replace(microsecond=0).isoformat()).replace("+00:00", "Z")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_CROPOMAT1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_see_solar_energy.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_1.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_tif_demo_2.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-collections/test_wms_no_time.yaml
RENAMED
|
File without changes
|
{eodash_catalog-0.0.29 → eodash_catalog-0.0.31}/tests/testing-indicators/test_indicator.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|