eodash_catalog 0.0.10__py3-none-any.whl → 0.0.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of eodash_catalog might be problematic. Click here for more details.
- eodash_catalog/__about__.py +1 -1
- eodash_catalog/endpoints.py +295 -198
- eodash_catalog/generate_indicators.py +116 -100
- eodash_catalog/sh_endpoint.py +4 -2
- eodash_catalog/stac_handling.py +132 -101
- eodash_catalog/thumbnails.py +27 -15
- eodash_catalog/utils.py +53 -16
- {eodash_catalog-0.0.10.dist-info → eodash_catalog-0.0.12.dist-info}/METADATA +1 -1
- eodash_catalog-0.0.12.dist-info/RECORD +14 -0
- eodash_catalog-0.0.10.dist-info/RECORD +0 -14
- {eodash_catalog-0.0.10.dist-info → eodash_catalog-0.0.12.dist-info}/WHEEL +0 -0
- {eodash_catalog-0.0.10.dist-info → eodash_catalog-0.0.12.dist-info}/entry_points.txt +0 -0
- {eodash_catalog-0.0.10.dist-info → eodash_catalog-0.0.12.dist-info}/licenses/LICENSE.txt +0 -0
eodash_catalog/stac_handling.py
CHANGED
|
@@ -6,6 +6,7 @@ import yaml
|
|
|
6
6
|
from dateutil import parser
|
|
7
7
|
from pystac import (
|
|
8
8
|
Asset,
|
|
9
|
+
Catalog,
|
|
9
10
|
Collection,
|
|
10
11
|
Extent,
|
|
11
12
|
Link,
|
|
@@ -18,69 +19,75 @@ from yaml.loader import SafeLoader
|
|
|
18
19
|
from eodash_catalog.utils import generateDateIsostringsFromInterval
|
|
19
20
|
|
|
20
21
|
|
|
21
|
-
def
|
|
22
|
+
def get_or_create_collection(
|
|
23
|
+
catalog: Catalog,
|
|
24
|
+
collection_id: str,
|
|
25
|
+
collection_config: dict,
|
|
26
|
+
catalog_config: dict,
|
|
27
|
+
endpoint_config: dict,
|
|
28
|
+
) -> Collection:
|
|
22
29
|
# Check if collection already in catalog
|
|
23
30
|
for collection in catalog.get_collections():
|
|
24
31
|
if collection.id == collection_id:
|
|
25
|
-
return collection
|
|
32
|
+
return collection
|
|
26
33
|
# If none found create a new one
|
|
27
|
-
spatial_extent =
|
|
34
|
+
spatial_extent = endpoint_config.get("OverwriteBBox", [-180.0, -90.0, 180.0, 90.0])
|
|
28
35
|
|
|
29
36
|
spatial_extent = SpatialExtent(
|
|
30
37
|
[
|
|
31
38
|
spatial_extent,
|
|
32
39
|
]
|
|
33
40
|
)
|
|
34
|
-
times = []
|
|
41
|
+
times: list[str] = []
|
|
35
42
|
temporal_extent = TemporalExtent([[datetime.now(), None]])
|
|
36
|
-
if
|
|
37
|
-
if
|
|
38
|
-
times =
|
|
43
|
+
if endpoint_config and endpoint_config.get("Type") == "OverwriteTimes":
|
|
44
|
+
if endpoint_config.get("Times"):
|
|
45
|
+
times = list(endpoint_config.get("Times", []))
|
|
39
46
|
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
40
47
|
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
41
|
-
elif
|
|
42
|
-
start =
|
|
43
|
-
end =
|
|
44
|
-
timedelta_config =
|
|
48
|
+
elif endpoint_config.get("DateTimeInterval"):
|
|
49
|
+
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00")
|
|
50
|
+
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00")
|
|
51
|
+
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
45
52
|
times = generateDateIsostringsFromInterval(start, end, timedelta_config)
|
|
46
53
|
times_datetimes = sorted([parser.isoparse(time) for time in times])
|
|
47
54
|
temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]])
|
|
48
55
|
extent = Extent(spatial=spatial_extent, temporal=temporal_extent)
|
|
49
56
|
|
|
50
57
|
# Check if description is link to markdown file
|
|
51
|
-
if "Description" in
|
|
52
|
-
description =
|
|
58
|
+
if "Description" in collection_config:
|
|
59
|
+
description = collection_config["Description"]
|
|
53
60
|
if description.endswith((".md", ".MD")):
|
|
54
61
|
if description.startswith("http"):
|
|
55
62
|
# if full absolute path is defined
|
|
56
63
|
response = requests.get(description)
|
|
57
64
|
if response.status_code == 200:
|
|
58
65
|
description = response.text
|
|
59
|
-
elif "Subtitle" in
|
|
66
|
+
elif "Subtitle" in collection_config:
|
|
60
67
|
print("WARNING: Markdown file could not be fetched")
|
|
61
|
-
description =
|
|
68
|
+
description = collection_config["Subtitle"]
|
|
62
69
|
else:
|
|
63
70
|
# relative path to assets was given
|
|
64
|
-
response = requests.get(f"{
|
|
71
|
+
response = requests.get(f"{catalog_config["assets_endpoint"]}/{description}")
|
|
65
72
|
if response.status_code == 200:
|
|
66
73
|
description = response.text
|
|
67
|
-
elif "Subtitle" in
|
|
74
|
+
elif "Subtitle" in collection_config:
|
|
68
75
|
print("WARNING: Markdown file could not be fetched")
|
|
69
|
-
description =
|
|
70
|
-
elif "Subtitle" in
|
|
76
|
+
description = collection_config["Subtitle"]
|
|
77
|
+
elif "Subtitle" in collection_config:
|
|
71
78
|
# Try to use at least subtitle to fill some information
|
|
72
|
-
description =
|
|
79
|
+
description = collection_config["Subtitle"]
|
|
73
80
|
|
|
74
81
|
collection = Collection(
|
|
75
82
|
id=collection_id,
|
|
76
|
-
title=
|
|
83
|
+
title=collection_config["Title"],
|
|
77
84
|
description=description,
|
|
78
85
|
extent=extent,
|
|
79
86
|
)
|
|
80
|
-
return
|
|
87
|
+
return collection
|
|
81
88
|
|
|
82
89
|
|
|
83
|
-
def create_web_map_link(layer, role):
|
|
90
|
+
def create_web_map_link(layer: dict, role: str) -> Link:
|
|
84
91
|
extra_fields = {
|
|
85
92
|
"roles": [role],
|
|
86
93
|
"id": layer["id"],
|
|
@@ -113,15 +120,20 @@ def create_web_map_link(layer, role):
|
|
|
113
120
|
return wml
|
|
114
121
|
|
|
115
122
|
|
|
116
|
-
def add_example_info(
|
|
117
|
-
|
|
118
|
-
|
|
123
|
+
def add_example_info(
|
|
124
|
+
stac_object: Collection | Catalog,
|
|
125
|
+
collection_config: dict,
|
|
126
|
+
endpoint_config: dict,
|
|
127
|
+
catalog_config: dict,
|
|
128
|
+
) -> None:
|
|
129
|
+
if "Services" in collection_config:
|
|
130
|
+
for service in collection_config["Services"]:
|
|
119
131
|
if service["Name"] == "Statistical API":
|
|
120
132
|
service_type = service.get("Type", "byoc")
|
|
121
133
|
stac_object.add_link(
|
|
122
134
|
Link(
|
|
123
135
|
rel="example",
|
|
124
|
-
target="{}/{}".format(
|
|
136
|
+
target="{}/{}".format(catalog_config["assets_endpoint"], service["Script"]),
|
|
125
137
|
title="evalscript",
|
|
126
138
|
media_type="application/javascript",
|
|
127
139
|
extra_fields={
|
|
@@ -156,13 +168,13 @@ def add_example_info(stac_object, data, endpoint, config):
|
|
|
156
168
|
},
|
|
157
169
|
)
|
|
158
170
|
)
|
|
159
|
-
elif "Resources" in
|
|
160
|
-
for service in
|
|
171
|
+
elif "Resources" in collection_config:
|
|
172
|
+
for service in collection_config["Resources"]:
|
|
161
173
|
if service.get("Name") == "xcube":
|
|
162
174
|
target_url = "{}/timeseries/{}/{}?aggMethods=median".format(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
175
|
+
endpoint_config["EndPoint"],
|
|
176
|
+
endpoint_config["DatacubeId"],
|
|
177
|
+
endpoint_config["Variable"],
|
|
166
178
|
)
|
|
167
179
|
stac_object.add_link(
|
|
168
180
|
Link(
|
|
@@ -178,25 +190,27 @@ def add_example_info(stac_object, data, endpoint, config):
|
|
|
178
190
|
)
|
|
179
191
|
|
|
180
192
|
|
|
181
|
-
def add_collection_information(
|
|
193
|
+
def add_collection_information(
|
|
194
|
+
catalog_config: dict, collection: Collection, collection_config: dict
|
|
195
|
+
) -> None:
|
|
182
196
|
# Add metadata information
|
|
183
197
|
# Check license identifier
|
|
184
|
-
if "License" in
|
|
198
|
+
if "License" in collection_config:
|
|
185
199
|
# Check if list was provided
|
|
186
|
-
if isinstance(
|
|
187
|
-
if len(
|
|
200
|
+
if isinstance(collection_config["License"], list):
|
|
201
|
+
if len(collection_config["License"]) == 1:
|
|
188
202
|
collection.license = "proprietary"
|
|
189
203
|
link = Link(
|
|
190
204
|
rel="license",
|
|
191
|
-
target=
|
|
192
|
-
media_type=(
|
|
205
|
+
target=collection_config["License"][0]["Url"],
|
|
206
|
+
media_type=(collection_config["License"][0].get("Type", "text/html")),
|
|
193
207
|
)
|
|
194
|
-
if "Title" in
|
|
195
|
-
link.title =
|
|
208
|
+
if "Title" in collection_config["License"][0]:
|
|
209
|
+
link.title = collection_config["License"][0]["Title"]
|
|
196
210
|
collection.links.append(link)
|
|
197
|
-
elif len(
|
|
211
|
+
elif len(collection_config["License"]) > 1:
|
|
198
212
|
collection.license = "various"
|
|
199
|
-
for license_entry in
|
|
213
|
+
for license_entry in collection_config["License"]:
|
|
200
214
|
link = Link(
|
|
201
215
|
rel="license",
|
|
202
216
|
target=license_entry["Url"],
|
|
@@ -208,7 +222,7 @@ def add_collection_information(config, collection, data):
|
|
|
208
222
|
link.title = license_entry["Title"]
|
|
209
223
|
collection.links.append(link)
|
|
210
224
|
else:
|
|
211
|
-
license_data = lookup.by_id(
|
|
225
|
+
license_data = lookup.by_id(collection_config["License"])
|
|
212
226
|
if license_data is not None:
|
|
213
227
|
collection.license = license_data.id
|
|
214
228
|
if license_data.sources:
|
|
@@ -229,65 +243,65 @@ def add_collection_information(config, collection, data):
|
|
|
229
243
|
# print("WARNING: No license was provided, falling back to proprietary")
|
|
230
244
|
pass
|
|
231
245
|
|
|
232
|
-
if "Provider" in
|
|
246
|
+
if "Provider" in collection_config:
|
|
233
247
|
try:
|
|
234
248
|
collection.providers = [
|
|
235
249
|
Provider(
|
|
236
250
|
# convert information to lower case
|
|
237
251
|
**{k.lower(): v for k, v in provider.items()}
|
|
238
252
|
)
|
|
239
|
-
for provider in
|
|
253
|
+
for provider in collection_config["Provider"]
|
|
240
254
|
]
|
|
241
255
|
except Exception:
|
|
242
256
|
print(f"WARNING: Issue creating provider information for collection: {collection.id}")
|
|
243
257
|
|
|
244
|
-
if "Citation" in
|
|
245
|
-
if "DOI" in
|
|
246
|
-
collection.extra_fields["sci:doi"] =
|
|
247
|
-
if "Citation" in
|
|
248
|
-
collection.extra_fields["sci:citation"] =
|
|
249
|
-
if "Publication" in
|
|
258
|
+
if "Citation" in collection_config:
|
|
259
|
+
if "DOI" in collection_config["Citation"]:
|
|
260
|
+
collection.extra_fields["sci:doi"] = collection_config["Citation"]["DOI"]
|
|
261
|
+
if "Citation" in collection_config["Citation"]:
|
|
262
|
+
collection.extra_fields["sci:citation"] = collection_config["Citation"]["Citation"]
|
|
263
|
+
if "Publication" in collection_config["Citation"]:
|
|
250
264
|
collection.extra_fields["sci:publications"] = [
|
|
251
265
|
# convert keys to lower case
|
|
252
266
|
{k.lower(): v for k, v in publication.items()}
|
|
253
|
-
for publication in
|
|
267
|
+
for publication in collection_config["Citation"]["Publication"]
|
|
254
268
|
]
|
|
255
269
|
|
|
256
|
-
if "Subtitle" in
|
|
257
|
-
collection.extra_fields["subtitle"] =
|
|
258
|
-
if "Legend" in
|
|
270
|
+
if "Subtitle" in collection_config:
|
|
271
|
+
collection.extra_fields["subtitle"] = collection_config["Subtitle"]
|
|
272
|
+
if "Legend" in collection_config:
|
|
259
273
|
collection.add_asset(
|
|
260
274
|
"legend",
|
|
261
275
|
Asset(
|
|
262
|
-
href=f"{
|
|
276
|
+
href=f"{catalog_config["assets_endpoint"]}/{collection_config["Legend"]}",
|
|
263
277
|
media_type="image/png",
|
|
264
278
|
roles=["metadata"],
|
|
265
279
|
),
|
|
266
280
|
)
|
|
267
|
-
if "Story" in
|
|
281
|
+
if "Story" in collection_config:
|
|
268
282
|
collection.add_asset(
|
|
269
283
|
"story",
|
|
270
284
|
Asset(
|
|
271
|
-
href=f"{
|
|
285
|
+
href=f"{catalog_config["assets_endpoint"]}/{collection_config["Story"]}",
|
|
272
286
|
media_type="text/markdown",
|
|
273
287
|
roles=["metadata"],
|
|
274
288
|
),
|
|
275
289
|
)
|
|
276
|
-
if "Image" in
|
|
290
|
+
if "Image" in collection_config:
|
|
277
291
|
collection.add_asset(
|
|
278
292
|
"thumbnail",
|
|
279
293
|
Asset(
|
|
280
|
-
href=f"{
|
|
294
|
+
href=f"{catalog_config["assets_endpoint"]}/{collection_config["Image"]}",
|
|
281
295
|
media_type="image/png",
|
|
282
296
|
roles=["thumbnail"],
|
|
283
297
|
),
|
|
284
298
|
)
|
|
285
299
|
# Add extra fields to collection if available
|
|
286
|
-
add_extra_fields(collection,
|
|
300
|
+
add_extra_fields(collection, collection_config)
|
|
287
301
|
|
|
288
|
-
if "References" in
|
|
302
|
+
if "References" in collection_config:
|
|
289
303
|
generic_counter = 1
|
|
290
|
-
for ref in
|
|
304
|
+
for ref in collection_config["References"]:
|
|
291
305
|
if "Key" in ref:
|
|
292
306
|
key = ref["Key"]
|
|
293
307
|
else:
|
|
@@ -304,56 +318,73 @@ def add_collection_information(config, collection, data):
|
|
|
304
318
|
)
|
|
305
319
|
|
|
306
320
|
|
|
307
|
-
def add_base_overlay_info(
|
|
321
|
+
def add_base_overlay_info(
|
|
322
|
+
collection: Collection, catalog_config: dict, collection_config: dict
|
|
323
|
+
) -> None:
|
|
308
324
|
# check if default base layers defined
|
|
309
|
-
if "default_base_layers" in
|
|
310
|
-
with open(f"{
|
|
325
|
+
if "default_base_layers" in catalog_config:
|
|
326
|
+
with open(f"{catalog_config["default_base_layers"]}.yaml") as f:
|
|
311
327
|
base_layers = yaml.load(f, Loader=SafeLoader)
|
|
312
328
|
for layer in base_layers:
|
|
313
329
|
collection.add_link(create_web_map_link(layer, role="baselayer"))
|
|
314
330
|
# check if default overlay layers defined
|
|
315
|
-
if "default_overlay_layers" in
|
|
316
|
-
with open("{}.yaml".format(
|
|
331
|
+
if "default_overlay_layers" in catalog_config:
|
|
332
|
+
with open("{}.yaml".format(catalog_config["default_overlay_layers"])) as f:
|
|
317
333
|
overlay_layers = yaml.load(f, Loader=SafeLoader)
|
|
318
334
|
for layer in overlay_layers:
|
|
319
335
|
collection.add_link(create_web_map_link(layer, role="overlay"))
|
|
320
|
-
if "BaseLayers" in
|
|
321
|
-
for layer in
|
|
336
|
+
if "BaseLayers" in collection_config:
|
|
337
|
+
for layer in collection_config["BaseLayers"]:
|
|
322
338
|
collection.add_link(create_web_map_link(layer, role="baselayer"))
|
|
323
|
-
if "OverlayLayers" in
|
|
324
|
-
for layer in
|
|
339
|
+
if "OverlayLayers" in collection_config:
|
|
340
|
+
for layer in collection_config["OverlayLayers"]:
|
|
325
341
|
collection.add_link(create_web_map_link(layer, role="overlay"))
|
|
326
342
|
# TODO: possibility to overwrite default base and overlay layers
|
|
327
343
|
|
|
328
344
|
|
|
329
|
-
def add_extra_fields(stac_object,
|
|
330
|
-
if "yAxis" in
|
|
331
|
-
stac_object.extra_fields["yAxis"] =
|
|
332
|
-
if "Themes" in
|
|
333
|
-
stac_object.extra_fields["themes"] =
|
|
334
|
-
if "Locations" in
|
|
345
|
+
def add_extra_fields(stac_object: Collection | Catalog | Link, collection_config: dict) -> None:
|
|
346
|
+
if "yAxis" in collection_config:
|
|
347
|
+
stac_object.extra_fields["yAxis"] = collection_config["yAxis"]
|
|
348
|
+
if "Themes" in collection_config:
|
|
349
|
+
stac_object.extra_fields["themes"] = collection_config["Themes"]
|
|
350
|
+
if "Locations" in collection_config or "Subcollections" in collection_config:
|
|
335
351
|
stac_object.extra_fields["locations"] = True
|
|
336
|
-
if "Tags" in
|
|
337
|
-
stac_object.extra_fields["tags"] =
|
|
338
|
-
if "Satellite" in
|
|
339
|
-
stac_object.extra_fields["satellite"] =
|
|
340
|
-
if "Sensor" in
|
|
341
|
-
stac_object.extra_fields["sensor"] =
|
|
342
|
-
if "Agency" in
|
|
343
|
-
stac_object.extra_fields["agency"] =
|
|
344
|
-
if "yAxis" in
|
|
345
|
-
stac_object.extra_fields["yAxis"] =
|
|
346
|
-
if "EodashIdentifier" in
|
|
347
|
-
stac_object.extra_fields["subcode"] =
|
|
348
|
-
if "DataSource" in
|
|
349
|
-
if "Spaceborne" in
|
|
350
|
-
if "Sensor" in
|
|
351
|
-
stac_object.extra_fields["sensor"] =
|
|
352
|
-
|
|
353
|
-
stac_object.extra_fields["satellite"] = data["DataSource"]["Spaceborne"][
|
|
354
|
-
"Satellite"
|
|
352
|
+
if "Tags" in collection_config:
|
|
353
|
+
stac_object.extra_fields["tags"] = collection_config["Tags"]
|
|
354
|
+
if "Satellite" in collection_config:
|
|
355
|
+
stac_object.extra_fields["satellite"] = collection_config["Satellite"]
|
|
356
|
+
if "Sensor" in collection_config:
|
|
357
|
+
stac_object.extra_fields["sensor"] = collection_config["Sensor"]
|
|
358
|
+
if "Agency" in collection_config:
|
|
359
|
+
stac_object.extra_fields["agency"] = collection_config["Agency"]
|
|
360
|
+
if "yAxis" in collection_config:
|
|
361
|
+
stac_object.extra_fields["yAxis"] = collection_config["yAxis"]
|
|
362
|
+
if "EodashIdentifier" in collection_config:
|
|
363
|
+
stac_object.extra_fields["subcode"] = collection_config["EodashIdentifier"]
|
|
364
|
+
if "DataSource" in collection_config:
|
|
365
|
+
if "Spaceborne" in collection_config["DataSource"]:
|
|
366
|
+
if "Sensor" in collection_config["DataSource"]["Spaceborne"]:
|
|
367
|
+
stac_object.extra_fields["sensor"] = collection_config["DataSource"]["Spaceborne"][
|
|
368
|
+
"Sensor"
|
|
355
369
|
]
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
370
|
+
if "Satellite" in collection_config["DataSource"]["Spaceborne"]:
|
|
371
|
+
stac_object.extra_fields["satellite"] = collection_config["DataSource"][
|
|
372
|
+
"Spaceborne"
|
|
373
|
+
]["Satellite"]
|
|
374
|
+
if "InSitu" in collection_config["DataSource"]:
|
|
375
|
+
stac_object.extra_fields["insituSources"] = collection_config["DataSource"]["InSitu"]
|
|
376
|
+
if "Other" in collection_config["DataSource"]:
|
|
377
|
+
stac_object.extra_fields["otherSources"] = collection_config["DataSource"]["Other"]
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def get_collection_times_from_config(endpoint_config: dict) -> list[str]:
|
|
381
|
+
times: list[str] = []
|
|
382
|
+
if endpoint_config and endpoint_config.get("Type") == "OverwriteTimes":
|
|
383
|
+
if endpoint_config.get("Times"):
|
|
384
|
+
times = list(endpoint_config.get("Times", []))
|
|
385
|
+
elif endpoint_config.get("DateTimeInterval"):
|
|
386
|
+
start = endpoint_config["DateTimeInterval"].get("Start", "2020-09-01T00:00:00")
|
|
387
|
+
end = endpoint_config["DateTimeInterval"].get("End", "2020-10-01T00:00:00")
|
|
388
|
+
timedelta_config = endpoint_config["DateTimeInterval"].get("Timedelta", {"days": 1})
|
|
389
|
+
times = generateDateIsostringsFromInterval(start, end, timedelta_config)
|
|
390
|
+
return times
|
eodash_catalog/thumbnails.py
CHANGED
|
@@ -3,31 +3,43 @@ import re
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
import requests
|
|
6
|
+
from pystac import (
|
|
7
|
+
Item,
|
|
8
|
+
)
|
|
6
9
|
|
|
7
10
|
from eodash_catalog.utils import generate_veda_cog_link
|
|
8
11
|
|
|
9
12
|
|
|
10
|
-
def fetch_and_save_thumbnail(
|
|
11
|
-
collection_path = "../thumbnails/{}_{}/".format(
|
|
13
|
+
def fetch_and_save_thumbnail(collection_config: dict, url: str) -> None:
|
|
14
|
+
collection_path = "../thumbnails/{}_{}/".format(
|
|
15
|
+
collection_config["EodashIdentifier"], collection_config["Name"]
|
|
16
|
+
)
|
|
12
17
|
Path(collection_path).mkdir(parents=True, exist_ok=True)
|
|
13
18
|
image_path = f"{collection_path}/thumbnail.png"
|
|
14
19
|
if not os.path.exists(image_path):
|
|
15
|
-
|
|
20
|
+
dd = requests.get(url).content
|
|
16
21
|
with open(image_path, "wb") as f:
|
|
17
|
-
f.write(
|
|
22
|
+
f.write(dd)
|
|
18
23
|
|
|
19
24
|
|
|
20
|
-
def generate_thumbnail(
|
|
21
|
-
|
|
25
|
+
def generate_thumbnail(
|
|
26
|
+
stac_object: Item,
|
|
27
|
+
collection_config: dict,
|
|
28
|
+
endpoint_config: dict,
|
|
29
|
+
file_url: str = "",
|
|
30
|
+
time: str | None = None,
|
|
31
|
+
) -> None:
|
|
32
|
+
if endpoint_config["Name"] == "Sentinel Hub" or endpoint_config["Name"] == "WMS":
|
|
22
33
|
instanceId = os.getenv("SH_INSTANCE_ID")
|
|
23
|
-
if "InstanceId" in
|
|
24
|
-
instanceId =
|
|
34
|
+
if "InstanceId" in endpoint_config:
|
|
35
|
+
instanceId = endpoint_config["InstanceId"]
|
|
25
36
|
# Build example url
|
|
26
37
|
wms_config = (
|
|
27
38
|
"REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image/png&STYLES=&TRANSPARENT=true"
|
|
28
39
|
)
|
|
29
|
-
|
|
30
|
-
|
|
40
|
+
bbox = [-180, -85, 180, 85]
|
|
41
|
+
if bbox_s := stac_object.bbox:
|
|
42
|
+
bbox = f"{bbox_s[1]},{bbox_s[0]},{bbox_s[3]},{bbox_s[2]}" # type: ignore
|
|
31
43
|
output_format = f"format=image/png&WIDTH=256&HEIGHT=128&CRS=EPSG:4326&BBOX={bbox}"
|
|
32
44
|
item_datetime = stac_object.get_datetime()
|
|
33
45
|
# it is possible for datetime to be null,
|
|
@@ -37,13 +49,13 @@ def generate_thumbnail(stac_object, data, endpoint, file_url=None, time=None, st
|
|
|
37
49
|
url = "https://services.sentinel-hub.com/ogc/wms/{}?{}&layers={}&time={}&{}".format(
|
|
38
50
|
instanceId,
|
|
39
51
|
wms_config,
|
|
40
|
-
|
|
52
|
+
endpoint_config["LayerId"],
|
|
41
53
|
time,
|
|
42
54
|
output_format,
|
|
43
55
|
)
|
|
44
|
-
fetch_and_save_thumbnail(
|
|
45
|
-
elif
|
|
46
|
-
target_url = generate_veda_cog_link(
|
|
56
|
+
fetch_and_save_thumbnail(collection_config, url)
|
|
57
|
+
elif endpoint_config["Name"] == "VEDA":
|
|
58
|
+
target_url = generate_veda_cog_link(endpoint_config, file_url)
|
|
47
59
|
# set to get 0/0/0 tile
|
|
48
60
|
url = re.sub(r"\{.\}", "0", target_url)
|
|
49
|
-
fetch_and_save_thumbnail(
|
|
61
|
+
fetch_and_save_thumbnail(collection_config, url)
|
eodash_catalog/utils.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import re
|
|
2
2
|
import threading
|
|
3
|
+
import uuid
|
|
3
4
|
from collections.abc import Iterator
|
|
5
|
+
from dataclasses import dataclass
|
|
4
6
|
from datetime import datetime, timedelta
|
|
5
7
|
from decimal import Decimal
|
|
6
8
|
from functools import reduce
|
|
@@ -9,7 +11,7 @@ from typing import Any
|
|
|
9
11
|
from dateutil import parser
|
|
10
12
|
from owslib.wms import WebMapService
|
|
11
13
|
from owslib.wmts import WebMapTileService
|
|
12
|
-
from pystac import Catalog
|
|
14
|
+
from pystac import Catalog, Collection, Item, RelType
|
|
13
15
|
from six import string_types
|
|
14
16
|
|
|
15
17
|
from eodash_catalog.duration import Duration
|
|
@@ -35,7 +37,7 @@ def create_geojson_point(lon: int | float, lat: int | float) -> dict[str, Any]:
|
|
|
35
37
|
|
|
36
38
|
def retrieveExtentFromWMSWMTS(
|
|
37
39
|
capabilities_url: str, layer: str, version: str = "1.1.1", wmts: bool = False
|
|
38
|
-
):
|
|
40
|
+
) -> tuple[list[float], list[str]]:
|
|
39
41
|
times = []
|
|
40
42
|
try:
|
|
41
43
|
if not wmts:
|
|
@@ -91,7 +93,9 @@ def parse_duration(datestring):
|
|
|
91
93
|
if not isinstance(datestring, string_types):
|
|
92
94
|
raise TypeError(f"Expecting a string {datestring}")
|
|
93
95
|
match = ISO8601_PERIOD_REGEX.match(datestring)
|
|
94
|
-
groups =
|
|
96
|
+
groups = {}
|
|
97
|
+
if match:
|
|
98
|
+
groups = match.groupdict()
|
|
95
99
|
for key, val in groups.items():
|
|
96
100
|
if key not in ("separator", "sign"):
|
|
97
101
|
if val is None:
|
|
@@ -128,7 +132,9 @@ def parse_duration(datestring):
|
|
|
128
132
|
return ret
|
|
129
133
|
|
|
130
134
|
|
|
131
|
-
def generateDateIsostringsFromInterval(
|
|
135
|
+
def generateDateIsostringsFromInterval(
|
|
136
|
+
start: str, end: str, timedelta_config: dict | None = None
|
|
137
|
+
) -> list[str]:
|
|
132
138
|
if timedelta_config is None:
|
|
133
139
|
timedelta_config = {}
|
|
134
140
|
start_dt = datetime.fromisoformat(start)
|
|
@@ -171,32 +177,63 @@ def iter_len_at_least(i, n: int) -> int:
|
|
|
171
177
|
return sum(1 for _ in zip(range(n), i, strict=False)) == n
|
|
172
178
|
|
|
173
179
|
|
|
174
|
-
def generate_veda_cog_link(
|
|
180
|
+
def generate_veda_cog_link(endpoint_config: dict, file_url: str | None) -> str:
|
|
175
181
|
bidx = ""
|
|
176
|
-
if "Bidx" in
|
|
182
|
+
if "Bidx" in endpoint_config:
|
|
177
183
|
# Check if an array was provided
|
|
178
|
-
if hasattr(
|
|
179
|
-
for band in
|
|
184
|
+
if hasattr(endpoint_config["Bidx"], "__len__"):
|
|
185
|
+
for band in endpoint_config["Bidx"]:
|
|
180
186
|
bidx = bidx + f"&bidx={band}"
|
|
181
187
|
else:
|
|
182
|
-
bidx = "&bidx={}".format(
|
|
188
|
+
bidx = "&bidx={}".format(endpoint_config["Bidx"])
|
|
183
189
|
|
|
184
190
|
colormap = ""
|
|
185
|
-
if "Colormap" in
|
|
186
|
-
colormap = "&colormap={}".format(
|
|
191
|
+
if "Colormap" in endpoint_config:
|
|
192
|
+
colormap = "&colormap={}".format(endpoint_config["Colormap"])
|
|
187
193
|
# TODO: For now we assume a already urlparsed colormap definition
|
|
188
194
|
# it could be nice to allow a json and better convert it on the fly
|
|
189
|
-
# colormap = "&colormap=%s"%(urllib.parse.quote(str(
|
|
195
|
+
# colormap = "&colormap=%s"%(urllib.parse.quote(str(endpoint_config["Colormap"])))
|
|
190
196
|
|
|
191
197
|
colormap_name = ""
|
|
192
|
-
if "ColormapName" in
|
|
193
|
-
colormap_name = "&colormap_name={}".format(
|
|
198
|
+
if "ColormapName" in endpoint_config:
|
|
199
|
+
colormap_name = "&colormap_name={}".format(endpoint_config["ColormapName"])
|
|
194
200
|
|
|
195
201
|
rescale = ""
|
|
196
|
-
if "Rescale" in
|
|
197
|
-
rescale = "&rescale={},{}".format(
|
|
202
|
+
if "Rescale" in endpoint_config:
|
|
203
|
+
rescale = "&rescale={},{}".format(
|
|
204
|
+
endpoint_config["Rescale"][0], endpoint_config["Rescale"][1]
|
|
205
|
+
)
|
|
198
206
|
|
|
199
207
|
file_url = f"url={file_url}&" if file_url else ""
|
|
200
208
|
|
|
201
209
|
target_url = f"https://staging-raster.delta-backend.com/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}?{file_url}resampling_method=nearest{bidx}{colormap}{colormap_name}{rescale}"
|
|
202
210
|
return target_url
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@dataclass
|
|
214
|
+
class Options:
|
|
215
|
+
catalogspath: str
|
|
216
|
+
collectionspath: str
|
|
217
|
+
indicatorspath: str
|
|
218
|
+
outputpath: str
|
|
219
|
+
vd: bool
|
|
220
|
+
ni: bool
|
|
221
|
+
tn: bool
|
|
222
|
+
collections: list[str]
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def add_single_item_if_collection_empty(collection: Collection) -> None:
|
|
226
|
+
for link in collection.links:
|
|
227
|
+
if link.rel == RelType.ITEM:
|
|
228
|
+
break
|
|
229
|
+
else:
|
|
230
|
+
item = Item(
|
|
231
|
+
id=str(uuid.uuid4()),
|
|
232
|
+
bbox=[-180, -85, 180, 85],
|
|
233
|
+
properties={},
|
|
234
|
+
geometry=None,
|
|
235
|
+
datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
236
|
+
start_datetime=datetime(1970, 1, 1, 0, 0, 0),
|
|
237
|
+
end_datetime=datetime.now(),
|
|
238
|
+
)
|
|
239
|
+
collection.add_item(item)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: eodash_catalog
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.12
|
|
4
4
|
Summary: This package is intended to help create a compatible STAC catalog for the eodash dashboard client. It supports configuration of multiple endpoint types for information extraction.
|
|
5
5
|
Project-URL: Documentation, https://github.com/eodash/eodash_catalog#readme
|
|
6
6
|
Project-URL: Issues, https://github.com/eodash/eodash_catalog/issues
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
eodash_catalog/__about__.py,sha256=yEv4frNTeX9AbT5lO3thIJjP9R5pEQJ0AzODE6sN9tI,138
|
|
2
|
+
eodash_catalog/__init__.py,sha256=_W_9emPYf6FUqc0P8L2SmADx6hGSd7PlQV3yRmCk5uM,115
|
|
3
|
+
eodash_catalog/duration.py,sha256=B6XOZfvNU7SuqpxuVtT1kNKODoOQJXDI6mocvA_U1ik,10816
|
|
4
|
+
eodash_catalog/endpoints.py,sha256=o0m0dMmfvZ2ybRnHW-am4g4vjoQhFMNbnQ_xI2kE_D8,30658
|
|
5
|
+
eodash_catalog/generate_indicators.py,sha256=aTh7RHhUVfDjaWNH4GYiLuzC7Z8fQEfJGfckdkjwFOs,18454
|
|
6
|
+
eodash_catalog/sh_endpoint.py,sha256=vELooJwk269v1DNnOzb32vil96vL_SRCio8UBlx10N0,618
|
|
7
|
+
eodash_catalog/stac_handling.py,sha256=uHsAR-H3Js2pDOcIw2ApTUWAuSApnLkpjf0OUBHQ7_Q,16637
|
|
8
|
+
eodash_catalog/thumbnails.py,sha256=31Wk38oNQDxfhSUbMLBpHuZFhsR8v_7luYr65XQtDf0,2213
|
|
9
|
+
eodash_catalog/utils.py,sha256=JnXrXtq3bOmECPlSn86Mz35sDTOkgptz87lrISfE1Uo,7968
|
|
10
|
+
eodash_catalog-0.0.12.dist-info/METADATA,sha256=1kYgr6SXzhIj2nE-EsoRwpw-bMJVYdz_sVLLCaZzT08,3203
|
|
11
|
+
eodash_catalog-0.0.12.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
12
|
+
eodash_catalog-0.0.12.dist-info/entry_points.txt,sha256=kuUQrDG1PtYd8kPjf5XM6H_NtQd9Ozwl0jjiGtAvZSM,87
|
|
13
|
+
eodash_catalog-0.0.12.dist-info/licenses/LICENSE.txt,sha256=oJCW5zQxnFD-J0hGz6Zh5Lkpdk1oAndmWhseTmV224E,1107
|
|
14
|
+
eodash_catalog-0.0.12.dist-info/RECORD,,
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
eodash_catalog/__about__.py,sha256=vzs3SY-zF_mgRKDxdhyQb7oaQQkNWxttzi_gL2wbP_c,138
|
|
2
|
-
eodash_catalog/__init__.py,sha256=_W_9emPYf6FUqc0P8L2SmADx6hGSd7PlQV3yRmCk5uM,115
|
|
3
|
-
eodash_catalog/duration.py,sha256=B6XOZfvNU7SuqpxuVtT1kNKODoOQJXDI6mocvA_U1ik,10816
|
|
4
|
-
eodash_catalog/endpoints.py,sha256=HrUjXNf6Eikd7uGmCnaZS31cHuZP_nYIgbGtAvSN4Q8,25942
|
|
5
|
-
eodash_catalog/generate_indicators.py,sha256=1k__jFWChnFb7We3E5_wE3_gQN56eslAm1EcFB4MBNw,17227
|
|
6
|
-
eodash_catalog/sh_endpoint.py,sha256=vHCqUnjXH4xB9T7L8UKd36TtUyqsyJLE84QBSXaONaA,582
|
|
7
|
-
eodash_catalog/stac_handling.py,sha256=6ollozeJZbB7_FKKewG1P6qRyBCtt6bPrTSQhOedc04,14366
|
|
8
|
-
eodash_catalog/thumbnails.py,sha256=yqjxJoZbTl2l2eyrRAnR13cU9fDHC0NSVT6ZnITSruw,1901
|
|
9
|
-
eodash_catalog/utils.py,sha256=-uOspvzGNUeGpr1tjeySyBVV13kQYEMsC6Njw6JvhDc,6969
|
|
10
|
-
eodash_catalog-0.0.10.dist-info/METADATA,sha256=PKpvEeXYSO24jDVWIUUrkhR7jOK72J6mz92OUJ24sRk,3203
|
|
11
|
-
eodash_catalog-0.0.10.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
|
12
|
-
eodash_catalog-0.0.10.dist-info/entry_points.txt,sha256=kuUQrDG1PtYd8kPjf5XM6H_NtQd9Ozwl0jjiGtAvZSM,87
|
|
13
|
-
eodash_catalog-0.0.10.dist-info/licenses/LICENSE.txt,sha256=oJCW5zQxnFD-J0hGz6Zh5Lkpdk1oAndmWhseTmV224E,1107
|
|
14
|
-
eodash_catalog-0.0.10.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|