ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
  2. ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
  3. ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
  4. ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
  5. ECOv003_L2T_STARS/BRDF/version.txt +1 -0
  6. ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
  7. ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
  8. ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +162 -0
  9. ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
  10. ECOv003_L2T_STARS/L2TSTARSConfig.py +188 -0
  11. ECOv003_L2T_STARS/L2T_STARS.py +489 -0
  12. ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
  13. ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
  14. ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
  15. ECOv003_L2T_STARS/Manifest.toml +2332 -0
  16. ECOv003_L2T_STARS/Project.toml +14 -0
  17. ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
  18. ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
  19. ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
  20. ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1277 -0
  21. ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
  22. ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
  23. ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
  24. ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
  25. ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
  26. ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
  27. ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
  28. ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
  29. ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
  30. ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
  31. ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
  32. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
  33. ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
  34. ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
  35. ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
  36. ECOv003_L2T_STARS/__init__.py +3 -0
  37. ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
  38. ECOv003_L2T_STARS/constants.py +38 -0
  39. ECOv003_L2T_STARS/daterange/__init__.py +1 -0
  40. ECOv003_L2T_STARS/daterange/daterange.py +35 -0
  41. ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
  42. ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
  43. ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
  44. ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
  45. ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
  46. ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
  47. ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
  48. ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
  49. ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
  50. ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
  51. ECOv003_L2T_STARS/generate_filename.py +37 -0
  52. ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
  53. ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
  54. ECOv003_L2T_STARS/generate_output_directory.py +28 -0
  55. ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
  56. ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
  57. ECOv003_L2T_STARS/load_prior.py +248 -0
  58. ECOv003_L2T_STARS/prior.py +56 -0
  59. ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
  60. ECOv003_L2T_STARS/process_STARS_product.py +507 -0
  61. ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
  62. ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
  63. ECOv003_L2T_STARS/runconfig.py +70 -0
  64. ECOv003_L2T_STARS/timer/__init__.py +1 -0
  65. ECOv003_L2T_STARS/timer/timer.py +77 -0
  66. ECOv003_L2T_STARS/version.py +8 -0
  67. ECOv003_L2T_STARS/version.txt +1 -0
  68. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/METADATA +30 -23
  69. ecov003_l2t_stars-1.1.0.dist-info/RECORD +73 -0
  70. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/WHEEL +1 -1
  71. ecov003_l2t_stars-1.1.0.dist-info/entry_points.txt +3 -0
  72. ecov003_l2t_stars-1.1.0.dist-info/top_level.txt +1 -0
  73. ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
  74. ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
  75. {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,1277 @@
1
+ import logging
2
+ import os
3
+ import warnings
4
+ from datetime import datetime, date
5
+ from os import remove
6
+ from os.path import exists, join, abspath, expanduser
7
+ import re
8
+ from pathlib import Path
9
+ import tempfile
10
+ from typing import List, Union
11
+
12
+ import earthaccess
13
+ import h5py
14
+ import numpy as np
15
+ import pandas as pd
16
+ from matplotlib.colors import LinearSegmentedColormap
17
+ from dateutil import parser
18
+ from skimage.transform import resize
19
+
20
+ import colored_logging as cl
21
+ import rasters
22
+ from rasters import Raster, RasterGrid, RasterGeometry, Point, Polygon
23
+ from modland import generate_modland_grid
24
+
25
+ from ECOv003_exit_codes import *
26
+
27
+ from ..daterange import get_date
28
+ from ..LPDAAC.LPDAACDataPool import RETRIES
29
+ from .VIIRSDataPool import VIIRSGranule
30
+ from .VIIRS_CMR_LOGIN import CMRServerUnreachable, VIIRS_CMR_login
31
+
32
+ NDVI_COLORMAP = LinearSegmentedColormap.from_list(
33
+ name="NDVI",
34
+ colors=[
35
+ "#0000ff",
36
+ "#000000",
37
+ "#745d1a",
38
+ "#e1dea2",
39
+ "#45ff01",
40
+ "#325e32"
41
+ ]
42
+ )
43
+
44
+ ALBEDO_COLORMAP = "gray"
45
+
46
+ logger = logging.getLogger(__name__)
47
+
48
+ class VIIRSUnavailableError(Exception):
49
+ pass
50
+
51
+
52
+ class VNP09GAGranule(VIIRSGranule):
53
+ CLOUD_DATASET_NAME = "HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SurfReflect_QF1_1"
54
+
55
+ def get_cloud_mask(self, target_shape: tuple = None) -> Raster:
56
+ h, v = self.hv
57
+
58
+ if self._cloud_mask is None:
59
+ with h5py.File(self.filename, "r") as f:
60
+ with warnings.catch_warnings():
61
+ warnings.filterwarnings("ignore")
62
+ QF1 = np.array(f[self.CLOUD_DATASET_NAME])
63
+
64
+ cloud_levels = (QF1 >> 2) & 3
65
+ cloud_mask = cloud_levels > 0
66
+ self._cloud_mask = cloud_mask
67
+ else:
68
+ cloud_mask = self._cloud_mask
69
+
70
+ if target_shape is not None:
71
+ cloud_mask = resize(cloud_mask, target_shape, order=0).astype(bool)
72
+ shape = target_shape
73
+ else:
74
+ shape = cloud_mask.shape
75
+
76
+ geometry = generate_modland_grid(h, v, shape[0])
77
+ cloud_mask = Raster(cloud_mask, geometry=geometry)
78
+
79
+ return cloud_mask
80
+
81
+ cloud_mask = property(get_cloud_mask)
82
+
83
+ def dataset(
84
+ self,
85
+ filename: str,
86
+ dataset_name: str,
87
+ scale_factor: float,
88
+ cloud_mask: Raster = None,
89
+ apply_cloud_mask: bool = True,
90
+ geometry: RasterGeometry = None,
91
+ resampling: str = None) -> Raster:
92
+
93
+ with h5py.File(filename, "r") as f:
94
+ with warnings.catch_warnings():
95
+ warnings.filterwarnings("ignore")
96
+ dataset = f[dataset_name]
97
+ DN = np.array(dataset)
98
+
99
+ if "_FillValue" in dataset.attrs:
100
+ fill_value = dataset.attrs["_FillValue"]
101
+ else:
102
+ fill_value = dataset.attrs["_Fillvalue"]
103
+
104
+ h, v = self.hv
105
+ grid = generate_modland_grid(h, v, DN.shape[0])
106
+ logger.info(f"opening VIIRS file: {cl.file(self.filename)}")
107
+ logger.info(f"loading {cl.val(dataset_name)} at {cl.val(f'{grid.cell_size:0.2f} m')} resolution")
108
+ DN = np.where(DN == fill_value, np.nan, DN)
109
+ DN = Raster(DN, geometry=grid)
110
+
111
+ data = DN * scale_factor
112
+
113
+ if apply_cloud_mask:
114
+ if cloud_mask is None:
115
+ cloud_mask = self.get_cloud_mask(target_shape=DN.shape)
116
+
117
+ data = rasters.where(cloud_mask, np.nan, data)
118
+
119
+ if geometry is not None:
120
+ data = data.to_geometry(geometry, resampling=resampling)
121
+
122
+ return data
123
+
124
+ @property
125
+ def geometry_M(self) -> RasterGrid:
126
+ return generate_modland_grid(*self.hv, 1200)
127
+
128
+ @property
129
+ def geometry_I(self) -> RasterGrid:
130
+ return generate_modland_grid(*self.hv, 2400)
131
+
132
+ def geometry(self, band: str) -> RasterGrid:
133
+ try:
134
+ band_letter = band[0]
135
+ except Exception as e:
136
+ raise ValueError(f"invalid band: {band}")
137
+
138
+ if band_letter == "I":
139
+ return self.geometry_I
140
+ elif band_letter == "M":
141
+ return self.geometry_M
142
+ else:
143
+ raise ValueError(f"invalid band: {band}")
144
+
145
+ def get_sensor_zenith_M(
146
+ self,
147
+ geometry: RasterGeometry = None,
148
+ save_data: bool = False,
149
+ save_preview: bool = False,
150
+ product_filename: str = None) -> Raster:
151
+ if product_filename is None:
152
+ product_filename = self.product_filename(f"sensor_zenith_M")
153
+
154
+ image = None
155
+
156
+ if product_filename is not None and exists(product_filename):
157
+ try:
158
+ logger.info(f"loading VIIRS sensor zenith: {cl.file(product_filename)}")
159
+ image = Raster.open(product_filename)
160
+ except Exception as e:
161
+ logger.exception(e)
162
+ logger.warning(f"removing corrupted file: {product_filename}")
163
+ remove(product_filename)
164
+ image = None
165
+
166
+ if image is None:
167
+ image = self.dataset(
168
+ self.filename,
169
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorZenith_1",
170
+ 0.01,
171
+ cloud_mask=None,
172
+ apply_cloud_mask=False
173
+ )
174
+
175
+ if np.all(np.isnan(image)):
176
+ raise ValueError("blank sensor zenith image")
177
+
178
+ if save_data and not exists(product_filename):
179
+ logger.info(f"writing VIIRS M-band sensor zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
180
+ image.to_geotiff(product_filename)
181
+
182
+ if save_preview:
183
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
184
+
185
+ if geometry is not None:
186
+ image = image.to_geometry(geometry)
187
+
188
+ return image
189
+
190
+ sensor_zenith_M = property(get_sensor_zenith_M)
191
+
192
+ def get_sensor_zenith_I(
193
+ self,
194
+ geometry: RasterGeometry = None,
195
+ save_data: bool = False,
196
+ save_preview: bool = False,
197
+ product_filename: str = None) -> Raster:
198
+ if product_filename is None:
199
+ product_filename = self.product_filename(f"sensor_zenith_I")
200
+
201
+ image = None
202
+
203
+ if product_filename is not None and exists(product_filename):
204
+ try:
205
+ logger.info(f"loading VIIRS I-band sensor zenith: {cl.file(product_filename)}")
206
+ image = Raster.open(product_filename)
207
+ except Exception as e:
208
+ logger.exception(e)
209
+ logger.warning(f"removing corrupted file: {product_filename}")
210
+ remove(product_filename)
211
+ image = None
212
+
213
+ if image is None:
214
+ h, v = self.hv
215
+ grid_I = generate_modland_grid(h, v, 2400)
216
+
217
+ image = self.dataset(
218
+ self.filename,
219
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorZenith_1",
220
+ 0.01,
221
+ cloud_mask=None,
222
+ apply_cloud_mask=False,
223
+ geometry=grid_I,
224
+ resampling="cubic"
225
+ )
226
+
227
+ if np.all(np.isnan(image)):
228
+ raise ValueError("blank sensor zenith image")
229
+
230
+ if save_data and not exists(product_filename):
231
+ logger.info(f"writing VIIRS sensor zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
232
+ image.to_geotiff(product_filename)
233
+
234
+ if save_preview:
235
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
236
+
237
+ if geometry is not None:
238
+ image = image.to_geometry(geometry)
239
+
240
+ return image
241
+
242
+ sensor_zenith_I = property(get_sensor_zenith_I)
243
+
244
+ def sensor_zenith(
245
+ self,
246
+ band: str,
247
+ geometry: RasterGeometry = None,
248
+ save_data: bool = False,
249
+ save_preview: bool = False,
250
+ product_filename: str = None) -> Raster:
251
+ try:
252
+ band_letter = band[0]
253
+ except Exception as e:
254
+ raise ValueError(f"invalid band: {band}")
255
+
256
+ if band_letter == "I":
257
+ return self.get_sensor_zenith_I(
258
+ geometry=geometry,
259
+ save_data=save_data,
260
+ save_preview=save_preview,
261
+ product_filename=product_filename
262
+ )
263
+ elif band_letter == "M":
264
+ return self.get_sensor_zenith_M(
265
+ geometry=geometry,
266
+ save_data=save_data,
267
+ save_preview=save_preview,
268
+ product_filename=product_filename
269
+ )
270
+ else:
271
+ raise ValueError(f"invalid band: {band}")
272
+
273
+ def get_sensor_azimuth_M(
274
+ self,
275
+ geometry: RasterGeometry = None,
276
+ save_data: bool = False,
277
+ save_preview: bool = False,
278
+ product_filename: str = None) -> Raster:
279
+ if product_filename is None:
280
+ product_filename = self.product_filename(f"sensor_azimuth_M")
281
+
282
+ image = None
283
+
284
+ if product_filename is not None and exists(product_filename):
285
+ try:
286
+ logger.info(f"loading VIIRS M-band sensor azimuth: {cl.file(product_filename)}")
287
+ image = Raster.open(product_filename)
288
+ except Exception as e:
289
+ logger.exception(e)
290
+ logger.warning(f"removing corrupted file: {product_filename}")
291
+ remove(product_filename)
292
+ image = None
293
+
294
+ if image is None:
295
+ image = self.dataset(
296
+ self.filename,
297
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorAzimuth_1",
298
+ 0.01,
299
+ cloud_mask=None,
300
+ apply_cloud_mask=False
301
+ )
302
+
303
+ if np.all(np.isnan(image)):
304
+ raise ValueError("blank sensor azimuth image")
305
+
306
+ if save_data and not exists(product_filename):
307
+ logger.info(f"writing VIIRS sensor azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
308
+ image.to_geotiff(product_filename)
309
+
310
+ if save_preview:
311
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
312
+
313
+ if geometry is not None:
314
+ image = image.to_geometry(geometry)
315
+
316
+ return image
317
+
318
+ sensor_azimuth_M = property(get_sensor_azimuth_M)
319
+
320
+ def get_sensor_azimuth_I(
321
+ self,
322
+ geometry: RasterGeometry = None,
323
+ save_data: bool = False,
324
+ save_preview: bool = False,
325
+ product_filename: str = None) -> Raster:
326
+ if product_filename is None:
327
+ product_filename = self.product_filename(f"sensor_azimuth_I")
328
+
329
+ image = None
330
+
331
+ if product_filename is not None and exists(product_filename):
332
+ try:
333
+ logger.info(f"loading VIIRS I-band sensor azimuth: {cl.file(product_filename)}")
334
+ image = Raster.open(product_filename)
335
+ except Exception as e:
336
+ logger.exception(e)
337
+ logger.warning(f"removing corrupted file: {product_filename}")
338
+ remove(product_filename)
339
+ image = None
340
+
341
+ if image is None:
342
+ h, v = self.hv
343
+ grid_I = generate_modland_grid(h, v, 2400)
344
+
345
+ image = self.dataset(
346
+ self.filename,
347
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorAzimuth_1",
348
+ 0.01,
349
+ cloud_mask=None,
350
+ apply_cloud_mask=False,
351
+ geometry=grid_I,
352
+ resampling="cubic"
353
+ )
354
+
355
+ if np.all(np.isnan(image)):
356
+ raise ValueError("blank sensor azimuth image")
357
+
358
+ if save_data and not exists(product_filename):
359
+ logger.info(f"writing VIIRS sensor azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
360
+ image.to_geotiff(product_filename)
361
+
362
+ if save_preview:
363
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
364
+
365
+ if geometry is not None:
366
+ image = image.to_geometry(geometry)
367
+
368
+ return image
369
+
370
+ sensor_azimuth_I = property(get_sensor_azimuth_I)
371
+
372
+ def sensor_azimuth(
373
+ self,
374
+ band: str,
375
+ geometry: RasterGeometry = None,
376
+ save_data: bool = False,
377
+ save_preview: bool = False,
378
+ product_filename: str = None) -> Raster:
379
+ try:
380
+ band_letter = band[0]
381
+ except Exception as e:
382
+ raise ValueError(f"invalid band: {band}")
383
+
384
+ if band_letter == "I":
385
+ return self.get_sensor_azimuth_I(
386
+ geometry=geometry,
387
+ save_data=save_data,
388
+ save_preview=save_preview,
389
+ product_filename=product_filename
390
+ )
391
+ elif band_letter == "M":
392
+ return self.get_sensor_azimuth_M(
393
+ geometry=geometry,
394
+ save_data=save_data,
395
+ save_preview=save_preview,
396
+ product_filename=product_filename
397
+ )
398
+ else:
399
+ raise ValueError(f"invalid band: {band}")
400
+
401
+ def get_solar_zenith_M(
402
+ self,
403
+ geometry: RasterGeometry = None,
404
+ save_data: bool = False,
405
+ save_preview: bool = False,
406
+ product_filename: str = None) -> Raster:
407
+ if product_filename is None:
408
+ product_filename = self.product_filename(f"solar_zenith_M")
409
+
410
+ image = None
411
+
412
+ if product_filename is not None and exists(product_filename):
413
+ try:
414
+ logger.info(f"loading VIIRS M-band solar zenith: {cl.file(product_filename)}")
415
+ image = Raster.open(product_filename)
416
+ except Exception as e:
417
+ logger.exception(e)
418
+ logger.warning(f"removing corrupted file: {product_filename}")
419
+ remove(product_filename)
420
+ image = None
421
+
422
+ if image is None:
423
+ image = self.dataset(
424
+ self.filename,
425
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarZenith_1",
426
+ 0.01,
427
+ cloud_mask=None,
428
+ apply_cloud_mask=False
429
+ )
430
+
431
+ if np.all(np.isnan(image)):
432
+ raise ValueError("blank solar zenith image")
433
+
434
+ if save_data and not exists(product_filename):
435
+ logger.info(f"writing VIIRS solar zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
436
+ image.to_geotiff(product_filename)
437
+
438
+ if save_preview:
439
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
440
+
441
+ if geometry is not None:
442
+ image = image.to_geometry(geometry)
443
+
444
+ return image
445
+
446
+ solar_zenith_M = property(get_solar_zenith_M)
447
+
448
+ def get_solar_zenith_I(
449
+ self,
450
+ geometry: RasterGeometry = None,
451
+ save_data: bool = False,
452
+ save_preview: bool = False,
453
+ product_filename: str = None) -> Raster:
454
+ if product_filename is None:
455
+ product_filename = self.product_filename("solar_zenith_I")
456
+
457
+ image = None
458
+
459
+ if product_filename is not None and exists(product_filename):
460
+ try:
461
+ logger.info(f"loading VIIRS I-band solar zenith: {cl.file(product_filename)}")
462
+ image = Raster.open(product_filename)
463
+ except Exception as e:
464
+ logger.exception(e)
465
+ logger.warning(f"removing corrupted file: {product_filename}")
466
+ remove(product_filename)
467
+ image = None
468
+
469
+ if image is None:
470
+ h, v = self.hv
471
+ grid_I = generate_modland_grid(h, v, 2400)
472
+
473
+ image = self.dataset(
474
+ self.filename,
475
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarZenith_1",
476
+ 0.01,
477
+ cloud_mask=None,
478
+ apply_cloud_mask=False,
479
+ geometry=grid_I,
480
+ resampling="cubic"
481
+ )
482
+
483
+ if np.all(np.isnan(image)):
484
+ raise ValueError("blank solar zenith image")
485
+
486
+ if save_data and not exists(product_filename):
487
+ logger.info(f"writing VIIRS solar zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
488
+
489
+ image.to_geotiff(product_filename)
490
+
491
+ if save_preview:
492
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
493
+
494
+ if geometry is not None:
495
+ image = image.to_geometry(geometry)
496
+
497
+ return image
498
+
499
+ solar_zenith_I = property(get_solar_zenith_I)
500
+
501
+ def solar_zenith(
502
+ self,
503
+ band: str,
504
+ geometry: RasterGeometry = None,
505
+ save_data: bool = False,
506
+ save_preview: bool = False,
507
+ product_filename: str = None) -> Raster:
508
+ try:
509
+ band_letter = band[0]
510
+ except Exception as e:
511
+ raise ValueError(f"invalid band: {band}")
512
+
513
+ if band_letter == "I":
514
+ return self.get_solar_zenith_I(
515
+ geometry=geometry,
516
+ save_data=save_data,
517
+ save_preview=save_preview,
518
+ product_filename=product_filename
519
+ )
520
+ elif band_letter == "M":
521
+ return self.get_solar_zenith_M(
522
+ geometry=geometry,
523
+ save_data=save_data,
524
+ save_preview=save_preview,
525
+ product_filename=product_filename
526
+ )
527
+ else:
528
+ raise ValueError(f"invalid band: {band}")
529
+
530
+ def get_solar_azimuth_M(
531
+ self,
532
+ geometry: RasterGeometry = None,
533
+ save_data: bool = False,
534
+ save_preview: bool = False,
535
+ product_filename: str = None) -> Raster:
536
+ if product_filename is None:
537
+ product_filename = self.product_filename("solar_azimuth_M")
538
+
539
+ image = None
540
+
541
+ if product_filename is not None and exists(product_filename):
542
+ try:
543
+ logger.info(f"loading VIIRS M-band solar azimuth: {cl.file(product_filename)}")
544
+ image = Raster.open(product_filename)
545
+ except Exception as e:
546
+ logger.exception(e)
547
+ logger.warning(f"removing corrupted file: {product_filename}")
548
+ remove(product_filename)
549
+ image = None
550
+
551
+ if image is None:
552
+ image = self.dataset(
553
+ self.filename,
554
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarAzimuth_1",
555
+ 0.01,
556
+ cloud_mask=None,
557
+ apply_cloud_mask=False
558
+ )
559
+
560
+ if np.all(np.isnan(image)):
561
+ raise ValueError("blank solar azimuth image")
562
+
563
+ if save_data and not exists(product_filename):
564
+ logger.info(f"writing VIIRS solar azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
565
+ image.to_geotiff(product_filename)
566
+
567
+ if save_preview:
568
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
569
+
570
+ if geometry is not None:
571
+ image = image.to_geometry(geometry)
572
+
573
+ return image
574
+
575
+ solar_azimuth_M = property(get_solar_azimuth_M)
576
+
577
+ def get_solar_azimuth_I(
578
+ self,
579
+ geometry: RasterGeometry = None,
580
+ save_data: bool = False,
581
+ save_preview: bool = False,
582
+ product_filename: str = None) -> Raster:
583
+ if product_filename is None:
584
+ product_filename = self.product_filename("solar_azimuth_I")
585
+
586
+ image = None
587
+
588
+ if product_filename is not None and exists(product_filename):
589
+ try:
590
+ logger.info(f"loading VIIRS I-band solar azimuth: {cl.file(product_filename)}")
591
+ image = Raster.open(product_filename)
592
+ except Exception as e:
593
+ logger.exception(e)
594
+ logger.warning(f"removing corrupted file: {product_filename}")
595
+ remove(product_filename)
596
+ image = None
597
+
598
+ if image is None:
599
+ h, v = self.hv
600
+ grid_I = generate_modland_grid(h, v, 2400)
601
+
602
+ image = self.dataset(
603
+ self.filename,
604
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarAzimuth_1",
605
+ 0.01,
606
+ cloud_mask=None,
607
+ apply_cloud_mask=False,
608
+ geometry=grid_I,
609
+ resampling="cubic"
610
+ )
611
+
612
+ if np.all(np.isnan(image)):
613
+ raise ValueError("blank solar azimuth image")
614
+
615
+ if save_data and not exists(product_filename):
616
+ logger.info(f"writing VIIRS solar azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
617
+ image.to_geotiff(product_filename)
618
+
619
+ if save_preview:
620
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
621
+
622
+ if geometry is not None:
623
+ image = image.to_geometry(geometry)
624
+
625
+ return image
626
+
627
+ solar_azimuth_I = property(get_solar_azimuth_I)
628
+
629
+ def solar_azimuth(
630
+ self,
631
+ band: str,
632
+ geometry: RasterGeometry = None,
633
+ save_data: bool = False,
634
+ save_preview: bool = False,
635
+ product_filename: str = None) -> Raster:
636
+ try:
637
+ band_letter = band[0]
638
+ except Exception as e:
639
+ raise ValueError(f"invalid band: {band}")
640
+
641
+ if band_letter == "I":
642
+ return self.get_solar_azimuth_I(
643
+ geometry=geometry,
644
+ save_data=save_data,
645
+ save_preview=save_preview,
646
+ product_filename=product_filename
647
+ )
648
+ elif band_letter == "M":
649
+ return self.get_solar_azimuth_M(
650
+ geometry=geometry,
651
+ save_data=save_data,
652
+ save_preview=save_preview,
653
+ product_filename=product_filename
654
+ )
655
+ else:
656
+ raise ValueError(f"invalid band: {band}")
657
+
658
+ def get_M_band(
659
+ self,
660
+ band: int,
661
+ cloud_mask: Raster = None,
662
+ apply_cloud_mask: bool = True,
663
+ geometry: RasterGeometry = None,
664
+ save_data: bool = False,
665
+ save_preview: bool = False,
666
+ product_filename: str = None) -> Raster:
667
+ if product_filename is None:
668
+ product_filename = self.product_filename(f"M{band}")
669
+
670
+ image = None
671
+
672
+ if product_filename is not None and exists(product_filename):
673
+ try:
674
+ logger.info(f"loading VIIRS M-band {band} surface reflectance: {cl.file(product_filename)}")
675
+ image = Raster.open(product_filename)
676
+ except Exception as e:
677
+ logger.exception(e)
678
+ logger.warning(f"removing corrupted file: {product_filename}")
679
+ remove(product_filename)
680
+ image = None
681
+
682
+ if image is None:
683
+ image = self.dataset(
684
+ self.filename,
685
+ f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SurfReflect_M{int(band)}_1",
686
+ 0.0001,
687
+ cloud_mask=cloud_mask,
688
+ apply_cloud_mask=apply_cloud_mask
689
+ )
690
+
691
+ if save_data and not exists(product_filename):
692
+ logger.info(f"writing VIIRS M{band}: {cl.file(product_filename)}")
693
+ image.to_geotiff(product_filename)
694
+
695
+ if save_preview:
696
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
697
+
698
+ if geometry is not None:
699
+ image = image.to_geometry(geometry)
700
+
701
+ return image
702
+
703
+ def get_I_band(
704
+ self,
705
+ band: int,
706
+ cloud_mask: Raster = None,
707
+ apply_cloud_mask: bool = True,
708
+ geometry: RasterGeometry = None,
709
+ save_data: bool = False,
710
+ save_preview: bool = False,
711
+ product_filename: str = None) -> Raster:
712
+ if product_filename is None:
713
+ product_filename = self.product_filename(f"I{band}")
714
+
715
+ image = None
716
+
717
+ if product_filename is not None and exists(product_filename):
718
+ try:
719
+ logger.info(f"loading VIIRS I-band {band} surface reflectance: {cl.file(product_filename)}")
720
+ image = Raster.open(product_filename)
721
+ except Exception as e:
722
+ logger.exception(e)
723
+ logger.warning(f"removing corrupted file: {product_filename}")
724
+ remove(product_filename)
725
+ image = None
726
+
727
+ if image is None:
728
+ image = self.dataset(
729
+ self.filename,
730
+ f"HDFEOS/GRIDS/VIIRS_Grid_500m_2D/Data Fields/SurfReflect_I{int(band)}_1",
731
+ 0.0001,
732
+ cloud_mask=cloud_mask,
733
+ apply_cloud_mask=apply_cloud_mask
734
+ )
735
+
736
+ if save_data and not exists(product_filename):
737
+ logger.info(f"writing VIIRS I{band}: {cl.file(product_filename)}")
738
+ image.to_geotiff(product_filename)
739
+
740
+ if save_preview:
741
+ image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
742
+
743
+ if geometry is not None:
744
+ image = image.to_geometry(geometry)
745
+
746
+ return image
747
+
748
+ def band(
749
+ self,
750
+ band: str,
751
+ cloud_mask: Raster = None,
752
+ apply_cloud_mask: bool = True,
753
+ geometry: RasterGeometry = None,
754
+ save_data: bool = False,
755
+ save_preview: bool = False,
756
+ product_filename: str = None) -> Raster:
757
+ try:
758
+ band_letter = band[0]
759
+ band_number = int(band[1:])
760
+ except Exception as e:
761
+ raise ValueError(f"invalid band: {band}")
762
+
763
+ if band_letter == "I":
764
+ return self.get_I_band(
765
+ band=band_number,
766
+ cloud_mask=cloud_mask,
767
+ apply_cloud_mask=apply_cloud_mask,
768
+ geometry=geometry,
769
+ save_data=save_data,
770
+ save_preview=save_preview,
771
+ product_filename=product_filename
772
+ )
773
+ elif band_letter == "M":
774
+ return self.get_M_band(
775
+ band=band_number,
776
+ cloud_mask=cloud_mask,
777
+ apply_cloud_mask=apply_cloud_mask,
778
+ geometry=geometry,
779
+ save_data=save_data,
780
+ save_preview=save_preview,
781
+ product_filename=product_filename
782
+ )
783
+ else:
784
+ raise ValueError(f"invalid band: {band}")
785
+
786
+ def get_red(
787
+ self,
788
+ cloud_mask: Raster = None,
789
+ apply_cloud_mask: bool = True,
790
+ geometry: RasterGeometry = None,
791
+ save_data: bool = False,
792
+ save_preview: bool = False,
793
+ product_filename: str = None) -> Raster:
794
+ return self.get_I_band(
795
+ band=1,
796
+ cloud_mask=cloud_mask,
797
+ apply_cloud_mask=apply_cloud_mask,
798
+ geometry=geometry,
799
+ save_data=save_data,
800
+ save_preview=save_preview,
801
+ product_filename=product_filename
802
+ )
803
+
804
+ red = property(get_red)
805
+
806
+ def get_NIR(
807
+ self,
808
+ cloud_mask: Raster = None,
809
+ apply_cloud_mask: bool = True,
810
+ geometry: RasterGeometry = None,
811
+ save_data: bool = False,
812
+ save_preview: bool = False,
813
+ product_filename: str = None) -> Raster:
814
+ return self.get_I_band(
815
+ band=2,
816
+ cloud_mask=cloud_mask,
817
+ apply_cloud_mask=apply_cloud_mask,
818
+ geometry=geometry,
819
+ save_data=save_data,
820
+ save_preview=save_preview,
821
+ product_filename=product_filename
822
+ )
823
+
824
+ NIR = property(get_NIR)
825
+
826
+ def get_NDVI(
827
+ self,
828
+ cloud_mask: Raster = None,
829
+ apply_cloud_mask: bool = True,
830
+ geometry: RasterGeometry = None,
831
+ save_data: bool = False,
832
+ save_preview: bool = False,
833
+ product_filename: str = None) -> Raster:
834
+ if product_filename is None:
835
+ product_filename = self.product_filename("NDVI")
836
+
837
+ if product_filename is not None and exists(product_filename):
838
+ logger.info(f"loading VIIRS NDVI: {cl.file(product_filename)}")
839
+ NDVI = Raster.open(product_filename)
840
+ else:
841
+ red = self.get_red(
842
+ cloud_mask=cloud_mask,
843
+ apply_cloud_mask=apply_cloud_mask,
844
+ geometry=geometry,
845
+ save_data=save_data,
846
+ save_preview=save_preview
847
+ )
848
+
849
+ NIR = self.get_NIR(
850
+ cloud_mask=cloud_mask,
851
+ apply_cloud_mask=apply_cloud_mask,
852
+ geometry=geometry,
853
+ save_data=save_data,
854
+ save_preview=save_preview
855
+ )
856
+
857
+ NDVI = np.clip((NIR - red) / (NIR + red), -1, 1)
858
+
859
+ if save_data and not exists(product_filename):
860
+ logger.info(f"writing VIIRS NDVI: {cl.file(product_filename)}")
861
+ NDVI.to_geotiff(product_filename)
862
+
863
+ if save_preview:
864
+ NDVI.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"))
865
+
866
+ if geometry is not None:
867
+ NDVI = NDVI.to_geometry(geometry)
868
+
869
+ NDVI.cmap = NDVI_COLORMAP
870
+
871
+ return NDVI
872
+
873
+ NDVI = property(get_NDVI)
874
+
875
+ def get_albedo(
876
+ self,
877
+ cloud_mask: Raster = None,
878
+ apply_cloud_mask: bool = True,
879
+ geometry: RasterGeometry = None,
880
+ save_data: bool = False,
881
+ save_preview: bool = False,
882
+ product_filename: str = None) -> Raster:
883
+ if product_filename is None:
884
+ product_filename = self.product_filename("albedo")
885
+
886
+ if product_filename is not None and exists(product_filename):
887
+ logger.info(f"loading VIIRS albedo: {cl.file(product_filename)}")
888
+ albedo = Raster.open(product_filename)
889
+ else:
890
+ b1 = self.get_M_band(
891
+ 1,
892
+ cloud_mask=cloud_mask,
893
+ apply_cloud_mask=apply_cloud_mask,
894
+ geometry=geometry,
895
+ save_data=save_data,
896
+ save_preview=save_preview
897
+ )
898
+
899
+ b2 = self.get_M_band(
900
+ 2,
901
+ cloud_mask=cloud_mask,
902
+ apply_cloud_mask=apply_cloud_mask,
903
+ geometry=geometry,
904
+ save_data=save_data,
905
+ save_preview=save_preview
906
+ )
907
+
908
+ b3 = self.get_M_band(
909
+ 3,
910
+ cloud_mask=cloud_mask,
911
+ apply_cloud_mask=apply_cloud_mask,
912
+ geometry=geometry,
913
+ save_data=save_data,
914
+ save_preview=save_preview
915
+ )
916
+
917
+ b4 = self.get_M_band(
918
+ 4,
919
+ cloud_mask=cloud_mask,
920
+ apply_cloud_mask=apply_cloud_mask,
921
+ geometry=geometry,
922
+ save_data=save_data,
923
+ save_preview=save_preview
924
+ )
925
+
926
+ b5 = self.get_M_band(
927
+ 5,
928
+ cloud_mask=cloud_mask,
929
+ apply_cloud_mask=apply_cloud_mask,
930
+ geometry=geometry,
931
+ save_data=save_data,
932
+ save_preview=save_preview
933
+ )
934
+
935
+ b7 = self.get_M_band(
936
+ 7,
937
+ cloud_mask=cloud_mask,
938
+ apply_cloud_mask=apply_cloud_mask,
939
+ geometry=geometry,
940
+ save_data=save_data,
941
+ save_preview=save_preview
942
+ )
943
+
944
+ b8 = self.get_M_band(
945
+ 8,
946
+ cloud_mask=cloud_mask,
947
+ apply_cloud_mask=apply_cloud_mask,
948
+ geometry=geometry,
949
+ save_data=save_data,
950
+ save_preview=save_preview
951
+ )
952
+
953
+ b10 = self.get_M_band(
954
+ 10,
955
+ cloud_mask=cloud_mask,
956
+ apply_cloud_mask=apply_cloud_mask,
957
+ geometry=geometry,
958
+ save_data=save_data,
959
+ save_preview=save_preview
960
+ )
961
+
962
+ b11 = self.get_M_band(
963
+ 11,
964
+ cloud_mask=cloud_mask,
965
+ apply_cloud_mask=apply_cloud_mask,
966
+ geometry=geometry,
967
+ save_data=save_data,
968
+ save_preview=save_preview
969
+ )
970
+
971
+ # https://lpdaac.usgs.gov/documents/194/VNP43_ATBD_V1.pdf
972
+ albedo = 0.2418 * b1 \
973
+ - 0.201 * b2 \
974
+ + 0.2093 * b3 \
975
+ + 0.1146 * b4 \
976
+ + 0.1348 * b5 \
977
+ + 0.2251 * b7 \
978
+ + 0.1123 * b8 \
979
+ + 0.0860 * b10 \
980
+ + 0.0803 * b11 \
981
+ - 0.0131
982
+
983
+ albedo = np.clip(albedo, 0, 1)
984
+
985
+ if save_data and not exists(product_filename):
986
+ logger.info(f"writing VIIRS albedo: {cl.file(product_filename)}")
987
+ albedo.to_geotiff(product_filename)
988
+
989
+ if geometry is not None:
990
+ logger.info(f"projecting VIIRS albedo from {cl.val(albedo.geometry.cell_size)} to {cl.val(geometry.cell_size)}")
991
+ albedo = albedo.to_geometry(geometry)
992
+
993
+ albedo.cmap = ALBEDO_COLORMAP
994
+
995
+ return albedo
996
+
997
+ albedo = property(get_albedo)
998
+
999
+
1000
+ VIIRS_CONCEPT = "C2631841556-LPCLOUD"
1001
+
1002
+ def earliest_datetime(date_in: Union[date, str]) -> datetime:
1003
+ if isinstance(date_in, str):
1004
+ datetime_in = parser.parse(date_in)
1005
+ else:
1006
+ datetime_in = date_in
1007
+
1008
+ date_string = datetime_in.strftime("%Y-%m-%d")
1009
+ return parser.parse(f"{date_string}T00:00:00Z")
1010
+
1011
+
1012
+ def latest_datetime(date_in: Union[date, str]) -> datetime:
1013
+ if isinstance(date_in, str):
1014
+ datetime_in = parser.parse(date_in)
1015
+ else:
1016
+ datetime_in = date_in
1017
+
1018
+ date_string = datetime_in.strftime("%Y-%m-%d")
1019
+ return parser.parse(f"{date_string}T23:59:59Z")
1020
+
1021
+
1022
+ VIIRS_FILENAME_REGEX = re.compile("^VNP09GA\.[^.]+\.([^.]+)\.002\.\d+\.h5$")
1023
+ def modland_tile_from_filename(filename: str) -> str:
1024
+ match = VIIRS_FILENAME_REGEX.match(filename)
1025
+ if match is None:
1026
+ raise RuntimeError(f"Invalid filename found through VIIRS CMR search: {filename}")
1027
+
1028
+ return match.group(1)
1029
+
1030
+
1031
+ # TODO: Deduplicate between VIIRS and HLS
1032
+ def VIIRS_CMR_query(
1033
+ start_date: Union[date, str],
1034
+ end_date: Union[date, str],
1035
+ target_geometry: Point or Polygon or RasterGeometry = None,
1036
+ tile: str = None,
1037
+ ) -> List[earthaccess.search.DataGranule]:
1038
+ """function to search for VIIRS at tile in date range"""
1039
+ query = earthaccess.granule_query() \
1040
+ .concept_id(VIIRS_CONCEPT) \
1041
+ .temporal(earliest_datetime(start_date), latest_datetime(end_date))
1042
+
1043
+ if isinstance(target_geometry, Point):
1044
+ query = query.point(target_geometry.x, target_geometry.y)
1045
+ if isinstance(target_geometry, Polygon):
1046
+ ring = target_geometry.exterior
1047
+ if not ring.is_ccw:
1048
+ ring = ring.reverse()
1049
+ coordinates = ring.coords
1050
+ query = query.polygon(coordinates)
1051
+ if isinstance(target_geometry, RasterGeometry):
1052
+ ring = target_geometry.corner_polygon_latlon.exterior
1053
+ if not ring.is_ccw:
1054
+ ring = ring.reverse()
1055
+ coordinates = ring.coords
1056
+ query = query.polygon(coordinates)
1057
+ if tile is not None:
1058
+ query = query.readable_granule_name(f"*.{tile}.*")
1059
+
1060
+ granules: List[earthaccess.search.DataGranule]
1061
+ try:
1062
+ granules = query.get()
1063
+ except Exception as e:
1064
+ raise CMRServerUnreachable(e)
1065
+ granules = sorted(granules, key=lambda granule: granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"])
1066
+
1067
+ logger.info("Found the following granules for VIIRS 2 using the CMR search:")
1068
+ for granule in granules:
1069
+ logger.info(" " + cl.file(granule["meta"]["native-id"]))
1070
+ logger.info(f"Number of VIIRS 2 granules found using CMR search: {len(granules)}")
1071
+
1072
+ return granules
1073
+
1074
+
1075
+ class VNP09GA:
1076
+ DEFAULT_WORKING_DIRECTORY = "."
1077
+ DEFAULT_DOWNLOAD_DIRECTORY = "VNP09GA_download"
1078
+ DEFAULT_PRODUCTS_DIRECTORY = "VNP09GA_products"
1079
+ DEFAULT_MOSAIC_DIRECTORY = "VNP09GA_mosaics"
1080
+ DEFAULT_RESAMPLING = "nearest"
1081
+
1082
+ CLOUD_DATASET_NAME = "HDFEOS/GRIDS/VNP_Grid_1km_2D/Data Fields/SurfReflect_QF1_1"
1083
+
1084
+ def __init__(
1085
+ self,
1086
+ working_directory: str = None,
1087
+ download_directory: str = None,
1088
+ products_directory: str = None,
1089
+ mosaic_directory: str = None,
1090
+ resampling: str = None):
1091
+
1092
+ if resampling is None:
1093
+ resampling = self.DEFAULT_RESAMPLING
1094
+
1095
+ self.resampling = resampling
1096
+
1097
+ self._granules = pd.DataFrame({"date_UTC": {}, "tile": {}, "granule": {}})
1098
+
1099
+ if working_directory is None:
1100
+ working_directory = self.DEFAULT_WORKING_DIRECTORY
1101
+
1102
+ working_directory = abspath(expanduser(working_directory))
1103
+
1104
+ if download_directory is None:
1105
+ download_directory = join(working_directory, self.DEFAULT_DOWNLOAD_DIRECTORY)
1106
+
1107
+ download_directory = abspath(expanduser(download_directory))
1108
+
1109
+ if products_directory is None:
1110
+ products_directory = join(working_directory, self.DEFAULT_PRODUCTS_DIRECTORY)
1111
+
1112
+ products_directory = abspath(expanduser(products_directory))
1113
+
1114
+ if mosaic_directory is None:
1115
+ mosaic_directory = join(working_directory, self.DEFAULT_MOSAIC_DIRECTORY)
1116
+
1117
+ mosaic_directory = abspath(expanduser(mosaic_directory))
1118
+
1119
+ self.working_directory = working_directory
1120
+ self.download_directory = download_directory
1121
+ self.products_directory = products_directory
1122
+ self.mosaic_directory = mosaic_directory
1123
+
1124
+ self.auth = VIIRS_CMR_login()
1125
+
1126
+ def add_granules(self, granules: List[earthaccess.search.DataGranule]):
1127
+ data = pd.DataFrame([
1128
+ {
1129
+ "date_UTC": get_date(granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]),
1130
+ "tile": modland_tile_from_filename(Path(granule.data_links()[0]).name),
1131
+ "granule": granule,
1132
+ }
1133
+ for granule in granules
1134
+ ])
1135
+
1136
+ self._granules = pd.concat([self._granules, data]).drop_duplicates(subset=["date_UTC", "tile"])
1137
+
1138
+ def download_granules(self, granules: List[earthaccess.search.DataGranule]) -> List[str]:
1139
+ # Check if any of the granules have already been downloaded, and if so record the file path for that granule.
1140
+ # Save the granules that haven't been downloaded to download them later.
1141
+ granules_to_download = []
1142
+ output_paths = []
1143
+ for granule in granules:
1144
+ date_UTC = get_date(granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"])
1145
+ output_file_path = join(
1146
+ self.download_directory,
1147
+ "VNP09GA",
1148
+ f"{date_UTC:%Y.%m.%d}",
1149
+ Path(granule.data_links()[0]).name
1150
+ )
1151
+ if Path(output_file_path).exists():
1152
+ output_paths.append(output_file_path)
1153
+ else:
1154
+ granules_to_download.append(granule)
1155
+
1156
+ # Early exit
1157
+ if len(granules_to_download) == 0:
1158
+ logger.info("All VIIRS granules have already been downloaded")
1159
+ return output_paths
1160
+
1161
+ # Make sure to remove this before we return, so we use try..finally to avoid exceptions causing issues
1162
+ temporary_parent_directory = join(self.download_directory, "tmp")
1163
+ os.makedirs(temporary_parent_directory, exist_ok=True)
1164
+ temporary_download_directory = tempfile.mkdtemp(dir=temporary_parent_directory)
1165
+
1166
+ try:
1167
+ last_download_exception = None
1168
+ for _ in range(0, RETRIES):
1169
+ download_exception = None
1170
+ downloaded_granules = []
1171
+
1172
+ file_paths = earthaccess.download(granules_to_download, local_path=temporary_download_directory)
1173
+
1174
+ for (granule, download_file_path) in zip(granules_to_download, file_paths):
1175
+ if isinstance(download_file_path, Exception):
1176
+ if download_exception is None:
1177
+ download_exception = download_file_path
1178
+ continue
1179
+ date_UTC = get_date(granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"])
1180
+
1181
+ download_file_path = Path(download_file_path)
1182
+ output_file_path = join(
1183
+ self.download_directory,
1184
+ "VNP09GA",
1185
+ f"{date_UTC:%Y.%m.%d}",
1186
+ download_file_path.name
1187
+ )
1188
+ Path(output_file_path).parent.mkdir(parents=True, exist_ok=True)
1189
+ download_file_path.rename(output_file_path)
1190
+
1191
+ output_paths.append(output_file_path)
1192
+ downloaded_granules.append(granule)
1193
+
1194
+ if download_exception is not None:
1195
+ last_download_exception = download_exception
1196
+ for granule in downloaded_granules:
1197
+ granules_to_download.remove(granule)
1198
+ logger.warning("Encountered an exception while downloading VIIRS files:", exc_info=download_exception)
1199
+ logger.info(f"Retrying the VIIRS download with the remaining {len(granules_to_download)} granules.")
1200
+ else:
1201
+ granules_to_download = []
1202
+ break
1203
+
1204
+ if len(granules_to_download) > 0:
1205
+ raise DownloadFailed("Error when downloading VIIRS files") from last_download_exception
1206
+ finally:
1207
+ Path(temporary_download_directory).rmdir()
1208
+
1209
+ return output_paths
1210
+
1211
+ def prefetch_VNP09GA(
1212
+ self,
1213
+ start_date: Union[date, str],
1214
+ end_date: Union[date, str],
1215
+ geometry: Point or Polygon or RasterGeometry = None):
1216
+ # Fetch list of granules to download
1217
+ granules = VIIRS_CMR_query(
1218
+ start_date,
1219
+ end_date,
1220
+ geometry,
1221
+ )
1222
+
1223
+ self.add_granules(granules)
1224
+
1225
+ self.download_granules(granules)
1226
+
1227
+ def search(
1228
+ self,
1229
+ date_UTC: date,
1230
+ tile: str) -> Union[earthaccess.search.DataGranule, None]:
1231
+ if "date_UTC" not in self._granules.columns:
1232
+ raise ValueError(f"date_UTC column not in granules table")
1233
+
1234
+ subset = self._granules[(self._granules.date_UTC == date_UTC) & (self._granules.tile == tile)]
1235
+ if len(subset) > 0:
1236
+ return subset.iloc[0].granule
1237
+
1238
+ granules = VIIRS_CMR_query(
1239
+ start_date=date_UTC,
1240
+ end_date=date_UTC,
1241
+ tile=tile,
1242
+ )
1243
+
1244
+ if len(granules) == 0:
1245
+ return None
1246
+
1247
+ if len(granules) > 0:
1248
+ logger.warning("Found more VIIRS granules than expected")
1249
+
1250
+ self.add_granules(granules)
1251
+
1252
+ return granules[0]
1253
+
1254
+ def granule(
1255
+ self,
1256
+ date_UTC: date,
1257
+ tile: str) -> VNP09GAGranule:
1258
+ if isinstance(date_UTC, str):
1259
+ date_UTC = parser.parse(date_UTC).date()
1260
+
1261
+ logger.info(f"searching VNP09GA tile {tile} date {date_UTC}")
1262
+ granule = self.search(
1263
+ date_UTC=date_UTC,
1264
+ tile=tile
1265
+ )
1266
+
1267
+ if granule is None:
1268
+ raise VIIRSUnavailableError(f"VNP09GA URL not available at tile {tile} on date {date_UTC}")
1269
+
1270
+ output_path = self.download_granules([granule])[0]
1271
+
1272
+ output_granule = VNP09GAGranule(
1273
+ filename=output_path,
1274
+ products_directory=self.products_directory
1275
+ )
1276
+
1277
+ return output_granule