ECOv003-L2T-STARS 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ECOv003_L2T_STARS/BRDF/BRDF.py +57 -0
- ECOv003_L2T_STARS/BRDF/SZA.py +65 -0
- ECOv003_L2T_STARS/BRDF/__init__.py +1 -0
- ECOv003_L2T_STARS/BRDF/statistical_radiative_transport.txt +90 -0
- ECOv003_L2T_STARS/BRDF/version.txt +1 -0
- ECOv003_L2T_STARS/ECOv003_DL.py +527 -0
- ECOv003_L2T_STARS/ECOv003_DL.xml +47 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.py +162 -0
- ECOv003_L2T_STARS/ECOv003_L2T_STARS.xml +47 -0
- ECOv003_L2T_STARS/L2TSTARSConfig.py +188 -0
- ECOv003_L2T_STARS/L2T_STARS.py +489 -0
- ECOv003_L2T_STARS/LPDAAC/LPDAACDataPool.py +444 -0
- ECOv003_L2T_STARS/LPDAAC/__init__.py +9 -0
- ECOv003_L2T_STARS/LPDAAC/version.txt +1 -0
- ECOv003_L2T_STARS/Manifest.toml +2332 -0
- ECOv003_L2T_STARS/Project.toml +14 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDataPool.py +294 -0
- ECOv003_L2T_STARS/VIIRS/VIIRSDownloader.py +26 -0
- ECOv003_L2T_STARS/VIIRS/VIIRS_CMR_LOGIN.py +36 -0
- ECOv003_L2T_STARS/VIIRS/VNP09GA.py +1277 -0
- ECOv003_L2T_STARS/VIIRS/VNP43IA4.py +288 -0
- ECOv003_L2T_STARS/VIIRS/VNP43MA3.py +323 -0
- ECOv003_L2T_STARS/VIIRS/__init__.py +9 -0
- ECOv003_L2T_STARS/VIIRS/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT/VNP43NRT.py +863 -0
- ECOv003_L2T_STARS/VNP43NRT/__init__.py +1 -0
- ECOv003_L2T_STARS/VNP43NRT/process_VNP43NRT.jl +169 -0
- ECOv003_L2T_STARS/VNP43NRT/version.txt +1 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Manifest.toml +995 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/Project.toml +15 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/__init__.py +0 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.jl +25 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/instantiate.py +13 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/VNP43NRT.jl +411 -0
- ECOv003_L2T_STARS/VNP43NRT_jl/src/__init__.py +0 -0
- ECOv003_L2T_STARS/__init__.py +3 -0
- ECOv003_L2T_STARS/calibrate_fine_to_coarse.py +60 -0
- ECOv003_L2T_STARS/constants.py +38 -0
- ECOv003_L2T_STARS/daterange/__init__.py +1 -0
- ECOv003_L2T_STARS/daterange/daterange.py +35 -0
- ECOv003_L2T_STARS/generate_L2T_STARS_runconfig.py +249 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_directory.py +21 -0
- ECOv003_L2T_STARS/generate_NDVI_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_directory.py +14 -0
- ECOv003_L2T_STARS/generate_NDVI_fine_image.py +28 -0
- ECOv003_L2T_STARS/generate_STARS_inputs.py +231 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_directory.py +18 -0
- ECOv003_L2T_STARS/generate_albedo_coarse_image.py +30 -0
- ECOv003_L2T_STARS/generate_albedo_fine_directory.py +17 -0
- ECOv003_L2T_STARS/generate_albedo_fine_image.py +30 -0
- ECOv003_L2T_STARS/generate_filename.py +37 -0
- ECOv003_L2T_STARS/generate_input_staging_directory.py +23 -0
- ECOv003_L2T_STARS/generate_model_state_tile_date_directory.py +28 -0
- ECOv003_L2T_STARS/generate_output_directory.py +28 -0
- ECOv003_L2T_STARS/install_STARS_jl.py +43 -0
- ECOv003_L2T_STARS/instantiate_STARS_jl.py +38 -0
- ECOv003_L2T_STARS/load_prior.py +248 -0
- ECOv003_L2T_STARS/prior.py +56 -0
- ECOv003_L2T_STARS/process_ECOSTRESS_data_fusion_distributed_bias.jl +420 -0
- ECOv003_L2T_STARS/process_STARS_product.py +507 -0
- ECOv003_L2T_STARS/process_julia_data_fusion.py +110 -0
- ECOv003_L2T_STARS/retrieve_STARS_sources.py +101 -0
- ECOv003_L2T_STARS/runconfig.py +70 -0
- ECOv003_L2T_STARS/timer/__init__.py +1 -0
- ECOv003_L2T_STARS/timer/timer.py +77 -0
- ECOv003_L2T_STARS/version.py +8 -0
- ECOv003_L2T_STARS/version.txt +1 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/METADATA +30 -23
- ecov003_l2t_stars-1.1.0.dist-info/RECORD +73 -0
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info}/WHEEL +1 -1
- ecov003_l2t_stars-1.1.0.dist-info/entry_points.txt +3 -0
- ecov003_l2t_stars-1.1.0.dist-info/top_level.txt +1 -0
- ECOv003_L2T_STARS-1.0.1.dist-info/RECORD +0 -5
- ECOv003_L2T_STARS-1.0.1.dist-info/top_level.txt +0 -1
- {ECOv003_L2T_STARS-1.0.1.dist-info → ecov003_l2t_stars-1.1.0.dist-info/licenses}/LICENSE +0 -0
@@ -0,0 +1,1277 @@
|
|
1
|
+
import logging
|
2
|
+
import os
|
3
|
+
import warnings
|
4
|
+
from datetime import datetime, date
|
5
|
+
from os import remove
|
6
|
+
from os.path import exists, join, abspath, expanduser
|
7
|
+
import re
|
8
|
+
from pathlib import Path
|
9
|
+
import tempfile
|
10
|
+
from typing import List, Union
|
11
|
+
|
12
|
+
import earthaccess
|
13
|
+
import h5py
|
14
|
+
import numpy as np
|
15
|
+
import pandas as pd
|
16
|
+
from matplotlib.colors import LinearSegmentedColormap
|
17
|
+
from dateutil import parser
|
18
|
+
from skimage.transform import resize
|
19
|
+
|
20
|
+
import colored_logging as cl
|
21
|
+
import rasters
|
22
|
+
from rasters import Raster, RasterGrid, RasterGeometry, Point, Polygon
|
23
|
+
from modland import generate_modland_grid
|
24
|
+
|
25
|
+
from ECOv003_exit_codes import *
|
26
|
+
|
27
|
+
from ..daterange import get_date
|
28
|
+
from ..LPDAAC.LPDAACDataPool import RETRIES
|
29
|
+
from .VIIRSDataPool import VIIRSGranule
|
30
|
+
from .VIIRS_CMR_LOGIN import CMRServerUnreachable, VIIRS_CMR_login
|
31
|
+
|
32
|
+
NDVI_COLORMAP = LinearSegmentedColormap.from_list(
|
33
|
+
name="NDVI",
|
34
|
+
colors=[
|
35
|
+
"#0000ff",
|
36
|
+
"#000000",
|
37
|
+
"#745d1a",
|
38
|
+
"#e1dea2",
|
39
|
+
"#45ff01",
|
40
|
+
"#325e32"
|
41
|
+
]
|
42
|
+
)
|
43
|
+
|
44
|
+
ALBEDO_COLORMAP = "gray"
|
45
|
+
|
46
|
+
logger = logging.getLogger(__name__)
|
47
|
+
|
48
|
+
class VIIRSUnavailableError(Exception):
|
49
|
+
pass
|
50
|
+
|
51
|
+
|
52
|
+
class VNP09GAGranule(VIIRSGranule):
|
53
|
+
CLOUD_DATASET_NAME = "HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SurfReflect_QF1_1"
|
54
|
+
|
55
|
+
def get_cloud_mask(self, target_shape: tuple = None) -> Raster:
|
56
|
+
h, v = self.hv
|
57
|
+
|
58
|
+
if self._cloud_mask is None:
|
59
|
+
with h5py.File(self.filename, "r") as f:
|
60
|
+
with warnings.catch_warnings():
|
61
|
+
warnings.filterwarnings("ignore")
|
62
|
+
QF1 = np.array(f[self.CLOUD_DATASET_NAME])
|
63
|
+
|
64
|
+
cloud_levels = (QF1 >> 2) & 3
|
65
|
+
cloud_mask = cloud_levels > 0
|
66
|
+
self._cloud_mask = cloud_mask
|
67
|
+
else:
|
68
|
+
cloud_mask = self._cloud_mask
|
69
|
+
|
70
|
+
if target_shape is not None:
|
71
|
+
cloud_mask = resize(cloud_mask, target_shape, order=0).astype(bool)
|
72
|
+
shape = target_shape
|
73
|
+
else:
|
74
|
+
shape = cloud_mask.shape
|
75
|
+
|
76
|
+
geometry = generate_modland_grid(h, v, shape[0])
|
77
|
+
cloud_mask = Raster(cloud_mask, geometry=geometry)
|
78
|
+
|
79
|
+
return cloud_mask
|
80
|
+
|
81
|
+
cloud_mask = property(get_cloud_mask)
|
82
|
+
|
83
|
+
def dataset(
|
84
|
+
self,
|
85
|
+
filename: str,
|
86
|
+
dataset_name: str,
|
87
|
+
scale_factor: float,
|
88
|
+
cloud_mask: Raster = None,
|
89
|
+
apply_cloud_mask: bool = True,
|
90
|
+
geometry: RasterGeometry = None,
|
91
|
+
resampling: str = None) -> Raster:
|
92
|
+
|
93
|
+
with h5py.File(filename, "r") as f:
|
94
|
+
with warnings.catch_warnings():
|
95
|
+
warnings.filterwarnings("ignore")
|
96
|
+
dataset = f[dataset_name]
|
97
|
+
DN = np.array(dataset)
|
98
|
+
|
99
|
+
if "_FillValue" in dataset.attrs:
|
100
|
+
fill_value = dataset.attrs["_FillValue"]
|
101
|
+
else:
|
102
|
+
fill_value = dataset.attrs["_Fillvalue"]
|
103
|
+
|
104
|
+
h, v = self.hv
|
105
|
+
grid = generate_modland_grid(h, v, DN.shape[0])
|
106
|
+
logger.info(f"opening VIIRS file: {cl.file(self.filename)}")
|
107
|
+
logger.info(f"loading {cl.val(dataset_name)} at {cl.val(f'{grid.cell_size:0.2f} m')} resolution")
|
108
|
+
DN = np.where(DN == fill_value, np.nan, DN)
|
109
|
+
DN = Raster(DN, geometry=grid)
|
110
|
+
|
111
|
+
data = DN * scale_factor
|
112
|
+
|
113
|
+
if apply_cloud_mask:
|
114
|
+
if cloud_mask is None:
|
115
|
+
cloud_mask = self.get_cloud_mask(target_shape=DN.shape)
|
116
|
+
|
117
|
+
data = rasters.where(cloud_mask, np.nan, data)
|
118
|
+
|
119
|
+
if geometry is not None:
|
120
|
+
data = data.to_geometry(geometry, resampling=resampling)
|
121
|
+
|
122
|
+
return data
|
123
|
+
|
124
|
+
@property
|
125
|
+
def geometry_M(self) -> RasterGrid:
|
126
|
+
return generate_modland_grid(*self.hv, 1200)
|
127
|
+
|
128
|
+
@property
|
129
|
+
def geometry_I(self) -> RasterGrid:
|
130
|
+
return generate_modland_grid(*self.hv, 2400)
|
131
|
+
|
132
|
+
def geometry(self, band: str) -> RasterGrid:
|
133
|
+
try:
|
134
|
+
band_letter = band[0]
|
135
|
+
except Exception as e:
|
136
|
+
raise ValueError(f"invalid band: {band}")
|
137
|
+
|
138
|
+
if band_letter == "I":
|
139
|
+
return self.geometry_I
|
140
|
+
elif band_letter == "M":
|
141
|
+
return self.geometry_M
|
142
|
+
else:
|
143
|
+
raise ValueError(f"invalid band: {band}")
|
144
|
+
|
145
|
+
def get_sensor_zenith_M(
|
146
|
+
self,
|
147
|
+
geometry: RasterGeometry = None,
|
148
|
+
save_data: bool = False,
|
149
|
+
save_preview: bool = False,
|
150
|
+
product_filename: str = None) -> Raster:
|
151
|
+
if product_filename is None:
|
152
|
+
product_filename = self.product_filename(f"sensor_zenith_M")
|
153
|
+
|
154
|
+
image = None
|
155
|
+
|
156
|
+
if product_filename is not None and exists(product_filename):
|
157
|
+
try:
|
158
|
+
logger.info(f"loading VIIRS sensor zenith: {cl.file(product_filename)}")
|
159
|
+
image = Raster.open(product_filename)
|
160
|
+
except Exception as e:
|
161
|
+
logger.exception(e)
|
162
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
163
|
+
remove(product_filename)
|
164
|
+
image = None
|
165
|
+
|
166
|
+
if image is None:
|
167
|
+
image = self.dataset(
|
168
|
+
self.filename,
|
169
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorZenith_1",
|
170
|
+
0.01,
|
171
|
+
cloud_mask=None,
|
172
|
+
apply_cloud_mask=False
|
173
|
+
)
|
174
|
+
|
175
|
+
if np.all(np.isnan(image)):
|
176
|
+
raise ValueError("blank sensor zenith image")
|
177
|
+
|
178
|
+
if save_data and not exists(product_filename):
|
179
|
+
logger.info(f"writing VIIRS M-band sensor zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
|
180
|
+
image.to_geotiff(product_filename)
|
181
|
+
|
182
|
+
if save_preview:
|
183
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
184
|
+
|
185
|
+
if geometry is not None:
|
186
|
+
image = image.to_geometry(geometry)
|
187
|
+
|
188
|
+
return image
|
189
|
+
|
190
|
+
sensor_zenith_M = property(get_sensor_zenith_M)
|
191
|
+
|
192
|
+
def get_sensor_zenith_I(
|
193
|
+
self,
|
194
|
+
geometry: RasterGeometry = None,
|
195
|
+
save_data: bool = False,
|
196
|
+
save_preview: bool = False,
|
197
|
+
product_filename: str = None) -> Raster:
|
198
|
+
if product_filename is None:
|
199
|
+
product_filename = self.product_filename(f"sensor_zenith_I")
|
200
|
+
|
201
|
+
image = None
|
202
|
+
|
203
|
+
if product_filename is not None and exists(product_filename):
|
204
|
+
try:
|
205
|
+
logger.info(f"loading VIIRS I-band sensor zenith: {cl.file(product_filename)}")
|
206
|
+
image = Raster.open(product_filename)
|
207
|
+
except Exception as e:
|
208
|
+
logger.exception(e)
|
209
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
210
|
+
remove(product_filename)
|
211
|
+
image = None
|
212
|
+
|
213
|
+
if image is None:
|
214
|
+
h, v = self.hv
|
215
|
+
grid_I = generate_modland_grid(h, v, 2400)
|
216
|
+
|
217
|
+
image = self.dataset(
|
218
|
+
self.filename,
|
219
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorZenith_1",
|
220
|
+
0.01,
|
221
|
+
cloud_mask=None,
|
222
|
+
apply_cloud_mask=False,
|
223
|
+
geometry=grid_I,
|
224
|
+
resampling="cubic"
|
225
|
+
)
|
226
|
+
|
227
|
+
if np.all(np.isnan(image)):
|
228
|
+
raise ValueError("blank sensor zenith image")
|
229
|
+
|
230
|
+
if save_data and not exists(product_filename):
|
231
|
+
logger.info(f"writing VIIRS sensor zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
|
232
|
+
image.to_geotiff(product_filename)
|
233
|
+
|
234
|
+
if save_preview:
|
235
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
236
|
+
|
237
|
+
if geometry is not None:
|
238
|
+
image = image.to_geometry(geometry)
|
239
|
+
|
240
|
+
return image
|
241
|
+
|
242
|
+
sensor_zenith_I = property(get_sensor_zenith_I)
|
243
|
+
|
244
|
+
def sensor_zenith(
|
245
|
+
self,
|
246
|
+
band: str,
|
247
|
+
geometry: RasterGeometry = None,
|
248
|
+
save_data: bool = False,
|
249
|
+
save_preview: bool = False,
|
250
|
+
product_filename: str = None) -> Raster:
|
251
|
+
try:
|
252
|
+
band_letter = band[0]
|
253
|
+
except Exception as e:
|
254
|
+
raise ValueError(f"invalid band: {band}")
|
255
|
+
|
256
|
+
if band_letter == "I":
|
257
|
+
return self.get_sensor_zenith_I(
|
258
|
+
geometry=geometry,
|
259
|
+
save_data=save_data,
|
260
|
+
save_preview=save_preview,
|
261
|
+
product_filename=product_filename
|
262
|
+
)
|
263
|
+
elif band_letter == "M":
|
264
|
+
return self.get_sensor_zenith_M(
|
265
|
+
geometry=geometry,
|
266
|
+
save_data=save_data,
|
267
|
+
save_preview=save_preview,
|
268
|
+
product_filename=product_filename
|
269
|
+
)
|
270
|
+
else:
|
271
|
+
raise ValueError(f"invalid band: {band}")
|
272
|
+
|
273
|
+
def get_sensor_azimuth_M(
|
274
|
+
self,
|
275
|
+
geometry: RasterGeometry = None,
|
276
|
+
save_data: bool = False,
|
277
|
+
save_preview: bool = False,
|
278
|
+
product_filename: str = None) -> Raster:
|
279
|
+
if product_filename is None:
|
280
|
+
product_filename = self.product_filename(f"sensor_azimuth_M")
|
281
|
+
|
282
|
+
image = None
|
283
|
+
|
284
|
+
if product_filename is not None and exists(product_filename):
|
285
|
+
try:
|
286
|
+
logger.info(f"loading VIIRS M-band sensor azimuth: {cl.file(product_filename)}")
|
287
|
+
image = Raster.open(product_filename)
|
288
|
+
except Exception as e:
|
289
|
+
logger.exception(e)
|
290
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
291
|
+
remove(product_filename)
|
292
|
+
image = None
|
293
|
+
|
294
|
+
if image is None:
|
295
|
+
image = self.dataset(
|
296
|
+
self.filename,
|
297
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorAzimuth_1",
|
298
|
+
0.01,
|
299
|
+
cloud_mask=None,
|
300
|
+
apply_cloud_mask=False
|
301
|
+
)
|
302
|
+
|
303
|
+
if np.all(np.isnan(image)):
|
304
|
+
raise ValueError("blank sensor azimuth image")
|
305
|
+
|
306
|
+
if save_data and not exists(product_filename):
|
307
|
+
logger.info(f"writing VIIRS sensor azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
|
308
|
+
image.to_geotiff(product_filename)
|
309
|
+
|
310
|
+
if save_preview:
|
311
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
312
|
+
|
313
|
+
if geometry is not None:
|
314
|
+
image = image.to_geometry(geometry)
|
315
|
+
|
316
|
+
return image
|
317
|
+
|
318
|
+
sensor_azimuth_M = property(get_sensor_azimuth_M)
|
319
|
+
|
320
|
+
def get_sensor_azimuth_I(
|
321
|
+
self,
|
322
|
+
geometry: RasterGeometry = None,
|
323
|
+
save_data: bool = False,
|
324
|
+
save_preview: bool = False,
|
325
|
+
product_filename: str = None) -> Raster:
|
326
|
+
if product_filename is None:
|
327
|
+
product_filename = self.product_filename(f"sensor_azimuth_I")
|
328
|
+
|
329
|
+
image = None
|
330
|
+
|
331
|
+
if product_filename is not None and exists(product_filename):
|
332
|
+
try:
|
333
|
+
logger.info(f"loading VIIRS I-band sensor azimuth: {cl.file(product_filename)}")
|
334
|
+
image = Raster.open(product_filename)
|
335
|
+
except Exception as e:
|
336
|
+
logger.exception(e)
|
337
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
338
|
+
remove(product_filename)
|
339
|
+
image = None
|
340
|
+
|
341
|
+
if image is None:
|
342
|
+
h, v = self.hv
|
343
|
+
grid_I = generate_modland_grid(h, v, 2400)
|
344
|
+
|
345
|
+
image = self.dataset(
|
346
|
+
self.filename,
|
347
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SensorAzimuth_1",
|
348
|
+
0.01,
|
349
|
+
cloud_mask=None,
|
350
|
+
apply_cloud_mask=False,
|
351
|
+
geometry=grid_I,
|
352
|
+
resampling="cubic"
|
353
|
+
)
|
354
|
+
|
355
|
+
if np.all(np.isnan(image)):
|
356
|
+
raise ValueError("blank sensor azimuth image")
|
357
|
+
|
358
|
+
if save_data and not exists(product_filename):
|
359
|
+
logger.info(f"writing VIIRS sensor azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
|
360
|
+
image.to_geotiff(product_filename)
|
361
|
+
|
362
|
+
if save_preview:
|
363
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
364
|
+
|
365
|
+
if geometry is not None:
|
366
|
+
image = image.to_geometry(geometry)
|
367
|
+
|
368
|
+
return image
|
369
|
+
|
370
|
+
sensor_azimuth_I = property(get_sensor_azimuth_I)
|
371
|
+
|
372
|
+
def sensor_azimuth(
|
373
|
+
self,
|
374
|
+
band: str,
|
375
|
+
geometry: RasterGeometry = None,
|
376
|
+
save_data: bool = False,
|
377
|
+
save_preview: bool = False,
|
378
|
+
product_filename: str = None) -> Raster:
|
379
|
+
try:
|
380
|
+
band_letter = band[0]
|
381
|
+
except Exception as e:
|
382
|
+
raise ValueError(f"invalid band: {band}")
|
383
|
+
|
384
|
+
if band_letter == "I":
|
385
|
+
return self.get_sensor_azimuth_I(
|
386
|
+
geometry=geometry,
|
387
|
+
save_data=save_data,
|
388
|
+
save_preview=save_preview,
|
389
|
+
product_filename=product_filename
|
390
|
+
)
|
391
|
+
elif band_letter == "M":
|
392
|
+
return self.get_sensor_azimuth_M(
|
393
|
+
geometry=geometry,
|
394
|
+
save_data=save_data,
|
395
|
+
save_preview=save_preview,
|
396
|
+
product_filename=product_filename
|
397
|
+
)
|
398
|
+
else:
|
399
|
+
raise ValueError(f"invalid band: {band}")
|
400
|
+
|
401
|
+
def get_solar_zenith_M(
|
402
|
+
self,
|
403
|
+
geometry: RasterGeometry = None,
|
404
|
+
save_data: bool = False,
|
405
|
+
save_preview: bool = False,
|
406
|
+
product_filename: str = None) -> Raster:
|
407
|
+
if product_filename is None:
|
408
|
+
product_filename = self.product_filename(f"solar_zenith_M")
|
409
|
+
|
410
|
+
image = None
|
411
|
+
|
412
|
+
if product_filename is not None and exists(product_filename):
|
413
|
+
try:
|
414
|
+
logger.info(f"loading VIIRS M-band solar zenith: {cl.file(product_filename)}")
|
415
|
+
image = Raster.open(product_filename)
|
416
|
+
except Exception as e:
|
417
|
+
logger.exception(e)
|
418
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
419
|
+
remove(product_filename)
|
420
|
+
image = None
|
421
|
+
|
422
|
+
if image is None:
|
423
|
+
image = self.dataset(
|
424
|
+
self.filename,
|
425
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarZenith_1",
|
426
|
+
0.01,
|
427
|
+
cloud_mask=None,
|
428
|
+
apply_cloud_mask=False
|
429
|
+
)
|
430
|
+
|
431
|
+
if np.all(np.isnan(image)):
|
432
|
+
raise ValueError("blank solar zenith image")
|
433
|
+
|
434
|
+
if save_data and not exists(product_filename):
|
435
|
+
logger.info(f"writing VIIRS solar zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
|
436
|
+
image.to_geotiff(product_filename)
|
437
|
+
|
438
|
+
if save_preview:
|
439
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
440
|
+
|
441
|
+
if geometry is not None:
|
442
|
+
image = image.to_geometry(geometry)
|
443
|
+
|
444
|
+
return image
|
445
|
+
|
446
|
+
solar_zenith_M = property(get_solar_zenith_M)
|
447
|
+
|
448
|
+
def get_solar_zenith_I(
|
449
|
+
self,
|
450
|
+
geometry: RasterGeometry = None,
|
451
|
+
save_data: bool = False,
|
452
|
+
save_preview: bool = False,
|
453
|
+
product_filename: str = None) -> Raster:
|
454
|
+
if product_filename is None:
|
455
|
+
product_filename = self.product_filename("solar_zenith_I")
|
456
|
+
|
457
|
+
image = None
|
458
|
+
|
459
|
+
if product_filename is not None and exists(product_filename):
|
460
|
+
try:
|
461
|
+
logger.info(f"loading VIIRS I-band solar zenith: {cl.file(product_filename)}")
|
462
|
+
image = Raster.open(product_filename)
|
463
|
+
except Exception as e:
|
464
|
+
logger.exception(e)
|
465
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
466
|
+
remove(product_filename)
|
467
|
+
image = None
|
468
|
+
|
469
|
+
if image is None:
|
470
|
+
h, v = self.hv
|
471
|
+
grid_I = generate_modland_grid(h, v, 2400)
|
472
|
+
|
473
|
+
image = self.dataset(
|
474
|
+
self.filename,
|
475
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarZenith_1",
|
476
|
+
0.01,
|
477
|
+
cloud_mask=None,
|
478
|
+
apply_cloud_mask=False,
|
479
|
+
geometry=grid_I,
|
480
|
+
resampling="cubic"
|
481
|
+
)
|
482
|
+
|
483
|
+
if np.all(np.isnan(image)):
|
484
|
+
raise ValueError("blank solar zenith image")
|
485
|
+
|
486
|
+
if save_data and not exists(product_filename):
|
487
|
+
logger.info(f"writing VIIRS solar zenith: {cl.file(product_filename)} {cl.val(image.shape)}")
|
488
|
+
|
489
|
+
image.to_geotiff(product_filename)
|
490
|
+
|
491
|
+
if save_preview:
|
492
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
493
|
+
|
494
|
+
if geometry is not None:
|
495
|
+
image = image.to_geometry(geometry)
|
496
|
+
|
497
|
+
return image
|
498
|
+
|
499
|
+
solar_zenith_I = property(get_solar_zenith_I)
|
500
|
+
|
501
|
+
def solar_zenith(
|
502
|
+
self,
|
503
|
+
band: str,
|
504
|
+
geometry: RasterGeometry = None,
|
505
|
+
save_data: bool = False,
|
506
|
+
save_preview: bool = False,
|
507
|
+
product_filename: str = None) -> Raster:
|
508
|
+
try:
|
509
|
+
band_letter = band[0]
|
510
|
+
except Exception as e:
|
511
|
+
raise ValueError(f"invalid band: {band}")
|
512
|
+
|
513
|
+
if band_letter == "I":
|
514
|
+
return self.get_solar_zenith_I(
|
515
|
+
geometry=geometry,
|
516
|
+
save_data=save_data,
|
517
|
+
save_preview=save_preview,
|
518
|
+
product_filename=product_filename
|
519
|
+
)
|
520
|
+
elif band_letter == "M":
|
521
|
+
return self.get_solar_zenith_M(
|
522
|
+
geometry=geometry,
|
523
|
+
save_data=save_data,
|
524
|
+
save_preview=save_preview,
|
525
|
+
product_filename=product_filename
|
526
|
+
)
|
527
|
+
else:
|
528
|
+
raise ValueError(f"invalid band: {band}")
|
529
|
+
|
530
|
+
def get_solar_azimuth_M(
|
531
|
+
self,
|
532
|
+
geometry: RasterGeometry = None,
|
533
|
+
save_data: bool = False,
|
534
|
+
save_preview: bool = False,
|
535
|
+
product_filename: str = None) -> Raster:
|
536
|
+
if product_filename is None:
|
537
|
+
product_filename = self.product_filename("solar_azimuth_M")
|
538
|
+
|
539
|
+
image = None
|
540
|
+
|
541
|
+
if product_filename is not None and exists(product_filename):
|
542
|
+
try:
|
543
|
+
logger.info(f"loading VIIRS M-band solar azimuth: {cl.file(product_filename)}")
|
544
|
+
image = Raster.open(product_filename)
|
545
|
+
except Exception as e:
|
546
|
+
logger.exception(e)
|
547
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
548
|
+
remove(product_filename)
|
549
|
+
image = None
|
550
|
+
|
551
|
+
if image is None:
|
552
|
+
image = self.dataset(
|
553
|
+
self.filename,
|
554
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarAzimuth_1",
|
555
|
+
0.01,
|
556
|
+
cloud_mask=None,
|
557
|
+
apply_cloud_mask=False
|
558
|
+
)
|
559
|
+
|
560
|
+
if np.all(np.isnan(image)):
|
561
|
+
raise ValueError("blank solar azimuth image")
|
562
|
+
|
563
|
+
if save_data and not exists(product_filename):
|
564
|
+
logger.info(f"writing VIIRS solar azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
|
565
|
+
image.to_geotiff(product_filename)
|
566
|
+
|
567
|
+
if save_preview:
|
568
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
569
|
+
|
570
|
+
if geometry is not None:
|
571
|
+
image = image.to_geometry(geometry)
|
572
|
+
|
573
|
+
return image
|
574
|
+
|
575
|
+
solar_azimuth_M = property(get_solar_azimuth_M)
|
576
|
+
|
577
|
+
def get_solar_azimuth_I(
|
578
|
+
self,
|
579
|
+
geometry: RasterGeometry = None,
|
580
|
+
save_data: bool = False,
|
581
|
+
save_preview: bool = False,
|
582
|
+
product_filename: str = None) -> Raster:
|
583
|
+
if product_filename is None:
|
584
|
+
product_filename = self.product_filename("solar_azimuth_I")
|
585
|
+
|
586
|
+
image = None
|
587
|
+
|
588
|
+
if product_filename is not None and exists(product_filename):
|
589
|
+
try:
|
590
|
+
logger.info(f"loading VIIRS I-band solar azimuth: {cl.file(product_filename)}")
|
591
|
+
image = Raster.open(product_filename)
|
592
|
+
except Exception as e:
|
593
|
+
logger.exception(e)
|
594
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
595
|
+
remove(product_filename)
|
596
|
+
image = None
|
597
|
+
|
598
|
+
if image is None:
|
599
|
+
h, v = self.hv
|
600
|
+
grid_I = generate_modland_grid(h, v, 2400)
|
601
|
+
|
602
|
+
image = self.dataset(
|
603
|
+
self.filename,
|
604
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SolarAzimuth_1",
|
605
|
+
0.01,
|
606
|
+
cloud_mask=None,
|
607
|
+
apply_cloud_mask=False,
|
608
|
+
geometry=grid_I,
|
609
|
+
resampling="cubic"
|
610
|
+
)
|
611
|
+
|
612
|
+
if np.all(np.isnan(image)):
|
613
|
+
raise ValueError("blank solar azimuth image")
|
614
|
+
|
615
|
+
if save_data and not exists(product_filename):
|
616
|
+
logger.info(f"writing VIIRS solar azimuth: {cl.file(product_filename)} {cl.val(image.shape)}")
|
617
|
+
image.to_geotiff(product_filename)
|
618
|
+
|
619
|
+
if save_preview:
|
620
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
621
|
+
|
622
|
+
if geometry is not None:
|
623
|
+
image = image.to_geometry(geometry)
|
624
|
+
|
625
|
+
return image
|
626
|
+
|
627
|
+
solar_azimuth_I = property(get_solar_azimuth_I)
|
628
|
+
|
629
|
+
def solar_azimuth(
|
630
|
+
self,
|
631
|
+
band: str,
|
632
|
+
geometry: RasterGeometry = None,
|
633
|
+
save_data: bool = False,
|
634
|
+
save_preview: bool = False,
|
635
|
+
product_filename: str = None) -> Raster:
|
636
|
+
try:
|
637
|
+
band_letter = band[0]
|
638
|
+
except Exception as e:
|
639
|
+
raise ValueError(f"invalid band: {band}")
|
640
|
+
|
641
|
+
if band_letter == "I":
|
642
|
+
return self.get_solar_azimuth_I(
|
643
|
+
geometry=geometry,
|
644
|
+
save_data=save_data,
|
645
|
+
save_preview=save_preview,
|
646
|
+
product_filename=product_filename
|
647
|
+
)
|
648
|
+
elif band_letter == "M":
|
649
|
+
return self.get_solar_azimuth_M(
|
650
|
+
geometry=geometry,
|
651
|
+
save_data=save_data,
|
652
|
+
save_preview=save_preview,
|
653
|
+
product_filename=product_filename
|
654
|
+
)
|
655
|
+
else:
|
656
|
+
raise ValueError(f"invalid band: {band}")
|
657
|
+
|
658
|
+
def get_M_band(
|
659
|
+
self,
|
660
|
+
band: int,
|
661
|
+
cloud_mask: Raster = None,
|
662
|
+
apply_cloud_mask: bool = True,
|
663
|
+
geometry: RasterGeometry = None,
|
664
|
+
save_data: bool = False,
|
665
|
+
save_preview: bool = False,
|
666
|
+
product_filename: str = None) -> Raster:
|
667
|
+
if product_filename is None:
|
668
|
+
product_filename = self.product_filename(f"M{band}")
|
669
|
+
|
670
|
+
image = None
|
671
|
+
|
672
|
+
if product_filename is not None and exists(product_filename):
|
673
|
+
try:
|
674
|
+
logger.info(f"loading VIIRS M-band {band} surface reflectance: {cl.file(product_filename)}")
|
675
|
+
image = Raster.open(product_filename)
|
676
|
+
except Exception as e:
|
677
|
+
logger.exception(e)
|
678
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
679
|
+
remove(product_filename)
|
680
|
+
image = None
|
681
|
+
|
682
|
+
if image is None:
|
683
|
+
image = self.dataset(
|
684
|
+
self.filename,
|
685
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_1km_2D/Data Fields/SurfReflect_M{int(band)}_1",
|
686
|
+
0.0001,
|
687
|
+
cloud_mask=cloud_mask,
|
688
|
+
apply_cloud_mask=apply_cloud_mask
|
689
|
+
)
|
690
|
+
|
691
|
+
if save_data and not exists(product_filename):
|
692
|
+
logger.info(f"writing VIIRS M{band}: {cl.file(product_filename)}")
|
693
|
+
image.to_geotiff(product_filename)
|
694
|
+
|
695
|
+
if save_preview:
|
696
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
697
|
+
|
698
|
+
if geometry is not None:
|
699
|
+
image = image.to_geometry(geometry)
|
700
|
+
|
701
|
+
return image
|
702
|
+
|
703
|
+
def get_I_band(
|
704
|
+
self,
|
705
|
+
band: int,
|
706
|
+
cloud_mask: Raster = None,
|
707
|
+
apply_cloud_mask: bool = True,
|
708
|
+
geometry: RasterGeometry = None,
|
709
|
+
save_data: bool = False,
|
710
|
+
save_preview: bool = False,
|
711
|
+
product_filename: str = None) -> Raster:
|
712
|
+
if product_filename is None:
|
713
|
+
product_filename = self.product_filename(f"I{band}")
|
714
|
+
|
715
|
+
image = None
|
716
|
+
|
717
|
+
if product_filename is not None and exists(product_filename):
|
718
|
+
try:
|
719
|
+
logger.info(f"loading VIIRS I-band {band} surface reflectance: {cl.file(product_filename)}")
|
720
|
+
image = Raster.open(product_filename)
|
721
|
+
except Exception as e:
|
722
|
+
logger.exception(e)
|
723
|
+
logger.warning(f"removing corrupted file: {product_filename}")
|
724
|
+
remove(product_filename)
|
725
|
+
image = None
|
726
|
+
|
727
|
+
if image is None:
|
728
|
+
image = self.dataset(
|
729
|
+
self.filename,
|
730
|
+
f"HDFEOS/GRIDS/VIIRS_Grid_500m_2D/Data Fields/SurfReflect_I{int(band)}_1",
|
731
|
+
0.0001,
|
732
|
+
cloud_mask=cloud_mask,
|
733
|
+
apply_cloud_mask=apply_cloud_mask
|
734
|
+
)
|
735
|
+
|
736
|
+
if save_data and not exists(product_filename):
|
737
|
+
logger.info(f"writing VIIRS I{band}: {cl.file(product_filename)}")
|
738
|
+
image.to_geotiff(product_filename)
|
739
|
+
|
740
|
+
if save_preview:
|
741
|
+
image.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"), quality=20, remove_XML=True)
|
742
|
+
|
743
|
+
if geometry is not None:
|
744
|
+
image = image.to_geometry(geometry)
|
745
|
+
|
746
|
+
return image
|
747
|
+
|
748
|
+
def band(
|
749
|
+
self,
|
750
|
+
band: str,
|
751
|
+
cloud_mask: Raster = None,
|
752
|
+
apply_cloud_mask: bool = True,
|
753
|
+
geometry: RasterGeometry = None,
|
754
|
+
save_data: bool = False,
|
755
|
+
save_preview: bool = False,
|
756
|
+
product_filename: str = None) -> Raster:
|
757
|
+
try:
|
758
|
+
band_letter = band[0]
|
759
|
+
band_number = int(band[1:])
|
760
|
+
except Exception as e:
|
761
|
+
raise ValueError(f"invalid band: {band}")
|
762
|
+
|
763
|
+
if band_letter == "I":
|
764
|
+
return self.get_I_band(
|
765
|
+
band=band_number,
|
766
|
+
cloud_mask=cloud_mask,
|
767
|
+
apply_cloud_mask=apply_cloud_mask,
|
768
|
+
geometry=geometry,
|
769
|
+
save_data=save_data,
|
770
|
+
save_preview=save_preview,
|
771
|
+
product_filename=product_filename
|
772
|
+
)
|
773
|
+
elif band_letter == "M":
|
774
|
+
return self.get_M_band(
|
775
|
+
band=band_number,
|
776
|
+
cloud_mask=cloud_mask,
|
777
|
+
apply_cloud_mask=apply_cloud_mask,
|
778
|
+
geometry=geometry,
|
779
|
+
save_data=save_data,
|
780
|
+
save_preview=save_preview,
|
781
|
+
product_filename=product_filename
|
782
|
+
)
|
783
|
+
else:
|
784
|
+
raise ValueError(f"invalid band: {band}")
|
785
|
+
|
786
|
+
def get_red(
|
787
|
+
self,
|
788
|
+
cloud_mask: Raster = None,
|
789
|
+
apply_cloud_mask: bool = True,
|
790
|
+
geometry: RasterGeometry = None,
|
791
|
+
save_data: bool = False,
|
792
|
+
save_preview: bool = False,
|
793
|
+
product_filename: str = None) -> Raster:
|
794
|
+
return self.get_I_band(
|
795
|
+
band=1,
|
796
|
+
cloud_mask=cloud_mask,
|
797
|
+
apply_cloud_mask=apply_cloud_mask,
|
798
|
+
geometry=geometry,
|
799
|
+
save_data=save_data,
|
800
|
+
save_preview=save_preview,
|
801
|
+
product_filename=product_filename
|
802
|
+
)
|
803
|
+
|
804
|
+
red = property(get_red)
|
805
|
+
|
806
|
+
def get_NIR(
|
807
|
+
self,
|
808
|
+
cloud_mask: Raster = None,
|
809
|
+
apply_cloud_mask: bool = True,
|
810
|
+
geometry: RasterGeometry = None,
|
811
|
+
save_data: bool = False,
|
812
|
+
save_preview: bool = False,
|
813
|
+
product_filename: str = None) -> Raster:
|
814
|
+
return self.get_I_band(
|
815
|
+
band=2,
|
816
|
+
cloud_mask=cloud_mask,
|
817
|
+
apply_cloud_mask=apply_cloud_mask,
|
818
|
+
geometry=geometry,
|
819
|
+
save_data=save_data,
|
820
|
+
save_preview=save_preview,
|
821
|
+
product_filename=product_filename
|
822
|
+
)
|
823
|
+
|
824
|
+
NIR = property(get_NIR)
|
825
|
+
|
826
|
+
def get_NDVI(
|
827
|
+
self,
|
828
|
+
cloud_mask: Raster = None,
|
829
|
+
apply_cloud_mask: bool = True,
|
830
|
+
geometry: RasterGeometry = None,
|
831
|
+
save_data: bool = False,
|
832
|
+
save_preview: bool = False,
|
833
|
+
product_filename: str = None) -> Raster:
|
834
|
+
if product_filename is None:
|
835
|
+
product_filename = self.product_filename("NDVI")
|
836
|
+
|
837
|
+
if product_filename is not None and exists(product_filename):
|
838
|
+
logger.info(f"loading VIIRS NDVI: {cl.file(product_filename)}")
|
839
|
+
NDVI = Raster.open(product_filename)
|
840
|
+
else:
|
841
|
+
red = self.get_red(
|
842
|
+
cloud_mask=cloud_mask,
|
843
|
+
apply_cloud_mask=apply_cloud_mask,
|
844
|
+
geometry=geometry,
|
845
|
+
save_data=save_data,
|
846
|
+
save_preview=save_preview
|
847
|
+
)
|
848
|
+
|
849
|
+
NIR = self.get_NIR(
|
850
|
+
cloud_mask=cloud_mask,
|
851
|
+
apply_cloud_mask=apply_cloud_mask,
|
852
|
+
geometry=geometry,
|
853
|
+
save_data=save_data,
|
854
|
+
save_preview=save_preview
|
855
|
+
)
|
856
|
+
|
857
|
+
NDVI = np.clip((NIR - red) / (NIR + red), -1, 1)
|
858
|
+
|
859
|
+
if save_data and not exists(product_filename):
|
860
|
+
logger.info(f"writing VIIRS NDVI: {cl.file(product_filename)}")
|
861
|
+
NDVI.to_geotiff(product_filename)
|
862
|
+
|
863
|
+
if save_preview:
|
864
|
+
NDVI.percentilecut.to_geojpeg(product_filename.replace(".tif", ".jpeg"))
|
865
|
+
|
866
|
+
if geometry is not None:
|
867
|
+
NDVI = NDVI.to_geometry(geometry)
|
868
|
+
|
869
|
+
NDVI.cmap = NDVI_COLORMAP
|
870
|
+
|
871
|
+
return NDVI
|
872
|
+
|
873
|
+
NDVI = property(get_NDVI)
|
874
|
+
|
875
|
+
def get_albedo(
|
876
|
+
self,
|
877
|
+
cloud_mask: Raster = None,
|
878
|
+
apply_cloud_mask: bool = True,
|
879
|
+
geometry: RasterGeometry = None,
|
880
|
+
save_data: bool = False,
|
881
|
+
save_preview: bool = False,
|
882
|
+
product_filename: str = None) -> Raster:
|
883
|
+
if product_filename is None:
|
884
|
+
product_filename = self.product_filename("albedo")
|
885
|
+
|
886
|
+
if product_filename is not None and exists(product_filename):
|
887
|
+
logger.info(f"loading VIIRS albedo: {cl.file(product_filename)}")
|
888
|
+
albedo = Raster.open(product_filename)
|
889
|
+
else:
|
890
|
+
b1 = self.get_M_band(
|
891
|
+
1,
|
892
|
+
cloud_mask=cloud_mask,
|
893
|
+
apply_cloud_mask=apply_cloud_mask,
|
894
|
+
geometry=geometry,
|
895
|
+
save_data=save_data,
|
896
|
+
save_preview=save_preview
|
897
|
+
)
|
898
|
+
|
899
|
+
b2 = self.get_M_band(
|
900
|
+
2,
|
901
|
+
cloud_mask=cloud_mask,
|
902
|
+
apply_cloud_mask=apply_cloud_mask,
|
903
|
+
geometry=geometry,
|
904
|
+
save_data=save_data,
|
905
|
+
save_preview=save_preview
|
906
|
+
)
|
907
|
+
|
908
|
+
b3 = self.get_M_band(
|
909
|
+
3,
|
910
|
+
cloud_mask=cloud_mask,
|
911
|
+
apply_cloud_mask=apply_cloud_mask,
|
912
|
+
geometry=geometry,
|
913
|
+
save_data=save_data,
|
914
|
+
save_preview=save_preview
|
915
|
+
)
|
916
|
+
|
917
|
+
b4 = self.get_M_band(
|
918
|
+
4,
|
919
|
+
cloud_mask=cloud_mask,
|
920
|
+
apply_cloud_mask=apply_cloud_mask,
|
921
|
+
geometry=geometry,
|
922
|
+
save_data=save_data,
|
923
|
+
save_preview=save_preview
|
924
|
+
)
|
925
|
+
|
926
|
+
b5 = self.get_M_band(
|
927
|
+
5,
|
928
|
+
cloud_mask=cloud_mask,
|
929
|
+
apply_cloud_mask=apply_cloud_mask,
|
930
|
+
geometry=geometry,
|
931
|
+
save_data=save_data,
|
932
|
+
save_preview=save_preview
|
933
|
+
)
|
934
|
+
|
935
|
+
b7 = self.get_M_band(
|
936
|
+
7,
|
937
|
+
cloud_mask=cloud_mask,
|
938
|
+
apply_cloud_mask=apply_cloud_mask,
|
939
|
+
geometry=geometry,
|
940
|
+
save_data=save_data,
|
941
|
+
save_preview=save_preview
|
942
|
+
)
|
943
|
+
|
944
|
+
b8 = self.get_M_band(
|
945
|
+
8,
|
946
|
+
cloud_mask=cloud_mask,
|
947
|
+
apply_cloud_mask=apply_cloud_mask,
|
948
|
+
geometry=geometry,
|
949
|
+
save_data=save_data,
|
950
|
+
save_preview=save_preview
|
951
|
+
)
|
952
|
+
|
953
|
+
b10 = self.get_M_band(
|
954
|
+
10,
|
955
|
+
cloud_mask=cloud_mask,
|
956
|
+
apply_cloud_mask=apply_cloud_mask,
|
957
|
+
geometry=geometry,
|
958
|
+
save_data=save_data,
|
959
|
+
save_preview=save_preview
|
960
|
+
)
|
961
|
+
|
962
|
+
b11 = self.get_M_band(
|
963
|
+
11,
|
964
|
+
cloud_mask=cloud_mask,
|
965
|
+
apply_cloud_mask=apply_cloud_mask,
|
966
|
+
geometry=geometry,
|
967
|
+
save_data=save_data,
|
968
|
+
save_preview=save_preview
|
969
|
+
)
|
970
|
+
|
971
|
+
# https://lpdaac.usgs.gov/documents/194/VNP43_ATBD_V1.pdf
|
972
|
+
albedo = 0.2418 * b1 \
|
973
|
+
- 0.201 * b2 \
|
974
|
+
+ 0.2093 * b3 \
|
975
|
+
+ 0.1146 * b4 \
|
976
|
+
+ 0.1348 * b5 \
|
977
|
+
+ 0.2251 * b7 \
|
978
|
+
+ 0.1123 * b8 \
|
979
|
+
+ 0.0860 * b10 \
|
980
|
+
+ 0.0803 * b11 \
|
981
|
+
- 0.0131
|
982
|
+
|
983
|
+
albedo = np.clip(albedo, 0, 1)
|
984
|
+
|
985
|
+
if save_data and not exists(product_filename):
|
986
|
+
logger.info(f"writing VIIRS albedo: {cl.file(product_filename)}")
|
987
|
+
albedo.to_geotiff(product_filename)
|
988
|
+
|
989
|
+
if geometry is not None:
|
990
|
+
logger.info(f"projecting VIIRS albedo from {cl.val(albedo.geometry.cell_size)} to {cl.val(geometry.cell_size)}")
|
991
|
+
albedo = albedo.to_geometry(geometry)
|
992
|
+
|
993
|
+
albedo.cmap = ALBEDO_COLORMAP
|
994
|
+
|
995
|
+
return albedo
|
996
|
+
|
997
|
+
albedo = property(get_albedo)
|
998
|
+
|
999
|
+
|
1000
|
+
VIIRS_CONCEPT = "C2631841556-LPCLOUD"
|
1001
|
+
|
1002
|
+
def earliest_datetime(date_in: Union[date, str]) -> datetime:
|
1003
|
+
if isinstance(date_in, str):
|
1004
|
+
datetime_in = parser.parse(date_in)
|
1005
|
+
else:
|
1006
|
+
datetime_in = date_in
|
1007
|
+
|
1008
|
+
date_string = datetime_in.strftime("%Y-%m-%d")
|
1009
|
+
return parser.parse(f"{date_string}T00:00:00Z")
|
1010
|
+
|
1011
|
+
|
1012
|
+
def latest_datetime(date_in: Union[date, str]) -> datetime:
|
1013
|
+
if isinstance(date_in, str):
|
1014
|
+
datetime_in = parser.parse(date_in)
|
1015
|
+
else:
|
1016
|
+
datetime_in = date_in
|
1017
|
+
|
1018
|
+
date_string = datetime_in.strftime("%Y-%m-%d")
|
1019
|
+
return parser.parse(f"{date_string}T23:59:59Z")
|
1020
|
+
|
1021
|
+
|
1022
|
+
VIIRS_FILENAME_REGEX = re.compile("^VNP09GA\.[^.]+\.([^.]+)\.002\.\d+\.h5$")
|
1023
|
+
def modland_tile_from_filename(filename: str) -> str:
|
1024
|
+
match = VIIRS_FILENAME_REGEX.match(filename)
|
1025
|
+
if match is None:
|
1026
|
+
raise RuntimeError(f"Invalid filename found through VIIRS CMR search: {filename}")
|
1027
|
+
|
1028
|
+
return match.group(1)
|
1029
|
+
|
1030
|
+
|
1031
|
+
# TODO: Deduplicate between VIIRS and HLS
|
1032
|
+
def VIIRS_CMR_query(
|
1033
|
+
start_date: Union[date, str],
|
1034
|
+
end_date: Union[date, str],
|
1035
|
+
target_geometry: Point or Polygon or RasterGeometry = None,
|
1036
|
+
tile: str = None,
|
1037
|
+
) -> List[earthaccess.search.DataGranule]:
|
1038
|
+
"""function to search for VIIRS at tile in date range"""
|
1039
|
+
query = earthaccess.granule_query() \
|
1040
|
+
.concept_id(VIIRS_CONCEPT) \
|
1041
|
+
.temporal(earliest_datetime(start_date), latest_datetime(end_date))
|
1042
|
+
|
1043
|
+
if isinstance(target_geometry, Point):
|
1044
|
+
query = query.point(target_geometry.x, target_geometry.y)
|
1045
|
+
if isinstance(target_geometry, Polygon):
|
1046
|
+
ring = target_geometry.exterior
|
1047
|
+
if not ring.is_ccw:
|
1048
|
+
ring = ring.reverse()
|
1049
|
+
coordinates = ring.coords
|
1050
|
+
query = query.polygon(coordinates)
|
1051
|
+
if isinstance(target_geometry, RasterGeometry):
|
1052
|
+
ring = target_geometry.corner_polygon_latlon.exterior
|
1053
|
+
if not ring.is_ccw:
|
1054
|
+
ring = ring.reverse()
|
1055
|
+
coordinates = ring.coords
|
1056
|
+
query = query.polygon(coordinates)
|
1057
|
+
if tile is not None:
|
1058
|
+
query = query.readable_granule_name(f"*.{tile}.*")
|
1059
|
+
|
1060
|
+
granules: List[earthaccess.search.DataGranule]
|
1061
|
+
try:
|
1062
|
+
granules = query.get()
|
1063
|
+
except Exception as e:
|
1064
|
+
raise CMRServerUnreachable(e)
|
1065
|
+
granules = sorted(granules, key=lambda granule: granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"])
|
1066
|
+
|
1067
|
+
logger.info("Found the following granules for VIIRS 2 using the CMR search:")
|
1068
|
+
for granule in granules:
|
1069
|
+
logger.info(" " + cl.file(granule["meta"]["native-id"]))
|
1070
|
+
logger.info(f"Number of VIIRS 2 granules found using CMR search: {len(granules)}")
|
1071
|
+
|
1072
|
+
return granules
|
1073
|
+
|
1074
|
+
|
1075
|
+
class VNP09GA:
|
1076
|
+
DEFAULT_WORKING_DIRECTORY = "."
|
1077
|
+
DEFAULT_DOWNLOAD_DIRECTORY = "VNP09GA_download"
|
1078
|
+
DEFAULT_PRODUCTS_DIRECTORY = "VNP09GA_products"
|
1079
|
+
DEFAULT_MOSAIC_DIRECTORY = "VNP09GA_mosaics"
|
1080
|
+
DEFAULT_RESAMPLING = "nearest"
|
1081
|
+
|
1082
|
+
CLOUD_DATASET_NAME = "HDFEOS/GRIDS/VNP_Grid_1km_2D/Data Fields/SurfReflect_QF1_1"
|
1083
|
+
|
1084
|
+
def __init__(
|
1085
|
+
self,
|
1086
|
+
working_directory: str = None,
|
1087
|
+
download_directory: str = None,
|
1088
|
+
products_directory: str = None,
|
1089
|
+
mosaic_directory: str = None,
|
1090
|
+
resampling: str = None):
|
1091
|
+
|
1092
|
+
if resampling is None:
|
1093
|
+
resampling = self.DEFAULT_RESAMPLING
|
1094
|
+
|
1095
|
+
self.resampling = resampling
|
1096
|
+
|
1097
|
+
self._granules = pd.DataFrame({"date_UTC": {}, "tile": {}, "granule": {}})
|
1098
|
+
|
1099
|
+
if working_directory is None:
|
1100
|
+
working_directory = self.DEFAULT_WORKING_DIRECTORY
|
1101
|
+
|
1102
|
+
working_directory = abspath(expanduser(working_directory))
|
1103
|
+
|
1104
|
+
if download_directory is None:
|
1105
|
+
download_directory = join(working_directory, self.DEFAULT_DOWNLOAD_DIRECTORY)
|
1106
|
+
|
1107
|
+
download_directory = abspath(expanduser(download_directory))
|
1108
|
+
|
1109
|
+
if products_directory is None:
|
1110
|
+
products_directory = join(working_directory, self.DEFAULT_PRODUCTS_DIRECTORY)
|
1111
|
+
|
1112
|
+
products_directory = abspath(expanduser(products_directory))
|
1113
|
+
|
1114
|
+
if mosaic_directory is None:
|
1115
|
+
mosaic_directory = join(working_directory, self.DEFAULT_MOSAIC_DIRECTORY)
|
1116
|
+
|
1117
|
+
mosaic_directory = abspath(expanduser(mosaic_directory))
|
1118
|
+
|
1119
|
+
self.working_directory = working_directory
|
1120
|
+
self.download_directory = download_directory
|
1121
|
+
self.products_directory = products_directory
|
1122
|
+
self.mosaic_directory = mosaic_directory
|
1123
|
+
|
1124
|
+
self.auth = VIIRS_CMR_login()
|
1125
|
+
|
1126
|
+
def add_granules(self, granules: List[earthaccess.search.DataGranule]):
|
1127
|
+
data = pd.DataFrame([
|
1128
|
+
{
|
1129
|
+
"date_UTC": get_date(granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"]),
|
1130
|
+
"tile": modland_tile_from_filename(Path(granule.data_links()[0]).name),
|
1131
|
+
"granule": granule,
|
1132
|
+
}
|
1133
|
+
for granule in granules
|
1134
|
+
])
|
1135
|
+
|
1136
|
+
self._granules = pd.concat([self._granules, data]).drop_duplicates(subset=["date_UTC", "tile"])
|
1137
|
+
|
1138
|
+
def download_granules(self, granules: List[earthaccess.search.DataGranule]) -> List[str]:
|
1139
|
+
# Check if any of the granules have already been downloaded, and if so record the file path for that granule.
|
1140
|
+
# Save the granules that haven't been downloaded to download them later.
|
1141
|
+
granules_to_download = []
|
1142
|
+
output_paths = []
|
1143
|
+
for granule in granules:
|
1144
|
+
date_UTC = get_date(granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"])
|
1145
|
+
output_file_path = join(
|
1146
|
+
self.download_directory,
|
1147
|
+
"VNP09GA",
|
1148
|
+
f"{date_UTC:%Y.%m.%d}",
|
1149
|
+
Path(granule.data_links()[0]).name
|
1150
|
+
)
|
1151
|
+
if Path(output_file_path).exists():
|
1152
|
+
output_paths.append(output_file_path)
|
1153
|
+
else:
|
1154
|
+
granules_to_download.append(granule)
|
1155
|
+
|
1156
|
+
# Early exit
|
1157
|
+
if len(granules_to_download) == 0:
|
1158
|
+
logger.info("All VIIRS granules have already been downloaded")
|
1159
|
+
return output_paths
|
1160
|
+
|
1161
|
+
# Make sure to remove this before we return, so we use try..finally to avoid exceptions causing issues
|
1162
|
+
temporary_parent_directory = join(self.download_directory, "tmp")
|
1163
|
+
os.makedirs(temporary_parent_directory, exist_ok=True)
|
1164
|
+
temporary_download_directory = tempfile.mkdtemp(dir=temporary_parent_directory)
|
1165
|
+
|
1166
|
+
try:
|
1167
|
+
last_download_exception = None
|
1168
|
+
for _ in range(0, RETRIES):
|
1169
|
+
download_exception = None
|
1170
|
+
downloaded_granules = []
|
1171
|
+
|
1172
|
+
file_paths = earthaccess.download(granules_to_download, local_path=temporary_download_directory)
|
1173
|
+
|
1174
|
+
for (granule, download_file_path) in zip(granules_to_download, file_paths):
|
1175
|
+
if isinstance(download_file_path, Exception):
|
1176
|
+
if download_exception is None:
|
1177
|
+
download_exception = download_file_path
|
1178
|
+
continue
|
1179
|
+
date_UTC = get_date(granule["umm"]["TemporalExtent"]["RangeDateTime"]["BeginningDateTime"])
|
1180
|
+
|
1181
|
+
download_file_path = Path(download_file_path)
|
1182
|
+
output_file_path = join(
|
1183
|
+
self.download_directory,
|
1184
|
+
"VNP09GA",
|
1185
|
+
f"{date_UTC:%Y.%m.%d}",
|
1186
|
+
download_file_path.name
|
1187
|
+
)
|
1188
|
+
Path(output_file_path).parent.mkdir(parents=True, exist_ok=True)
|
1189
|
+
download_file_path.rename(output_file_path)
|
1190
|
+
|
1191
|
+
output_paths.append(output_file_path)
|
1192
|
+
downloaded_granules.append(granule)
|
1193
|
+
|
1194
|
+
if download_exception is not None:
|
1195
|
+
last_download_exception = download_exception
|
1196
|
+
for granule in downloaded_granules:
|
1197
|
+
granules_to_download.remove(granule)
|
1198
|
+
logger.warning("Encountered an exception while downloading VIIRS files:", exc_info=download_exception)
|
1199
|
+
logger.info(f"Retrying the VIIRS download with the remaining {len(granules_to_download)} granules.")
|
1200
|
+
else:
|
1201
|
+
granules_to_download = []
|
1202
|
+
break
|
1203
|
+
|
1204
|
+
if len(granules_to_download) > 0:
|
1205
|
+
raise DownloadFailed("Error when downloading VIIRS files") from last_download_exception
|
1206
|
+
finally:
|
1207
|
+
Path(temporary_download_directory).rmdir()
|
1208
|
+
|
1209
|
+
return output_paths
|
1210
|
+
|
1211
|
+
def prefetch_VNP09GA(
|
1212
|
+
self,
|
1213
|
+
start_date: Union[date, str],
|
1214
|
+
end_date: Union[date, str],
|
1215
|
+
geometry: Point or Polygon or RasterGeometry = None):
|
1216
|
+
# Fetch list of granules to download
|
1217
|
+
granules = VIIRS_CMR_query(
|
1218
|
+
start_date,
|
1219
|
+
end_date,
|
1220
|
+
geometry,
|
1221
|
+
)
|
1222
|
+
|
1223
|
+
self.add_granules(granules)
|
1224
|
+
|
1225
|
+
self.download_granules(granules)
|
1226
|
+
|
1227
|
+
def search(
|
1228
|
+
self,
|
1229
|
+
date_UTC: date,
|
1230
|
+
tile: str) -> Union[earthaccess.search.DataGranule, None]:
|
1231
|
+
if "date_UTC" not in self._granules.columns:
|
1232
|
+
raise ValueError(f"date_UTC column not in granules table")
|
1233
|
+
|
1234
|
+
subset = self._granules[(self._granules.date_UTC == date_UTC) & (self._granules.tile == tile)]
|
1235
|
+
if len(subset) > 0:
|
1236
|
+
return subset.iloc[0].granule
|
1237
|
+
|
1238
|
+
granules = VIIRS_CMR_query(
|
1239
|
+
start_date=date_UTC,
|
1240
|
+
end_date=date_UTC,
|
1241
|
+
tile=tile,
|
1242
|
+
)
|
1243
|
+
|
1244
|
+
if len(granules) == 0:
|
1245
|
+
return None
|
1246
|
+
|
1247
|
+
if len(granules) > 0:
|
1248
|
+
logger.warning("Found more VIIRS granules than expected")
|
1249
|
+
|
1250
|
+
self.add_granules(granules)
|
1251
|
+
|
1252
|
+
return granules[0]
|
1253
|
+
|
1254
|
+
def granule(
|
1255
|
+
self,
|
1256
|
+
date_UTC: date,
|
1257
|
+
tile: str) -> VNP09GAGranule:
|
1258
|
+
if isinstance(date_UTC, str):
|
1259
|
+
date_UTC = parser.parse(date_UTC).date()
|
1260
|
+
|
1261
|
+
logger.info(f"searching VNP09GA tile {tile} date {date_UTC}")
|
1262
|
+
granule = self.search(
|
1263
|
+
date_UTC=date_UTC,
|
1264
|
+
tile=tile
|
1265
|
+
)
|
1266
|
+
|
1267
|
+
if granule is None:
|
1268
|
+
raise VIIRSUnavailableError(f"VNP09GA URL not available at tile {tile} on date {date_UTC}")
|
1269
|
+
|
1270
|
+
output_path = self.download_granules([granule])[0]
|
1271
|
+
|
1272
|
+
output_granule = VNP09GAGranule(
|
1273
|
+
filename=output_path,
|
1274
|
+
products_directory=self.products_directory
|
1275
|
+
)
|
1276
|
+
|
1277
|
+
return output_granule
|