goesgcp 2.1.1__py3-none-any.whl → 3.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- goesgcp/main.py +171 -122
- {goesgcp-2.1.1.dist-info → goesgcp-3.0.1.dist-info}/METADATA +38 -5
- goesgcp-3.0.1.dist-info/RECORD +8 -0
- {goesgcp-2.1.1.dist-info → goesgcp-3.0.1.dist-info}/WHEEL +1 -1
- goesgcp-2.1.1.dist-info/RECORD +0 -8
- {goesgcp-2.1.1.dist-info → goesgcp-3.0.1.dist-info}/entry_points.txt +0 -0
- {goesgcp-2.1.1.dist-info → goesgcp-3.0.1.dist-info/licenses}/LICENSE +0 -0
- {goesgcp-2.1.1.dist-info → goesgcp-3.0.1.dist-info}/top_level.txt +0 -0
goesgcp/main.py
CHANGED
|
@@ -13,7 +13,8 @@ from google.cloud import storage
|
|
|
13
13
|
from datetime import datetime, timedelta, timezone
|
|
14
14
|
from pyproj import CRS, Transformer
|
|
15
15
|
from google.api_core.exceptions import GoogleAPIError
|
|
16
|
-
|
|
16
|
+
import netCDF4
|
|
17
|
+
import numpy as np
|
|
17
18
|
import warnings
|
|
18
19
|
warnings.filterwarnings('ignore')
|
|
19
20
|
|
|
@@ -34,7 +35,7 @@ def get_directory_prefix(year, julian_day, hour):
|
|
|
34
35
|
return f"{year}/{julian_day}/{str(hour).zfill(2)}/"
|
|
35
36
|
|
|
36
37
|
|
|
37
|
-
def get_files_period(connection, bucket_name, base_prefix, pattern,
|
|
38
|
+
def get_files_period(connection, bucket_name, base_prefix, pattern,
|
|
38
39
|
start, end, bt_hour=[], bt_min=[], freq=None):
|
|
39
40
|
"""
|
|
40
41
|
Fetches files from a GCP bucket within a specified time period and returns them as a DataFrame.
|
|
@@ -87,7 +88,7 @@ def get_files_period(connection, bucket_name, base_prefix, pattern,
|
|
|
87
88
|
print("No files found matching the pattern and time range.")
|
|
88
89
|
print(prefix)
|
|
89
90
|
sys.exit(1)
|
|
90
|
-
|
|
91
|
+
|
|
91
92
|
# Transform file_name to datetime
|
|
92
93
|
df['last_modified'] = pd.to_datetime(df['file_name'].str.extract(r'(\d{4}\d{3}\d{2}\d{2})').squeeze(), format='%Y%j%H%M')
|
|
93
94
|
|
|
@@ -161,7 +162,24 @@ def crop_reproject(args):
|
|
|
161
162
|
Crops and reprojects a GOES-16 file to EPSG:4326.
|
|
162
163
|
"""
|
|
163
164
|
|
|
164
|
-
file, output, var_name, lat_min, lat_max, lon_min, lon_max, resolution, save_format
|
|
165
|
+
file, output, var_name, lat_min, lat_max, lon_min, lon_max, resolution, save_format, \
|
|
166
|
+
more_info, file_pattern, classic_format, remap, method = args
|
|
167
|
+
|
|
168
|
+
if more_info:
|
|
169
|
+
# Open file using netCDF4
|
|
170
|
+
ds_s = xr.open_dataset(file, engine="netcdf4", decode_cf=False)
|
|
171
|
+
if var_name is None:
|
|
172
|
+
var_names = [var for var in ds_s.data_vars if len(ds_s[var].dims) == 2]
|
|
173
|
+
var_names = [var for var in var_names if 'DQF' not in var]
|
|
174
|
+
else:
|
|
175
|
+
var_names = [var_name]
|
|
176
|
+
scale_factors = [ds_s[var].attrs["scale_factor"] for var in var_names]
|
|
177
|
+
add_offsets = [ds_s[var].attrs["add_offset"] for var in var_names]
|
|
178
|
+
fill_values = [ds_s[var].attrs["_FillValue"] for var in var_names]
|
|
179
|
+
units = [ds_s[var].attrs["units"] for var in var_names]
|
|
180
|
+
sat_lat = ds_s["goes_imager_projection"].attrs["latitude_of_projection_origin"]
|
|
181
|
+
sat_lon = ds_s["goes_imager_projection"].attrs["longitude_of_projection_origin"]
|
|
182
|
+
ds_s.close()
|
|
165
183
|
|
|
166
184
|
# Open the file
|
|
167
185
|
ds = xr.open_dataset(file, engine="netcdf4")
|
|
@@ -169,8 +187,6 @@ def crop_reproject(args):
|
|
|
169
187
|
if var_name is None:
|
|
170
188
|
# Get all variables are 2D
|
|
171
189
|
var_names = [var for var in ds.data_vars if len(ds[var].dims) == 2]
|
|
172
|
-
|
|
173
|
-
# Remove DQF variables
|
|
174
190
|
var_names = [var for var in var_names if 'DQF' not in var]
|
|
175
191
|
else:
|
|
176
192
|
var_names = [var_name]
|
|
@@ -188,118 +204,151 @@ def crop_reproject(args):
|
|
|
188
204
|
crs = CRS.from_cf(ds["goes_imager_projection"].attrs)
|
|
189
205
|
ds = ds.rio.write_crs(crs)
|
|
190
206
|
|
|
191
|
-
#
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
# Calculate the margin
|
|
196
|
-
margin_ratio = 0.40 # 40% margin
|
|
197
|
-
|
|
198
|
-
# Get the bounding box
|
|
199
|
-
min_x, min_y = transformer.transform(lat_min, lon_min)
|
|
200
|
-
max_x, max_y = transformer.transform(lat_max, lon_max)
|
|
201
|
-
|
|
202
|
-
# Calculate the range
|
|
203
|
-
x_range = abs(max_x - min_x)
|
|
204
|
-
y_range = abs(max_y - min_y)
|
|
205
|
-
|
|
206
|
-
margin_x = x_range * margin_ratio
|
|
207
|
-
margin_y = y_range * margin_ratio
|
|
208
|
-
|
|
209
|
-
# Expand the bounding box
|
|
210
|
-
min_x -= margin_x
|
|
211
|
-
max_x += margin_x
|
|
212
|
-
min_y -= margin_y
|
|
213
|
-
max_y += margin_y
|
|
214
|
-
|
|
215
|
-
# Select the region
|
|
216
|
-
if ds["y"].values[0] > ds["y"].values[-1]: # Eixo y decrescente
|
|
217
|
-
ds_ = ds.sel(x=slice(min_x, max_x), y=slice(max_y, min_y))
|
|
218
|
-
else: # Eixo y crescente
|
|
219
|
-
ds_ = ds.sel(x=slice(min_x, max_x), y=slice(min_y, max_y))
|
|
220
|
-
# Sort by y
|
|
221
|
-
if ds_["y"].values[0] > ds_["y"].values[-1]:
|
|
222
|
-
ds_ = ds_.sortby("y")
|
|
223
|
-
# Assign to ds
|
|
224
|
-
ds = ds_
|
|
225
|
-
except:
|
|
226
|
-
pass
|
|
207
|
+
# Create a transformer
|
|
208
|
+
transformer = Transformer.from_crs(CRS.from_epsg(4326), crs)
|
|
209
|
+
# Calculate the margin
|
|
210
|
+
margin_ratio = 0.40 # 40% margin
|
|
227
211
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
212
|
+
# Get the bounding box
|
|
213
|
+
min_x, min_y = transformer.transform(lat_min, lon_min)
|
|
214
|
+
max_x, max_y = transformer.transform(lat_max, lon_max)
|
|
215
|
+
|
|
216
|
+
# Calculate the range
|
|
217
|
+
x_range = abs(max_x - min_x)
|
|
218
|
+
y_range = abs(max_y - min_y)
|
|
231
219
|
|
|
232
|
-
|
|
233
|
-
|
|
220
|
+
margin_x = x_range * margin_ratio
|
|
221
|
+
margin_y = y_range * margin_ratio
|
|
234
222
|
|
|
235
|
-
|
|
223
|
+
# Expand the bounding box
|
|
224
|
+
min_x -= margin_x
|
|
225
|
+
max_x += margin_x
|
|
226
|
+
min_y -= margin_y
|
|
227
|
+
max_y += margin_y
|
|
228
|
+
|
|
229
|
+
# Sort the values
|
|
230
|
+
y_min, y_max = sorted([min_y, max_y], reverse=ds["y"].values[0] > ds["y"].values[-1])
|
|
231
|
+
|
|
232
|
+
# Crop the dataset based on the bounding box
|
|
233
|
+
ds = ds.sel(x=slice(min_x, max_x), y=slice(y_min, y_max))
|
|
234
|
+
|
|
235
|
+
# Sort the values
|
|
236
|
+
if ds["y"].values[0] > ds["y"].values[-1]:
|
|
237
|
+
ds = ds.sortby("y")
|
|
238
|
+
|
|
239
|
+
# Reproject the dataset in serial and convert values to short
|
|
240
|
+
ds = ds.rio.reproject("EPSG:4326", resolution=resolution)
|
|
241
|
+
|
|
242
|
+
# Rename lat/lon coordinates
|
|
243
|
+
ds = ds.rename({"x": "lon", "y": "lat"})
|
|
244
|
+
|
|
245
|
+
# Check if remap is not a string
|
|
246
|
+
if type(remap) != str:
|
|
236
247
|
for var in var_names:
|
|
237
248
|
ds[var].attrs['resolution'] = "x={:.2f} y={:.2f} degree".format(resolution, resolution)
|
|
238
249
|
ds[var].attrs['comments'] = 'Cropped and reprojected to EPSG:4326 by goesgcp'
|
|
239
|
-
|
|
240
|
-
# Crop using lat/lon coordinates, in parallel
|
|
250
|
+
# Crop using lat/lon coordinates
|
|
241
251
|
ds = ds.rio.clip_box(minx=lon_min, miny=lat_min, maxx=lon_max, maxy=lat_max)
|
|
252
|
+
else:
|
|
253
|
+
# Add _FillValue to the variables
|
|
254
|
+
for var in var_names:
|
|
255
|
+
ds[var].attrs['_FillValue'] = float(fill_values[var_names.index(var)])
|
|
242
256
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
if save_format == 'by_date':
|
|
250
|
-
file_datetime = datetime.strptime(ds.time_coverage_start,
|
|
257
|
+
# Add global metadata comments
|
|
258
|
+
ds.attrs['comments'] = "Data processed by goesgcp, author: Helvecio B. L. Neto (helvecioblneto@gmail.com)"
|
|
259
|
+
|
|
260
|
+
# Get the file datetime
|
|
261
|
+
file_datetime = datetime.strptime(ds.time_coverage_start,
|
|
251
262
|
"%Y-%m-%dT%H:%M:%S.%fZ")
|
|
263
|
+
if save_format == 'by_date':
|
|
252
264
|
year = file_datetime.strftime("%Y")
|
|
253
265
|
month = file_datetime.strftime("%m")
|
|
254
266
|
day = file_datetime.strftime("%d")
|
|
255
|
-
output_directory = f"{output}{year}/{month}/
|
|
267
|
+
output_directory = f"{output}{year}/{month}/"
|
|
256
268
|
elif save_format == 'julian':
|
|
257
|
-
file_datetime = datetime.strptime(ds.time_coverage_start,
|
|
269
|
+
file_datetime = datetime.strptime(ds.time_coverage_start,
|
|
258
270
|
"%Y-%m-%dT%H:%M:%S.%fZ")
|
|
259
271
|
year = file_datetime.strftime("%Y")
|
|
260
272
|
julian_day = file_datetime.timetuple().tm_yday
|
|
261
273
|
output_directory = f"{output}{year}/{julian_day}/"
|
|
262
274
|
else:
|
|
263
275
|
output_directory = output
|
|
264
|
-
|
|
265
276
|
|
|
266
277
|
# Create the output directory
|
|
267
278
|
pathlib.Path(output_directory).mkdir(parents=True, exist_ok=True)
|
|
268
279
|
|
|
269
|
-
#
|
|
270
|
-
|
|
271
|
-
|
|
280
|
+
# Apply file pattern
|
|
281
|
+
if file_pattern is not None:
|
|
282
|
+
# Get the timestamp
|
|
283
|
+
file_datetime = datetime.strptime(ds.time_coverage_start,
|
|
284
|
+
"%Y-%m-%dT%H:%M:%S.%fZ")
|
|
285
|
+
file_pattern = file_datetime.strftime(file_pattern)
|
|
286
|
+
output_file = f"{output_directory}{file_pattern}.nc"
|
|
287
|
+
else:
|
|
288
|
+
output_file = f"{output_directory}{file.split('/')[-1]}"
|
|
272
289
|
|
|
273
|
-
#
|
|
290
|
+
# Write the file
|
|
291
|
+
if classic_format:
|
|
292
|
+
# Change the data type to round values to int16
|
|
293
|
+
ds.to_netcdf(output_file, mode='w', format='NETCDF3_CLASSIC', encoding={var: {'dtype': np.int16} for var in var_names
|
|
294
|
+
})
|
|
295
|
+
else:
|
|
296
|
+
ds.to_netcdf(output_file, mode='w', encoding={var: {'zlib': True} for var in var_names})
|
|
274
297
|
ds.close()
|
|
275
298
|
|
|
276
|
-
|
|
277
|
-
|
|
299
|
+
# Remap the file
|
|
300
|
+
if remap:
|
|
301
|
+
remap_file((remap, output_file, output, method))
|
|
302
|
+
|
|
303
|
+
if more_info:
|
|
304
|
+
with netCDF4.Dataset(output_file, 'r+') as ds:
|
|
305
|
+
# Clear old attributes
|
|
306
|
+
for var_name in var_names:
|
|
307
|
+
var = ds.variables[var_name]
|
|
308
|
+
for attr in var.ncattrs():
|
|
309
|
+
if attr == 'long_name' or attr == '_FillValue':
|
|
310
|
+
continue
|
|
311
|
+
var.delncattr(attr)
|
|
312
|
+
# Add new attributes
|
|
313
|
+
for var in range(len(var_names)):
|
|
314
|
+
ds[var_names[var]].setncattr('scale_factor', scale_factors[var])
|
|
315
|
+
ds[var_names[var]].setncattr('add_offset', add_offsets[var])
|
|
316
|
+
ds[var_names[var]].setncattr('missing_value', fill_values[var])
|
|
317
|
+
ds[var_names[var]].setncattr('units', np.float32(units[var]))
|
|
318
|
+
# Add variable satlat
|
|
319
|
+
ds.createDimension('satlat', 1)
|
|
320
|
+
ds.createVariable('satlat', 'f4', ('satlat',))
|
|
321
|
+
ds.variables['satlat'][:] = sat_lat
|
|
322
|
+
ds.variables['satlat'].long_name = 'Satellite Latitude'
|
|
323
|
+
ds.variables['satlat'].units = 'degrees_north'
|
|
324
|
+
ds.createDimension('satlon', 1)
|
|
325
|
+
ds.createVariable('satlon', 'f4', ('satlon',))
|
|
326
|
+
ds.variables['satlon'][:] = sat_lon
|
|
327
|
+
ds.variables['satlon'].long_name = 'Satellite Longitude'
|
|
328
|
+
ds.variables['satlon'].units = 'degrees_east'
|
|
329
|
+
ds.createDimension('julian_day', 1)
|
|
330
|
+
ds.createVariable('julian_day', 'i2', ('julian_day',))
|
|
331
|
+
ds.variables['julian_day'][:] = int(file_datetime.timetuple().tm_yday)
|
|
332
|
+
ds.variables['julian_day'].long_name = 'Julian day'
|
|
333
|
+
ds.variables['julian_day'].units = 'day'
|
|
334
|
+
time_of_day_char = netCDF4.stringtochar(np.array([str(file_datetime.strftime("%H%M"))], 'S4'))
|
|
335
|
+
# Add variable time_of_day
|
|
336
|
+
ds.createDimension('time_of_day', 4)
|
|
337
|
+
ds.createVariable('time_of_day', 'S1', ('time_of_day',))
|
|
338
|
+
ds.variables['time_of_day'][:] = time_of_day_char
|
|
339
|
+
ds.variables['time_of_day'].long_name = 'Time of day'
|
|
340
|
+
ds.variables['time_of_day'].units = 'hour and minute'
|
|
341
|
+
ds.variables['time_of_day'].comment = str(file_datetime.strftime("%H%M"))
|
|
278
342
|
|
|
279
343
|
|
|
280
344
|
def remap_file(args):
|
|
281
345
|
""" Remap the download file based on the input file. """
|
|
282
346
|
|
|
283
|
-
base_file, target_file,
|
|
347
|
+
base_file, target_file, output, method = args
|
|
284
348
|
|
|
285
|
-
#
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
if save_format == 'by_date':
|
|
289
|
-
file_datetime = datetime.strptime(base_ds.time_coverage_start,
|
|
290
|
-
"%Y-%m-%dT%H:%M:%S.%fZ")
|
|
291
|
-
year = file_datetime.strftime("%Y")
|
|
292
|
-
month = file_datetime.strftime("%m")
|
|
293
|
-
day = file_datetime.strftime("%d")
|
|
294
|
-
output_directory = f"{output}{year}/{month}/{day}/"
|
|
295
|
-
elif save_format == 'julian':
|
|
296
|
-
file_datetime = datetime.strptime(base_ds.time_coverage_start,
|
|
297
|
-
"%Y-%m-%dT%H:%M:%S.%fZ")
|
|
298
|
-
year = file_datetime.strftime("%Y")
|
|
299
|
-
julian_day = file_datetime.timetuple().tm_yday
|
|
300
|
-
output_directory = f"{output}{year}/{julian_day}/"
|
|
301
|
-
else:
|
|
302
|
-
output_directory = output
|
|
349
|
+
# Get output directory based on target_file
|
|
350
|
+
output_file = f"{output}{target_file.split('/')[-1]}"
|
|
351
|
+
output_directory = output_file.replace(target_file.split('/')[-1], "")
|
|
303
352
|
|
|
304
353
|
# Create the output directory
|
|
305
354
|
pathlib.Path(output_directory).mkdir(parents=True, exist_ok=True)
|
|
@@ -320,9 +369,6 @@ def remap_file(args):
|
|
|
320
369
|
print(f"Error remapping file {target_file}: {e}")
|
|
321
370
|
pass
|
|
322
371
|
|
|
323
|
-
# Close the files
|
|
324
|
-
base_ds.close()
|
|
325
|
-
|
|
326
372
|
# Delete the target file
|
|
327
373
|
pathlib.Path(target_file).unlink()
|
|
328
374
|
|
|
@@ -334,18 +380,17 @@ def process_file(args):
|
|
|
334
380
|
"""
|
|
335
381
|
Downloads and processes a GOES-16 file.
|
|
336
382
|
"""
|
|
337
|
-
|
|
383
|
+
|
|
338
384
|
bucket_name, blob_name, local_path, output_path, var_name, lat_min, lat_max, lon_min, lon_max, resolution, \
|
|
339
|
-
save_format, retries, remap, met = args
|
|
385
|
+
save_format, retries, remap, met, more_info, file_pattern, classic_format = args
|
|
340
386
|
|
|
387
|
+
# Download the file
|
|
341
388
|
attempt = 0
|
|
342
389
|
while attempt < retries:
|
|
343
390
|
try:
|
|
344
391
|
# Connect to the bucket
|
|
345
392
|
bucket = storage_client.bucket(bucket_name)
|
|
346
393
|
blob = bucket.blob(blob_name)
|
|
347
|
-
|
|
348
|
-
# Download the file
|
|
349
394
|
blob.download_to_filename(local_path, timeout=120)
|
|
350
395
|
break # Exit the loop if the download is successful
|
|
351
396
|
except (GoogleAPIError, Exception) as e: # Catch any exception
|
|
@@ -353,21 +398,20 @@ def process_file(args):
|
|
|
353
398
|
if attempt < retries:
|
|
354
399
|
time.sleep(2 ** attempt) # Backoff exponencial
|
|
355
400
|
else:
|
|
356
|
-
# Log the error to a file
|
|
357
401
|
with open('fail.log', 'a') as log_file:
|
|
358
402
|
log_file.write(f"Failed to download {blob_name} after {retries} attempts. Error: {e}\n")
|
|
359
|
-
|
|
360
403
|
# Crop the file
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
#
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
404
|
+
try:
|
|
405
|
+
crop_reproject((local_path, output_path,
|
|
406
|
+
var_name, lat_min, lat_max, lon_min, lon_max,
|
|
407
|
+
resolution, save_format,
|
|
408
|
+
more_info, file_pattern, classic_format, remap, met))
|
|
409
|
+
# Remove the local file
|
|
410
|
+
pathlib.Path(local_path).unlink()
|
|
411
|
+
except Exception as e:
|
|
412
|
+
with open('fail.log', 'a') as log_file:
|
|
413
|
+
log_file.write(f"Failed to process {blob_name}. Error: {e}\n")
|
|
414
|
+
pass
|
|
371
415
|
|
|
372
416
|
# Create connection
|
|
373
417
|
storage_client = storage.Client.create_anonymous_client()
|
|
@@ -377,7 +421,7 @@ def main():
|
|
|
377
421
|
|
|
378
422
|
epilog = """
|
|
379
423
|
Example usage:
|
|
380
|
-
|
|
424
|
+
|
|
381
425
|
- To download recent 3 files from the GOES-16 satellite for the ABI-L2-CMIPF product,
|
|
382
426
|
change resolution to 0.045, and crop the files between latitudes -35 and 5 and longitudes -80 and -30:
|
|
383
427
|
|
|
@@ -411,9 +455,9 @@ def main():
|
|
|
411
455
|
parser = argparse.ArgumentParser(description='Download and process GOES Satellite data files from GCP.',
|
|
412
456
|
epilog=epilog,
|
|
413
457
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
414
|
-
|
|
458
|
+
|
|
415
459
|
# Satellite and product settings
|
|
416
|
-
parser.add_argument('--satellite', type=str, default='goes-
|
|
460
|
+
parser.add_argument('--satellite', type=str, default='goes-19', choices=['goes-16', 'goes-18', 'goes-19'], help='Name of the satellite (e.g., goes16)')
|
|
417
461
|
parser.add_argument('--product', type=str, default='ABI-L2-CMIPF', help='Name of the satellite product', choices=product_names)
|
|
418
462
|
parser.add_argument('--var_name', type=str, default=None, help='Variable name to extract (e.g., CMI)')
|
|
419
463
|
parser.add_argument('--channel', type=int, default=13, help='Channel to use (e.g., 13)')
|
|
@@ -430,26 +474,26 @@ def main():
|
|
|
430
474
|
parser.add_argument('--bt_min', nargs=2, type=int, default=[0, 60], help='Filter data between these minutes (e.g., 0 60)')
|
|
431
475
|
|
|
432
476
|
# Geographic bounding box
|
|
433
|
-
parser.add_argument('--lat_min', type=float, default=-
|
|
434
|
-
parser.add_argument('--lat_max', type=float, default=
|
|
435
|
-
parser.add_argument('--lon_min', type=float, default=-
|
|
436
|
-
parser.add_argument('--lon_max', type=float, default
|
|
437
|
-
parser.add_argument('--resolution', type=float, default=0.
|
|
438
|
-
parser.add_argument('--output', type=str, default='output/', help='Path for saving output files')
|
|
477
|
+
parser.add_argument('--lat_min', type=float, default=-56, help='Minimum latitude of the bounding box')
|
|
478
|
+
parser.add_argument('--lat_max', type=float, default=35, help='Maximum latitude of the bounding box')
|
|
479
|
+
parser.add_argument('--lon_min', type=float, default=-116, help='Minimum longitude of the bounding box')
|
|
480
|
+
parser.add_argument('--lon_max', type=float, default=-25, help='Maximum longitude of the bounding box')
|
|
481
|
+
parser.add_argument('--resolution', type=float, default=0.03, help='Resolution of the output file')
|
|
482
|
+
parser.add_argument('--output', type=str, default='./output/', help='Path for saving output files')
|
|
439
483
|
|
|
440
484
|
# Remap
|
|
441
485
|
parser.add_argument('--remap', type=str, default=None, help='Give a input file to remap the output')
|
|
442
486
|
parser.add_argument('--method', type=str, default='remapnn', help='Remap method to use (e.g., remapnn)')
|
|
443
487
|
|
|
444
|
-
|
|
445
488
|
# Other settings
|
|
446
489
|
parser.add_argument('--parallel', type=lambda x: bool(strtobool(x)), default=True, help='Use parallel processing')
|
|
447
490
|
parser.add_argument('--processes', type=int, default=4, help='Number of processes for parallel execution')
|
|
448
491
|
parser.add_argument('--max_attempts', type=int, default=3, help='Number of attempts to download a file')
|
|
492
|
+
parser.add_argument('--info', type=lambda x: bool(strtobool(x)), default=False, help='Show information messages')
|
|
449
493
|
parser.add_argument('--save_format', type=str, default='flat', choices=['flat', 'by_date','julian'],
|
|
450
494
|
help="Save the files in a flat structure or by date")
|
|
451
|
-
|
|
452
|
-
|
|
495
|
+
parser.add_argument('--file_pattern', type=str, default=None, help='Pattern for the files')
|
|
496
|
+
parser.add_argument('--netcdf_classic', type=lambda x: bool(strtobool(x)), default=False, help='Save the files in netCDF classic format')
|
|
453
497
|
# Parse arguments
|
|
454
498
|
args = parser.parse_args()
|
|
455
499
|
|
|
@@ -480,7 +524,9 @@ def main():
|
|
|
480
524
|
save_format = args.save_format
|
|
481
525
|
remap = args.remap
|
|
482
526
|
method = args.method
|
|
483
|
-
|
|
527
|
+
more_info = args.info
|
|
528
|
+
file_pattern = args.file_pattern
|
|
529
|
+
classic_format = args.netcdf_classic
|
|
484
530
|
|
|
485
531
|
# Check mandatory arguments
|
|
486
532
|
if not args.recent and not (args.start and args.end):
|
|
@@ -532,12 +578,13 @@ def main():
|
|
|
532
578
|
loading_bar = tqdm.tqdm(total=len(files_list), ncols=100, position=0, leave=True,
|
|
533
579
|
bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} + \
|
|
534
580
|
[Elapsed:{elapsed} Remaining:<{remaining}]')
|
|
535
|
-
|
|
581
|
+
|
|
536
582
|
if parallel: # Run in parallel
|
|
537
583
|
# Create a list of tasks
|
|
538
|
-
tasks = [(bucket_name, file, f"tmp/{file.split('/')[-1]}", output_path, var_name,
|
|
584
|
+
tasks = [(bucket_name, file, f"tmp/{file.split('/')[-1]}", output_path, var_name,
|
|
539
585
|
lat_min, lat_max, lon_min, lon_max, resolution,
|
|
540
|
-
save_format, max_attempts, remap, method
|
|
586
|
+
save_format, max_attempts, remap, method,
|
|
587
|
+
more_info, file_pattern, classic_format) for file in files_list]
|
|
541
588
|
|
|
542
589
|
# Download files in parallel
|
|
543
590
|
with Pool(processes=args.processes) as pool:
|
|
@@ -549,10 +596,12 @@ def main():
|
|
|
549
596
|
local_path = f"tmp/{file.split('/')[-1]}"
|
|
550
597
|
process_file((bucket_name, file, local_path, output_path, var_name,
|
|
551
598
|
lat_min, lat_max, lon_min, lon_max, resolution,
|
|
552
|
-
save_format, max_attempts, remap, method
|
|
599
|
+
save_format, max_attempts, remap, method, more_info,
|
|
600
|
+
file_pattern, classic_format))
|
|
553
601
|
loading_bar.update(1)
|
|
554
602
|
loading_bar.close()
|
|
555
603
|
|
|
604
|
+
# Clean up the temporary directory
|
|
556
605
|
shutil.rmtree('tmp/')
|
|
557
606
|
|
|
558
607
|
if __name__ == '__main__':
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: goesgcp
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.1
|
|
4
4
|
Summary: A package to download and process GOES-16/17 data
|
|
5
5
|
Home-page: https://github.com/helvecioneto/goesgcp
|
|
6
6
|
Author: Helvecio B. L. Neto
|
|
@@ -19,11 +19,12 @@ Classifier: Topic :: Software Development
|
|
|
19
19
|
Classifier: Topic :: Utilities
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
License-File: LICENSE
|
|
22
|
-
Requires-Dist:
|
|
23
|
-
Requires-Dist: pyproj
|
|
22
|
+
Requires-Dist: pandas
|
|
24
23
|
Requires-Dist: xarray
|
|
24
|
+
Requires-Dist: google-cloud-storage
|
|
25
25
|
Requires-Dist: netcdf4
|
|
26
|
-
Requires-Dist:
|
|
26
|
+
Requires-Dist: tqdm
|
|
27
|
+
Requires-Dist: setuptools
|
|
27
28
|
Dynamic: author
|
|
28
29
|
Dynamic: author-email
|
|
29
30
|
Dynamic: classifier
|
|
@@ -31,6 +32,7 @@ Dynamic: description
|
|
|
31
32
|
Dynamic: description-content-type
|
|
32
33
|
Dynamic: home-page
|
|
33
34
|
Dynamic: license
|
|
35
|
+
Dynamic: license-file
|
|
34
36
|
Dynamic: requires-dist
|
|
35
37
|
Dynamic: summary
|
|
36
38
|
|
|
@@ -62,6 +64,37 @@ pip install goesgcp
|
|
|
62
64
|
```
|
|
63
65
|
|
|
64
66
|
|
|
67
|
+
Obs: If gdal is not installed, you can install it using the following command:
|
|
68
|
+
|
|
69
|
+
Linux:
|
|
70
|
+
```bash
|
|
71
|
+
sudo apt-get install gdal-bin
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Windows:
|
|
75
|
+
```bash
|
|
76
|
+
conda install -c conda-forge gdal
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
MacOS:
|
|
80
|
+
```bash
|
|
81
|
+
brew install gdal
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Or you can install the wheel file:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
python -m pip install gdal -f https://girder.github.io/large_image_wheels
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
and install other dependencies:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
pip install -r requirements.txt
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
65
98
|
## Usage
|
|
66
99
|
|
|
67
100
|
### Available Command-Line Arguments
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
|
|
2
|
+
goesgcp/main.py,sha256=hWM5w-e8RcFuwsk-XmBrh4n7Q-ISAZcAGOypxWyEtRM,25163
|
|
3
|
+
goesgcp-3.0.1.dist-info/licenses/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
|
|
4
|
+
goesgcp-3.0.1.dist-info/METADATA,sha256=S0DHKnLeP-_JzcBYZaF4EcxJIIbsb3mNxa4UY8cYB1E,7044
|
|
5
|
+
goesgcp-3.0.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
|
6
|
+
goesgcp-3.0.1.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
|
|
7
|
+
goesgcp-3.0.1.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
|
|
8
|
+
goesgcp-3.0.1.dist-info/RECORD,,
|
goesgcp-2.1.1.dist-info/RECORD
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
|
|
2
|
-
goesgcp/main.py,sha256=TgOT8bd5lQgGhFGLXjmluyFr1edgnPTeqI9VW0WwVoE,21557
|
|
3
|
-
goesgcp-2.1.1.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
|
|
4
|
-
goesgcp-2.1.1.dist-info/METADATA,sha256=mcSmEtQ6v3eFimsJrYqN4a9jhvXDW5FZFVlgrAmPOhY,6574
|
|
5
|
-
goesgcp-2.1.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
6
|
-
goesgcp-2.1.1.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
|
|
7
|
-
goesgcp-2.1.1.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
|
|
8
|
-
goesgcp-2.1.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|