goesgcp 2.1.0__tar.gz → 3.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: goesgcp
3
- Version: 2.1.0
3
+ Version: 3.0.1
4
4
  Summary: A package to download and process GOES-16/17 data
5
5
  Home-page: https://github.com/helvecioneto/goesgcp
6
6
  Author: Helvecio B. L. Neto
@@ -19,11 +19,12 @@ Classifier: Topic :: Software Development
19
19
  Classifier: Topic :: Utilities
20
20
  Description-Content-Type: text/markdown
21
21
  License-File: LICENSE
22
- Requires-Dist: google-cloud-storage
23
- Requires-Dist: pyproj
22
+ Requires-Dist: pandas
24
23
  Requires-Dist: xarray
24
+ Requires-Dist: google-cloud-storage
25
25
  Requires-Dist: netcdf4
26
- Requires-Dist: rioxarray
26
+ Requires-Dist: tqdm
27
+ Requires-Dist: setuptools
27
28
  Dynamic: author
28
29
  Dynamic: author-email
29
30
  Dynamic: classifier
@@ -31,6 +32,7 @@ Dynamic: description
31
32
  Dynamic: description-content-type
32
33
  Dynamic: home-page
33
34
  Dynamic: license
35
+ Dynamic: license-file
34
36
  Dynamic: requires-dist
35
37
  Dynamic: summary
36
38
 
@@ -62,6 +64,37 @@ pip install goesgcp
62
64
  ```
63
65
 
64
66
 
67
+ Obs: If gdal is not installed, you can install it using the following command:
68
+
69
+ Linux:
70
+ ```bash
71
+ sudo apt-get install gdal-bin
72
+ ```
73
+
74
+ Windows:
75
+ ```bash
76
+ conda install -c conda-forge gdal
77
+ ```
78
+
79
+ MacOS:
80
+ ```bash
81
+ brew install gdal
82
+ ```
83
+
84
+ Or you can install the wheel file:
85
+
86
+ ```bash
87
+ python -m pip install gdal -f https://girder.github.io/large_image_wheels
88
+ ```
89
+
90
+ and install other dependencies:
91
+
92
+ ```bash
93
+ pip install -r requirements.txt
94
+ ```
95
+
96
+
97
+
65
98
  ## Usage
66
99
 
67
100
  ### Available Command-Line Arguments
@@ -26,6 +26,37 @@ pip install goesgcp
26
26
  ```
27
27
 
28
28
 
29
+ Obs: If gdal is not installed, you can install it using the following command:
30
+
31
+ Linux:
32
+ ```bash
33
+ sudo apt-get install gdal-bin
34
+ ```
35
+
36
+ Windows:
37
+ ```bash
38
+ conda install -c conda-forge gdal
39
+ ```
40
+
41
+ MacOS:
42
+ ```bash
43
+ brew install gdal
44
+ ```
45
+
46
+ Or you can install the wheel file:
47
+
48
+ ```bash
49
+ python -m pip install gdal -f https://girder.github.io/large_image_wheels
50
+ ```
51
+
52
+ and install other dependencies:
53
+
54
+ ```bash
55
+ pip install -r requirements.txt
56
+ ```
57
+
58
+
59
+
29
60
  ## Usage
30
61
 
31
62
  ### Available Command-Line Arguments
@@ -13,7 +13,8 @@ from google.cloud import storage
13
13
  from datetime import datetime, timedelta, timezone
14
14
  from pyproj import CRS, Transformer
15
15
  from google.api_core.exceptions import GoogleAPIError
16
-
16
+ import netCDF4
17
+ import numpy as np
17
18
  import warnings
18
19
  warnings.filterwarnings('ignore')
19
20
 
@@ -34,7 +35,7 @@ def get_directory_prefix(year, julian_day, hour):
34
35
  return f"{year}/{julian_day}/{str(hour).zfill(2)}/"
35
36
 
36
37
 
37
- def get_files_period(connection, bucket_name, base_prefix, pattern,
38
+ def get_files_period(connection, bucket_name, base_prefix, pattern,
38
39
  start, end, bt_hour=[], bt_min=[], freq=None):
39
40
  """
40
41
  Fetches files from a GCP bucket within a specified time period and returns them as a DataFrame.
@@ -87,7 +88,7 @@ def get_files_period(connection, bucket_name, base_prefix, pattern,
87
88
  print("No files found matching the pattern and time range.")
88
89
  print(prefix)
89
90
  sys.exit(1)
90
-
91
+
91
92
  # Transform file_name to datetime
92
93
  df['last_modified'] = pd.to_datetime(df['file_name'].str.extract(r'(\d{4}\d{3}\d{2}\d{2})').squeeze(), format='%Y%j%H%M')
93
94
 
@@ -161,7 +162,24 @@ def crop_reproject(args):
161
162
  Crops and reprojects a GOES-16 file to EPSG:4326.
162
163
  """
163
164
 
164
- file, output, var_name, lat_min, lat_max, lon_min, lon_max, resolution, save_format = args
165
+ file, output, var_name, lat_min, lat_max, lon_min, lon_max, resolution, save_format, \
166
+ more_info, file_pattern, classic_format, remap, method = args
167
+
168
+ if more_info:
169
+ # Open file using netCDF4
170
+ ds_s = xr.open_dataset(file, engine="netcdf4", decode_cf=False)
171
+ if var_name is None:
172
+ var_names = [var for var in ds_s.data_vars if len(ds_s[var].dims) == 2]
173
+ var_names = [var for var in var_names if 'DQF' not in var]
174
+ else:
175
+ var_names = [var_name]
176
+ scale_factors = [ds_s[var].attrs["scale_factor"] for var in var_names]
177
+ add_offsets = [ds_s[var].attrs["add_offset"] for var in var_names]
178
+ fill_values = [ds_s[var].attrs["_FillValue"] for var in var_names]
179
+ units = [ds_s[var].attrs["units"] for var in var_names]
180
+ sat_lat = ds_s["goes_imager_projection"].attrs["latitude_of_projection_origin"]
181
+ sat_lon = ds_s["goes_imager_projection"].attrs["longitude_of_projection_origin"]
182
+ ds_s.close()
165
183
 
166
184
  # Open the file
167
185
  ds = xr.open_dataset(file, engine="netcdf4")
@@ -169,8 +187,6 @@ def crop_reproject(args):
169
187
  if var_name is None:
170
188
  # Get all variables are 2D
171
189
  var_names = [var for var in ds.data_vars if len(ds[var].dims) == 2]
172
-
173
- # Remove DQF variables
174
190
  var_names = [var for var in var_names if 'DQF' not in var]
175
191
  else:
176
192
  var_names = [var_name]
@@ -188,118 +204,151 @@ def crop_reproject(args):
188
204
  crs = CRS.from_cf(ds["goes_imager_projection"].attrs)
189
205
  ds = ds.rio.write_crs(crs)
190
206
 
191
- # Try to reduce the size of the dataset
192
- try:
193
- # Create a transformer
194
- transformer = Transformer.from_crs(CRS.from_epsg(4326), crs)
195
- # Calculate the margin
196
- margin_ratio = 0.40 # 40% margin
197
-
198
- # Get the bounding box
199
- min_x, min_y = transformer.transform(lat_min, lon_min)
200
- max_x, max_y = transformer.transform(lat_max, lon_max)
201
-
202
- # Calculate the range
203
- x_range = abs(max_x - min_x)
204
- y_range = abs(max_y - min_y)
205
-
206
- margin_x = x_range * margin_ratio
207
- margin_y = y_range * margin_ratio
208
-
209
- # Expand the bounding box
210
- min_x -= margin_x
211
- max_x += margin_x
212
- min_y -= margin_y
213
- max_y += margin_y
214
-
215
- # Select the region
216
- if ds["y"].values[0] > ds["y"].values[-1]: # Eixo y decrescente
217
- ds_ = ds.sel(x=slice(min_x, max_x), y=slice(max_y, min_y))
218
- else: # Eixo y crescente
219
- ds_ = ds.sel(x=slice(min_x, max_x), y=slice(min_y, max_y))
220
- # Sort by y
221
- if ds_["y"].values[0] > ds_["y"].values[-1]:
222
- ds_ = ds_.sortby("y")
223
- # Assign to ds
224
- ds = ds_
225
- except:
226
- pass
207
+ # Create a transformer
208
+ transformer = Transformer.from_crs(CRS.from_epsg(4326), crs)
209
+ # Calculate the margin
210
+ margin_ratio = 0.40 # 40% margin
227
211
 
228
- try:
229
- # Reproject to EPSG:4326
230
- ds = ds.rio.reproject("EPSG:4326", resolution=resolution)
212
+ # Get the bounding box
213
+ min_x, min_y = transformer.transform(lat_min, lon_min)
214
+ max_x, max_y = transformer.transform(lat_max, lon_max)
215
+
216
+ # Calculate the range
217
+ x_range = abs(max_x - min_x)
218
+ y_range = abs(max_y - min_y)
231
219
 
232
- # Rename lat/lon coordinates
233
- ds = ds.rename({"x": "lon", "y": "lat"})
220
+ margin_x = x_range * margin_ratio
221
+ margin_y = y_range * margin_ratio
234
222
 
235
- # Add resolution to attributes
223
+ # Expand the bounding box
224
+ min_x -= margin_x
225
+ max_x += margin_x
226
+ min_y -= margin_y
227
+ max_y += margin_y
228
+
229
+ # Sort the values
230
+ y_min, y_max = sorted([min_y, max_y], reverse=ds["y"].values[0] > ds["y"].values[-1])
231
+
232
+ # Crop the dataset based on the bounding box
233
+ ds = ds.sel(x=slice(min_x, max_x), y=slice(y_min, y_max))
234
+
235
+ # Sort the values
236
+ if ds["y"].values[0] > ds["y"].values[-1]:
237
+ ds = ds.sortby("y")
238
+
239
+ # Reproject the dataset in serial and convert values to short
240
+ ds = ds.rio.reproject("EPSG:4326", resolution=resolution)
241
+
242
+ # Rename lat/lon coordinates
243
+ ds = ds.rename({"x": "lon", "y": "lat"})
244
+
245
+ # Check if remap is not a string
246
+ if type(remap) != str:
236
247
  for var in var_names:
237
248
  ds[var].attrs['resolution'] = "x={:.2f} y={:.2f} degree".format(resolution, resolution)
238
249
  ds[var].attrs['comments'] = 'Cropped and reprojected to EPSG:4326 by goesgcp'
239
-
240
- # Crop using lat/lon coordinates, in parallel
250
+ # Crop using lat/lon coordinates
241
251
  ds = ds.rio.clip_box(minx=lon_min, miny=lat_min, maxx=lon_max, maxy=lat_max)
252
+ else:
253
+ # Add _FillValue to the variables
254
+ for var in var_names:
255
+ ds[var].attrs['_FillValue'] = float(fill_values[var_names.index(var)])
242
256
 
243
- # Add global metadata comments
244
- ds.attrs['comments'] = "Data processed by goesgcp, author: Helvecio B. L. Neto (helvecioblneto@gmail.com)"
245
- except Exception as e:
246
- print(f"Error processing file {file}: {e}")
247
- pass
248
-
249
- if save_format == 'by_date':
250
- file_datetime = datetime.strptime(ds.time_coverage_start,
257
+ # Add global metadata comments
258
+ ds.attrs['comments'] = "Data processed by goesgcp, author: Helvecio B. L. Neto (helvecioblneto@gmail.com)"
259
+
260
+ # Get the file datetime
261
+ file_datetime = datetime.strptime(ds.time_coverage_start,
251
262
  "%Y-%m-%dT%H:%M:%S.%fZ")
263
+ if save_format == 'by_date':
252
264
  year = file_datetime.strftime("%Y")
253
265
  month = file_datetime.strftime("%m")
254
266
  day = file_datetime.strftime("%d")
255
- output_directory = f"{output}{year}/{month}/{day}/"
267
+ output_directory = f"{output}{year}/{month}/"
256
268
  elif save_format == 'julian':
257
- file_datetime = datetime.strptime(ds.time_coverage_start,
269
+ file_datetime = datetime.strptime(ds.time_coverage_start,
258
270
  "%Y-%m-%dT%H:%M:%S.%fZ")
259
271
  year = file_datetime.strftime("%Y")
260
272
  julian_day = file_datetime.timetuple().tm_yday
261
273
  output_directory = f"{output}{year}/{julian_day}/"
262
274
  else:
263
275
  output_directory = output
264
-
265
276
 
266
277
  # Create the output directory
267
278
  pathlib.Path(output_directory).mkdir(parents=True, exist_ok=True)
268
279
 
269
- # Save the file
270
- output_file = f"{output_directory}{file.split('/')[-1]}"
271
- ds.to_netcdf(output_file, mode='w')
280
+ # Apply file pattern
281
+ if file_pattern is not None:
282
+ # Get the timestamp
283
+ file_datetime = datetime.strptime(ds.time_coverage_start,
284
+ "%Y-%m-%dT%H:%M:%S.%fZ")
285
+ file_pattern = file_datetime.strftime(file_pattern)
286
+ output_file = f"{output_directory}{file_pattern}.nc"
287
+ else:
288
+ output_file = f"{output_directory}{file.split('/')[-1]}"
272
289
 
273
- # Close the file
290
+ # Write the file
291
+ if classic_format:
292
+ # Change the data type to round values to int16
293
+ ds.to_netcdf(output_file, mode='w', format='NETCDF3_CLASSIC', encoding={var: {'dtype': np.int16} for var in var_names
294
+ })
295
+ else:
296
+ ds.to_netcdf(output_file, mode='w', encoding={var: {'zlib': True} for var in var_names})
274
297
  ds.close()
275
298
 
276
- return output_file
277
-
299
+ # Remap the file
300
+ if remap:
301
+ remap_file((remap, output_file, output, method))
302
+
303
+ if more_info:
304
+ with netCDF4.Dataset(output_file, 'r+') as ds:
305
+ # Clear old attributes
306
+ for var_name in var_names:
307
+ var = ds.variables[var_name]
308
+ for attr in var.ncattrs():
309
+ if attr == 'long_name' or attr == '_FillValue':
310
+ continue
311
+ var.delncattr(attr)
312
+ # Add new attributes
313
+ for var in range(len(var_names)):
314
+ ds[var_names[var]].setncattr('scale_factor', scale_factors[var])
315
+ ds[var_names[var]].setncattr('add_offset', add_offsets[var])
316
+ ds[var_names[var]].setncattr('missing_value', fill_values[var])
317
+ ds[var_names[var]].setncattr('units', np.float32(units[var]))
318
+ # Add variable satlat
319
+ ds.createDimension('satlat', 1)
320
+ ds.createVariable('satlat', 'f4', ('satlat',))
321
+ ds.variables['satlat'][:] = sat_lat
322
+ ds.variables['satlat'].long_name = 'Satellite Latitude'
323
+ ds.variables['satlat'].units = 'degrees_north'
324
+ ds.createDimension('satlon', 1)
325
+ ds.createVariable('satlon', 'f4', ('satlon',))
326
+ ds.variables['satlon'][:] = sat_lon
327
+ ds.variables['satlon'].long_name = 'Satellite Longitude'
328
+ ds.variables['satlon'].units = 'degrees_east'
329
+ ds.createDimension('julian_day', 1)
330
+ ds.createVariable('julian_day', 'i2', ('julian_day',))
331
+ ds.variables['julian_day'][:] = int(file_datetime.timetuple().tm_yday)
332
+ ds.variables['julian_day'].long_name = 'Julian day'
333
+ ds.variables['julian_day'].units = 'day'
334
+ time_of_day_char = netCDF4.stringtochar(np.array([str(file_datetime.strftime("%H%M"))], 'S4'))
335
+ # Add variable time_of_day
336
+ ds.createDimension('time_of_day', 4)
337
+ ds.createVariable('time_of_day', 'S1', ('time_of_day',))
338
+ ds.variables['time_of_day'][:] = time_of_day_char
339
+ ds.variables['time_of_day'].long_name = 'Time of day'
340
+ ds.variables['time_of_day'].units = 'hour and minute'
341
+ ds.variables['time_of_day'].comment = str(file_datetime.strftime("%H%M"))
278
342
 
279
343
 
280
344
  def remap_file(args):
281
345
  """ Remap the download file based on the input file. """
282
346
 
283
- base_file, target_file, var_name, save_format, output = args
284
-
285
- # Open the files
286
- base_ds = xr.open_dataset(base_file, engine="netcdf4")
347
+ base_file, target_file, output, method = args
287
348
 
288
- if save_format == 'by_date':
289
- file_datetime = datetime.strptime(ds.time_coverage_start,
290
- "%Y-%m-%dT%H:%M:%S.%fZ")
291
- year = file_datetime.strftime("%Y")
292
- month = file_datetime.strftime("%m")
293
- day = file_datetime.strftime("%d")
294
- output_directory = f"{output}{year}/{month}/{day}/"
295
- elif save_format == 'julian':
296
- file_datetime = datetime.strptime(ds.time_coverage_start,
297
- "%Y-%m-%dT%H:%M:%S.%fZ")
298
- year = file_datetime.strftime("%Y")
299
- julian_day = file_datetime.timetuple().tm_yday
300
- output_directory = f"{output}{year}/{julian_day}/"
301
- else:
302
- output_directory = output
349
+ # Get output directory based on target_file
350
+ output_file = f"{output}{target_file.split('/')[-1]}"
351
+ output_directory = output_file.replace(target_file.split('/')[-1], "")
303
352
 
304
353
  # Create the output directory
305
354
  pathlib.Path(output_directory).mkdir(parents=True, exist_ok=True)
@@ -311,7 +360,7 @@ def remap_file(args):
311
360
 
312
361
  # Run the cdo command
313
362
  cdo_command = [
314
- "cdo", "remapnn," + base_file, target_file, output_file
363
+ "cdo", method+"," + base_file, target_file, output_file
315
364
  ]
316
365
 
317
366
  try:
@@ -320,9 +369,6 @@ def remap_file(args):
320
369
  print(f"Error remapping file {target_file}: {e}")
321
370
  pass
322
371
 
323
- # Close the files
324
- base_ds.close()
325
-
326
372
  # Delete the target file
327
373
  pathlib.Path(target_file).unlink()
328
374
 
@@ -334,18 +380,17 @@ def process_file(args):
334
380
  """
335
381
  Downloads and processes a GOES-16 file.
336
382
  """
337
-
383
+
338
384
  bucket_name, blob_name, local_path, output_path, var_name, lat_min, lat_max, lon_min, lon_max, resolution, \
339
- save_format, retries, remap = args
385
+ save_format, retries, remap, met, more_info, file_pattern, classic_format = args
340
386
 
387
+ # Download the file
341
388
  attempt = 0
342
389
  while attempt < retries:
343
390
  try:
344
391
  # Connect to the bucket
345
392
  bucket = storage_client.bucket(bucket_name)
346
393
  blob = bucket.blob(blob_name)
347
-
348
- # Download the file
349
394
  blob.download_to_filename(local_path, timeout=120)
350
395
  break # Exit the loop if the download is successful
351
396
  except (GoogleAPIError, Exception) as e: # Catch any exception
@@ -353,21 +398,20 @@ def process_file(args):
353
398
  if attempt < retries:
354
399
  time.sleep(2 ** attempt) # Backoff exponencial
355
400
  else:
356
- # Log the error to a file
357
401
  with open('fail.log', 'a') as log_file:
358
402
  log_file.write(f"Failed to download {blob_name} after {retries} attempts. Error: {e}\n")
359
-
360
403
  # Crop the file
361
- output_file = crop_reproject((local_path, output_path, var_name,
362
- lat_min, lat_max, lon_min, lon_max, resolution, save_format))
363
-
364
- # Remap the file
365
- if remap is not None:
366
- # Remap the file
367
- remap_file((remap, output_file, var_name, save_format, output_path))
368
-
369
- # Remove the local file
370
- pathlib.Path(local_path).unlink()
404
+ try:
405
+ crop_reproject((local_path, output_path,
406
+ var_name, lat_min, lat_max, lon_min, lon_max,
407
+ resolution, save_format,
408
+ more_info, file_pattern, classic_format, remap, met))
409
+ # Remove the local file
410
+ pathlib.Path(local_path).unlink()
411
+ except Exception as e:
412
+ with open('fail.log', 'a') as log_file:
413
+ log_file.write(f"Failed to process {blob_name}. Error: {e}\n")
414
+ pass
371
415
 
372
416
  # Create connection
373
417
  storage_client = storage.Client.create_anonymous_client()
@@ -377,7 +421,7 @@ def main():
377
421
 
378
422
  epilog = """
379
423
  Example usage:
380
-
424
+
381
425
  - To download recent 3 files from the GOES-16 satellite for the ABI-L2-CMIPF product,
382
426
  change resolution to 0.045, and crop the files between latitudes -35 and 5 and longitudes -80 and -30:
383
427
 
@@ -411,9 +455,9 @@ def main():
411
455
  parser = argparse.ArgumentParser(description='Download and process GOES Satellite data files from GCP.',
412
456
  epilog=epilog,
413
457
  formatter_class=argparse.RawDescriptionHelpFormatter)
414
-
458
+
415
459
  # Satellite and product settings
416
- parser.add_argument('--satellite', type=str, default='goes-16', choices=['goes-16', 'goes-18'], help='Name of the satellite (e.g., goes16)')
460
+ parser.add_argument('--satellite', type=str, default='goes-19', choices=['goes-16', 'goes-18', 'goes-19'], help='Name of the satellite (e.g., goes16)')
417
461
  parser.add_argument('--product', type=str, default='ABI-L2-CMIPF', help='Name of the satellite product', choices=product_names)
418
462
  parser.add_argument('--var_name', type=str, default=None, help='Variable name to extract (e.g., CMI)')
419
463
  parser.add_argument('--channel', type=int, default=13, help='Channel to use (e.g., 13)')
@@ -430,24 +474,26 @@ def main():
430
474
  parser.add_argument('--bt_min', nargs=2, type=int, default=[0, 60], help='Filter data between these minutes (e.g., 0 60)')
431
475
 
432
476
  # Geographic bounding box
433
- parser.add_argument('--lat_min', type=float, default=-81.3282, help='Minimum latitude of the bounding box')
434
- parser.add_argument('--lat_max', type=float, default=81.3282, help='Maximum latitude of the bounding box')
435
- parser.add_argument('--lon_min', type=float, default=-156.2995, help='Minimum longitude of the bounding box')
436
- parser.add_argument('--lon_max', type=float, default=6.2995, help='Maximum longitude of the bounding box')
437
- parser.add_argument('--resolution', type=float, default=0.03208, help='Resolution of the output file')
438
- parser.add_argument('--output', type=str, default='output/', help='Path for saving output files')
477
+ parser.add_argument('--lat_min', type=float, default=-56, help='Minimum latitude of the bounding box')
478
+ parser.add_argument('--lat_max', type=float, default=35, help='Maximum latitude of the bounding box')
479
+ parser.add_argument('--lon_min', type=float, default=-116, help='Minimum longitude of the bounding box')
480
+ parser.add_argument('--lon_max', type=float, default=-25, help='Maximum longitude of the bounding box')
481
+ parser.add_argument('--resolution', type=float, default=0.03, help='Resolution of the output file')
482
+ parser.add_argument('--output', type=str, default='./output/', help='Path for saving output files')
439
483
 
440
484
  # Remap
441
485
  parser.add_argument('--remap', type=str, default=None, help='Give a input file to remap the output')
486
+ parser.add_argument('--method', type=str, default='remapnn', help='Remap method to use (e.g., remapnn)')
442
487
 
443
488
  # Other settings
444
489
  parser.add_argument('--parallel', type=lambda x: bool(strtobool(x)), default=True, help='Use parallel processing')
445
490
  parser.add_argument('--processes', type=int, default=4, help='Number of processes for parallel execution')
446
491
  parser.add_argument('--max_attempts', type=int, default=3, help='Number of attempts to download a file')
492
+ parser.add_argument('--info', type=lambda x: bool(strtobool(x)), default=False, help='Show information messages')
447
493
  parser.add_argument('--save_format', type=str, default='flat', choices=['flat', 'by_date','julian'],
448
494
  help="Save the files in a flat structure or by date")
449
-
450
-
495
+ parser.add_argument('--file_pattern', type=str, default=None, help='Pattern for the files')
496
+ parser.add_argument('--netcdf_classic', type=lambda x: bool(strtobool(x)), default=False, help='Save the files in netCDF classic format')
451
497
  # Parse arguments
452
498
  args = parser.parse_args()
453
499
 
@@ -477,7 +523,10 @@ def main():
477
523
  bt_min = args.bt_min
478
524
  save_format = args.save_format
479
525
  remap = args.remap
480
-
526
+ method = args.method
527
+ more_info = args.info
528
+ file_pattern = args.file_pattern
529
+ classic_format = args.netcdf_classic
481
530
 
482
531
  # Check mandatory arguments
483
532
  if not args.recent and not (args.start and args.end):
@@ -510,8 +559,8 @@ def main():
510
559
  # Check operational mode if is recent or specific date
511
560
  if start and end:
512
561
  files_list = get_files_period(storage_client, bucket_name,
513
- product, pattern, start, end,
514
- bt_hour, bt_min, freq)
562
+ product, pattern, start, end,
563
+ bt_hour, bt_min, freq)
515
564
  else:
516
565
  # Get recent files
517
566
  files_list = get_recent_files(storage_client, bucket_name, product, pattern, recent)
@@ -529,12 +578,13 @@ def main():
529
578
  loading_bar = tqdm.tqdm(total=len(files_list), ncols=100, position=0, leave=True,
530
579
  bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} + \
531
580
  [Elapsed:{elapsed} Remaining:<{remaining}]')
532
-
581
+
533
582
  if parallel: # Run in parallel
534
583
  # Create a list of tasks
535
- tasks = [(bucket_name, file, f"tmp/{file.split('/')[-1]}", output_path, var_name,
584
+ tasks = [(bucket_name, file, f"tmp/{file.split('/')[-1]}", output_path, var_name,
536
585
  lat_min, lat_max, lon_min, lon_max, resolution,
537
- save_format, max_attempts, remap) for file in files_list]
586
+ save_format, max_attempts, remap, method,
587
+ more_info, file_pattern, classic_format) for file in files_list]
538
588
 
539
589
  # Download files in parallel
540
590
  with Pool(processes=args.processes) as pool:
@@ -546,10 +596,12 @@ def main():
546
596
  local_path = f"tmp/{file.split('/')[-1]}"
547
597
  process_file((bucket_name, file, local_path, output_path, var_name,
548
598
  lat_min, lat_max, lon_min, lon_max, resolution,
549
- save_format, max_attempts, remap))
599
+ save_format, max_attempts, remap, method, more_info,
600
+ file_pattern, classic_format))
550
601
  loading_bar.update(1)
551
602
  loading_bar.close()
552
603
 
604
+ # Clean up the temporary directory
553
605
  shutil.rmtree('tmp/')
554
606
 
555
607
  if __name__ == '__main__':
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: goesgcp
3
- Version: 2.1.0
3
+ Version: 3.0.1
4
4
  Summary: A package to download and process GOES-16/17 data
5
5
  Home-page: https://github.com/helvecioneto/goesgcp
6
6
  Author: Helvecio B. L. Neto
@@ -19,11 +19,12 @@ Classifier: Topic :: Software Development
19
19
  Classifier: Topic :: Utilities
20
20
  Description-Content-Type: text/markdown
21
21
  License-File: LICENSE
22
- Requires-Dist: google-cloud-storage
23
- Requires-Dist: pyproj
22
+ Requires-Dist: pandas
24
23
  Requires-Dist: xarray
24
+ Requires-Dist: google-cloud-storage
25
25
  Requires-Dist: netcdf4
26
- Requires-Dist: rioxarray
26
+ Requires-Dist: tqdm
27
+ Requires-Dist: setuptools
27
28
  Dynamic: author
28
29
  Dynamic: author-email
29
30
  Dynamic: classifier
@@ -31,6 +32,7 @@ Dynamic: description
31
32
  Dynamic: description-content-type
32
33
  Dynamic: home-page
33
34
  Dynamic: license
35
+ Dynamic: license-file
34
36
  Dynamic: requires-dist
35
37
  Dynamic: summary
36
38
 
@@ -62,6 +64,37 @@ pip install goesgcp
62
64
  ```
63
65
 
64
66
 
67
+ Obs: If gdal is not installed, you can install it using the following command:
68
+
69
+ Linux:
70
+ ```bash
71
+ sudo apt-get install gdal-bin
72
+ ```
73
+
74
+ Windows:
75
+ ```bash
76
+ conda install -c conda-forge gdal
77
+ ```
78
+
79
+ MacOS:
80
+ ```bash
81
+ brew install gdal
82
+ ```
83
+
84
+ Or you can install the wheel file:
85
+
86
+ ```bash
87
+ python -m pip install gdal -f https://girder.github.io/large_image_wheels
88
+ ```
89
+
90
+ and install other dependencies:
91
+
92
+ ```bash
93
+ pip install -r requirements.txt
94
+ ```
95
+
96
+
97
+
65
98
  ## Usage
66
99
 
67
100
  ### Available Command-Line Arguments
@@ -1,5 +1,6 @@
1
- google-cloud-storage
2
- pyproj
1
+ pandas
3
2
  xarray
3
+ google-cloud-storage
4
4
  netcdf4
5
- rioxarray
5
+ tqdm
6
+ setuptools
@@ -1,5 +1,6 @@
1
- google-cloud-storage
2
- pyproj
1
+ pandas
3
2
  xarray
3
+ google-cloud-storage
4
4
  netcdf4
5
- rioxarray
5
+ tqdm
6
+ setuptools
goesgcp-3.0.1/setup.py ADDED
@@ -0,0 +1,101 @@
1
+ import subprocess
2
+
3
+ # Upgrade pip and setuptools before installing dependencies
4
+ try:
5
+ subprocess.run(["python", "-m", "ensurepip", "--upgrade"], check=True)
6
+ subprocess.run(["python", "-m", "pip", "install", "--upgrade", "pip"], check=True)
7
+ subprocess.run(["python", "-m", "pip", "install", "--upgrade", "setuptools"], check=True)
8
+ except subprocess.CalledProcessError:
9
+ print("Error upgrading pip and setuptools.")
10
+
11
+ import os
12
+ import sys
13
+ import platform
14
+ from setuptools import setup, find_packages
15
+
16
+
17
+ def install_gdal():
18
+ """Installs GDAL based on the operating system."""
19
+ system = platform.system()
20
+
21
+ try:
22
+ if system == "Linux":
23
+ print("Installing GDAL on Linux...")
24
+ subprocess.run(
25
+ ["python", "-m", "pip", "install", "gdal", "-f", "https://girder.github.io/large_image_wheels"],
26
+ check=True
27
+ )
28
+
29
+ elif system == "Windows":
30
+ python_version = sys.version_info
31
+ if python_version[0] == 3:
32
+ version_map = {
33
+ 10: "cp310",
34
+ 11: "cp311",
35
+ 12: "cp312",
36
+ 13: "cp313"
37
+ }
38
+ if python_version[1] in version_map:
39
+ gdal_url = f"https://github.com/cgohlke/geospatial-wheels/releases/download/v2025.1.20/GDAL-3.10.1-{version_map[python_version[1]]}-win_amd64.whl"
40
+ print(f"Downloading GDAL for Python {python_version[0]}.{python_version[1]}...")
41
+ subprocess.run(["python", "-m", "pip", "install", gdal_url], check=True)
42
+ else:
43
+ print("Unsupported Python version for GDAL.")
44
+ sys.exit(1)
45
+ else:
46
+ print("Unsupported Python version for GDAL.")
47
+ sys.exit(1)
48
+
49
+ elif system == "Darwin":
50
+ print("GDAL is not available for macOS via this installer.")
51
+ sys.exit(1)
52
+
53
+ else:
54
+ print("Unsupported operating system.")
55
+ sys.exit(1)
56
+
57
+ except subprocess.CalledProcessError:
58
+ print("Error installing GDAL.")
59
+
60
+ # Install GDAL before running setup
61
+ install_gdal()
62
+
63
+ # Read dependencies from requirements.txt
64
+ req_file = os.path.join(os.path.dirname(__file__), "requirements.txt")
65
+ if os.path.exists(req_file):
66
+ with open(req_file) as f:
67
+ requirements = f.read().splitlines()
68
+ else:
69
+ requirements = []
70
+
71
+ setup(
72
+ name="goesgcp",
73
+ version="3.0.1",
74
+ author="Helvecio B. L. Neto",
75
+ author_email="helvecioblneto@gmail.com",
76
+ description="A package to download and process GOES-16/17 data",
77
+ long_description=open("README.md").read(),
78
+ long_description_content_type="text/markdown",
79
+ url="https://github.com/helvecioneto/goesgcp",
80
+ packages=find_packages(),
81
+ install_requires=requirements,
82
+ license="LICENSE",
83
+ classifiers=[
84
+ "Programming Language :: Python",
85
+ "Development Status :: 5 - Production/Stable",
86
+ "Operating System :: OS Independent",
87
+ "Programming Language :: Python :: 3.10",
88
+ "Programming Language :: Python :: 3.11",
89
+ "Programming Language :: Python :: 3.12",
90
+ "Topic :: Scientific/Engineering :: Atmospheric Science",
91
+ "Topic :: Scientific/Engineering :: GIS",
92
+ "Topic :: Scientific/Engineering",
93
+ "Topic :: Software Development",
94
+ "Topic :: Utilities",
95
+ ],
96
+ entry_points={
97
+ 'console_scripts': [
98
+ 'goesgcp=goesgcp.main:main',
99
+ ],
100
+ },
101
+ )
goesgcp-2.1.0/setup.py DELETED
@@ -1,49 +0,0 @@
1
-
2
- import os
3
-
4
- try:
5
- os.system("python -m ensurepip --upgrade")
6
- os.system("python -m pip install --upgrade setuptools")
7
- except:
8
- pass
9
-
10
- from setuptools import setup, find_packages
11
-
12
- req_file = os.path.join(os.path.dirname(__file__), "requirements.txt")
13
- if os.path.exists(req_file):
14
- with open(req_file) as f:
15
- requirements = f.read().splitlines()
16
- else:
17
- requirements = []
18
-
19
- setup(
20
- name="goesgcp",
21
- version="2.1.0",
22
- author="Helvecio B. L. Neto",
23
- author_email="helvecioblneto@gmail.com",
24
- description="A package to download and process GOES-16/17 data",
25
- long_description=open("README.md").read(),
26
- long_description_content_type="text/markdown",
27
- url="https://github.com/helvecioneto/goesgcp",
28
- packages=find_packages(),
29
- install_requires=requirements,
30
- license="LICENSE",
31
- classifiers=[
32
- "Programming Language :: Python",
33
- "Development Status :: 5 - Production/Stable",
34
- "Operating System :: OS Independent",
35
- "Programming Language :: Python :: 3.10",
36
- "Programming Language :: Python :: 3.11",
37
- "Programming Language :: Python :: 3.12",
38
- "Topic :: Scientific/Engineering :: Atmospheric Science",
39
- "Topic :: Scientific/Engineering :: GIS",
40
- "Topic :: Scientific/Engineering",
41
- "Topic :: Software Development",
42
- "Topic :: Utilities",
43
- ],
44
- entry_points={
45
- 'console_scripts': [
46
- 'goesgcp=goesgcp.main:main',
47
- ],
48
- },
49
- )
File without changes
File without changes
File without changes
File without changes
File without changes