goesgcp 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

goesgcp/main.py

@@ -13,6 +13,9 @@ from datetime import datetime, timedelta, timezone
 from pyproj import CRS, Transformer
 from google.api_core.exceptions import GoogleAPIError
 
+import warnings
+warnings.filterwarnings('ignore')
+
 
 def list_blobs(connection, bucket_name, prefix):
     """
@@ -54,11 +57,11 @@ def get_files_period(connection, bucket_name, base_prefix, pattern,
     files_metadata = []
 
     # Generate the list of dates from start to end
-    current_time = start
-    while current_time < end:
-        year = current_time.year
-        julian_day = str(current_time.timetuple().tm_yday).zfill(3)  # Julian day
-        hour = current_time.hour
+    temp = start
+    while temp <= end:
+        year = temp.year
+        julian_day = str(temp.timetuple().tm_yday).zfill(3)  # Julian day
+        hour = temp.hour
 
         # Generate the directory prefix
         prefix = f"{base_prefix}/{get_directory_prefix(year, julian_day, hour)}"
@@ -71,26 +74,27 @@ def get_files_period(connection, bucket_name, base_prefix, pattern,
             if pattern in blob.name:
                 files_metadata.append({
                     'file_name': blob.name,
-                    'last_modified': blob.updated
                 })
 
         # Move to the next hour
-        current_time += timedelta(hours=1)
+        temp += timedelta(hours=1)
 
     # Create a DataFrame from the list of files
     df = pd.DataFrame(files_metadata)
 
+    # Transform file_name to datetime
+    df['last_modified'] = pd.to_datetime(df['file_name'].str.extract(r'(\d{4}\d{3}\d{2}\d{2})').squeeze(), format='%Y%j%H%M')
+
+
     if df.empty:
         print("No files found matching the pattern.")
         return pd.DataFrame()
     
     # Ensure 'last_modified' is in the correct datetime format without timezone
-    df['last_modified'] = pd.to_datetime(df['last_modified']).dt.tz_localize(None)
-    start = pd.to_datetime(start).tz_localize(None)
-    end = pd.to_datetime(end).tz_localize(None)
-    
-    # Filter the DataFrame based on the date range
-    df = df[(df['last_modified'] >= start) & (df['last_modified'] < end)]
+    df['last_modified'] = pd.to_datetime(df['last_modified']).dt.tz_localize('UTC')
+
+    # Filter the DataFrame based on the date range (inclusive)
+    df = df[(df['last_modified'] >= start) & (df['last_modified'] <= end)]
 
     # Filter the DataFrame based on the hour range
     df['hour'] = df['last_modified'].dt.hour
@@ -153,13 +157,22 @@ def crop_reproject(args):
     Crops and reprojects a GOES-16 file to EPSG:4326.
     """
 
-    file, output = args
+    file, output, var_name, lat_min, lat_max, lon_min, lon_max, resolution, save_format = args
 
     # Open the file
     ds = xr.open_dataset(file, engine="netcdf4")
 
+    if var_name is None:
+        # Get all variables are 2D
+        var_names = [var for var in ds.data_vars if len(ds[var].dims) == 2]
+
+        # Remove DQF variables
+        var_names = [var for var in var_names if 'DQF' not in var]
+    else:
+        var_names = [var_name]
+
     # Select only var_name and goes_imager_projection
-    ds = ds[[var_name, "goes_imager_projection"]]
+    ds = ds[var_names + ["goes_imager_projection"]]
 
     # Get projection
     sat_height = ds["goes_imager_projection"].attrs["perspective_point_height"]
@@ -215,14 +228,13 @@ def crop_reproject(args):
     ds = ds.rename({"x": "lon", "y": "lat"})
 
     # Add resolution to attributes
-    ds[var_name].attrs['resolution'] = "x={:.2f} y={:.2f} degree".format(resolution, resolution) 
+    for var in var_names:
+        ds[var].attrs['resolution'] = "x={:.2f} y={:.2f} degree".format(resolution, resolution)
+        ds[var].attrs['comments'] = 'Cropped and reprojected to EPSG:4326 by goesgcp'
 
     # Crop using lat/lon coordinates, in parallel
     ds = ds.rio.clip_box(minx=lon_min, miny=lat_min, maxx=lon_max, maxy=lat_max)
 
-    # Add comments
-    ds[var_name].attrs['comments'] = 'Cropped and reprojected to EPSG:4326 by goesgcp'
-
     # Add global metadata comments
     ds.attrs['comments'] = "Data processed by goesgcp, author: Helvecio B. L. Neto (helvecioblneto@gmail.com)"
         
@@ -249,20 +261,18 @@ def crop_reproject(args):
     output_file = f"{output_directory}{file.split('/')[-1]}"
     ds.to_netcdf(output_file, mode='w', format='NETCDF4_CLASSIC')
 
-    # Fechar o dataset
     ds.close()
-    return
 
 
 def process_file(args):
-    """ Downloads and processes a file in parallel. """
-
-    bucket_name, blob_name, local_path = args
+    """
+    Downloads and processes a GOES-16 file.
+    """
+    
+    bucket_name, blob_name, local_path, output_path, var_name, lat_min, lat_max, lon_min, lon_max, resolution, \
+    save_format, retries = args
 
-    # Download options
-    retries = 5
     attempt = 0
-
     while attempt < retries:
         try:
             # Connect to the bucket
@@ -282,46 +292,59 @@ def process_file(args):
                     log_file.write(f"Failed to download {blob_name} after {retries} attempts. Error: {e}\n")
 
     # Crop the file
-    crop_reproject((local_path, output_path))
+    crop_reproject((local_path, output_path, var_name, lat_min, lat_max, lon_min, lon_max, resolution, save_format))
 
     # Remove the local file
     pathlib.Path(local_path).unlink()
 
+# Create connection
+storage_client = storage.Client.create_anonymous_client()
 
 def main():
     ''' Main function to download and process GOES-16 files. '''
 
-
-    global output_path, var_name, \
-          lat_min, lat_max, lon_min, lon_max, \
-          max_attempts, parallel, recent, resolution, storage_client, \
-            satellite, product, domain, op_mode, channel, save_format
-
     epilog = """
     Example usage:
     
-    - To download recent 3 files from the GOES-16 satellite for the ABI-L2-CMIPF product:
+    - To download recent 3 files from the GOES-16 satellite for the ABI-L2-CMIPF product,
+    change resolution to 0.045, and crop the files between latitudes -35 and 5 and longitudes -80 and -30:
 
-    goesgcp --satellite goes16 --product ABI-L2-CMIP --recent 3"
+    goesgcp --satellite goes-16 --product ABI-L2-CMIPF --recent 3 --resolution 0.045 --lat_min -35 --lat_max 5 --lon_min -80 --lon_max -30
 
     - To download files from the GOES-16 satellite for the ABI-L2-CMIPF product between 2022-12-15 and 2022-12-20:
 
-    goesgcp --start '2022-12-15 00:00:00' --end '2022-12-20 10:00:00' --bt_hour 5 6 --save_format by_date --resolution 0.045 --lat_min -35 --lat_max 5 --lon_min -80 --lon_max -30
+    goesgcp --satellite goes-16 --product ABI-L2-CMIPF --start '2022-12-15 09:00:00' --end '2022-12-15 09:50:00' --resolution 0.045 --lat_min -35 --lat_max 5 --lon_min -80 --lon_max -30
 
     """
 
+    product_names = [
+    "ABI-L1b-RadF", "ABI-L1b-RadC", "ABI-L1b-RadM", "ABI-L2-ACHAC", "ABI-L2-ACHAF", "ABI-L2-ACHAM",
+    "ABI-L2-ACHTF", "ABI-L2-ACHTM", "ABI-L2-ACMC", "ABI-L2-ACMF", "ABI-L2-ACMM", "ABI-L2-ACTPC",
+    "ABI-L2-ACTPF", "ABI-L2-ACTPM", "ABI-L2-ADPC", "ABI-L2-ADPF", "ABI-L2-ADPM", "ABI-L2-AICEF",
+    "ABI-L2-AITAF", "ABI-L2-AODC", "ABI-L2-AODF", "ABI-L2-BRFC", "ABI-L2-BRFF", "ABI-L2-BRFM",
+    "ABI-L2-CMIPC", "ABI-L2-CMIPF", "ABI-L2-CMIPM", "ABI-L2-CODC", "ABI-L2-CODF", "ABI-L2-CPSC",
+    "ABI-L2-CPSF", "ABI-L2-CPSM", "ABI-L2-CTPC", "ABI-L2-CTPF", "ABI-L2-DMWC", "ABI-L2-DMWF",
+    "ABI-L2-DMWM", "ABI-L2-DMWVC", "ABI-L2-DMWVF", "ABI-L2-DMWVF", "ABI-L2-DSIC", "ABI-L2-DSIF",
+    "ABI-L2-DSIM", "ABI-L2-DSRC", "ABI-L2-DSRF", "ABI-L2-DSRM", "ABI-L2-FDCC", "ABI-L2-FDCF",
+    "ABI-L2-FDCM", "ABI-L2-LSAC", "ABI-L2-LSAF", "ABI-L2-LSAM", "ABI-L2-LSTC", "ABI-L2-LSTF",
+    "ABI-L2-LSTM", "ABI-L2-LVMPC", "ABI-L2-LVMPF", "ABI-L2-LVMPM", "ABI-L2-LVTPC", "ABI-L2-LVTPF",
+    "ABI-L2-LVTPM", "ABI-L2-MCMIPC", "ABI-L2-MCMIPF", "ABI-L2-MCMIPM", "ABI-L2-RRQPEF",
+    "ABI-L2-RSRC", "ABI-L2-RSRF", "ABI-L2-SSTF", "ABI-L2-TPWC", "ABI-L2-TPWF", "ABI-L2-TPWM",
+    "ABI-L2-VAAF", "EXIS-L1b-SFEU", "EXIS-L1b-SFXR", "GLM-L2-LCFA", "MAG-L1b-GEOF", "SEIS-L1b-EHIS",
+    "SEIS-L1b-MPSH", "SEIS-L1b-MPSL", "SEIS-L1b-SGPS", "SUVI-L1b-Fe093", "SUVI-L1b-Fe131",
+    "SUVI-L1b-Fe171", "SUVI-L1b-Fe195", "SUVI-L1b-Fe284", "SUVI-L1b-He303"
+    ]
 
     # Set arguments
-    parser = argparse.ArgumentParser(description='Converts GOES-16 L2 data to netCDF',
+    parser = argparse.ArgumentParser(description='Download and process GOES Satellite data files from GCP.',
                                     epilog=epilog,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
     
     # Satellite and product settings
-    parser.add_argument('--satellite', type=str, default='goes-16', help='Name of the satellite (e.g., goes16)')
-    parser.add_argument('--product', type=str, default='ABI-L2-CMIP', help='Name of the satellite product')
-    parser.add_argument('--var_name', type=str, default='CMI', help='Variable name to extract (e.g., CMI)')
+    parser.add_argument('--satellite', type=str, default='goes-16', choices=['goes-16', 'goes-18'], help='Name of the satellite (e.g., goes16)')
+    parser.add_argument('--product', type=str, default='ABI-L2-CMIPF', help='Name of the satellite product', choices=product_names)
+    parser.add_argument('--var_name', type=str, default=None, help='Variable name to extract (e.g., CMI)')
     parser.add_argument('--channel', type=int, default=13, help='Channel to use (e.g., 13)')
-    parser.add_argument('--domain', type=str, default='F', help='Domain to use (e.g., F or C)')
     parser.add_argument('--op_mode', type=str, default='M6C', help='Operational mode to use (e.g., M6C)')
 
     # Recent files settings
@@ -361,7 +384,6 @@ def main():
     output_path = args.output
     satellite = args.satellite
     product = args.product
-    domain = args.domain
     op_mode = args.op_mode
     channel = str(args.channel).zfill(2)
     var_name = args.var_name
@@ -392,9 +414,6 @@ def main():
     # Create output directory
     pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)
 
-    # Create connection
-    storage_client = storage.Client.create_anonymous_client()
-
     # Check if the bucket exists
     try:
         storage_client.get_bucket(bucket_name)
@@ -403,16 +422,16 @@ def main():
         sys.exit(1)
 
     # Set pattern for the files
-    pattern = "OR_"+product+domain+"-"+op_mode+channel+"_G" + satellite[-2:]
+    pattern = "OR_"+product+"-"+op_mode+channel+"_G" + satellite[-2:]
 
     # Check operational mode if is recent or specific date
     if start and end:
         files_list = get_files_period(storage_client, bucket_name,
-                                       product + domain, pattern, start, end,
+                                       product, pattern, start, end,
                                         bt_hour, bt_min, freq)
     else:
         # Get recent files
-        files_list = get_recent_files(storage_client, bucket_name, product + domain, pattern, recent)
+        files_list = get_recent_files(storage_client, bucket_name, product, pattern, recent)
 
     # Check if any files were found
     if not files_list:
@@ -430,7 +449,9 @@ def main():
     
     if parallel: # Run in parallel
         # Create a list of tasks
-        tasks = [(bucket_name, file, f"tmp/{file.split('/')[-1]}") for file in files_list]
+        tasks = [(bucket_name, file, f"tmp/{file.split('/')[-1]}", output_path, var_name, 
+        lat_min, lat_max, lon_min, lon_max, resolution,
+        save_format, max_attempts) for file in files_list]
 
         # Download files in parallel
         with Pool(processes=args.processes) as pool:
@@ -440,11 +461,12 @@ def main():
     else: # Run in serial
         for file in files_list:
             local_path = f"tmp/{file.split('/')[-1]}"
-            process_file((bucket_name, file, local_path))
+            process_file((bucket_name, file, local_path, output_path, var_name,
+            lat_min, lat_max, lon_min, lon_max, resolution,
+            save_format, max_attempts))
             loading_bar.update(1)
         loading_bar.close()
 
-    # Remove temporary directory
     shutil.rmtree('tmp/')
 
 if __name__ == '__main__':

{goesgcp-2.0.0.dist-info → goesgcp-2.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: goesgcp
-Version: 2.0.0
+Version: 2.0.2
 Summary: A package to download and process GOES-16/17 data
 Home-page: https://github.com/helvecioneto/goesgcp
 Author: Helvecio B. L. Neto
@@ -33,18 +33,32 @@ Dynamic: requires-dist
 Dynamic: summary
 
 # goesgcp
+<!-- badges: start -->
+[![pypi](https://badge.fury.io/py/goesgcp.svg)](https://pypi.python.org/pypi/goesgcp)
+[![Downloads](https://img.shields.io/pypi/dm/goesgcp.svg)](https://pypi.python.org/pypi/goesgcp)
+[![Contributors](https://img.shields.io/github/contributors/helvecioneto/goesgcp.svg)](https://github.com/helvecioneto/goesgcp/graphs/contributors)
+[![License](https://img.shields.io/pypi/l/goesgcp.svg)](https://github.com/helvecioneto/goesgcp/blob/main/LICENSE)
+<!-- badges: end -->
 
-goesgcp is a utility script for downloading and reprojecting GOES-R satellite data. The script uses the `google.cloud` library to download data from the Google Cloud Platform (GCP) and the `pyproj` library to reproject the data to EPSG:4326 and crop it to a specified bounding box.
 
+`goesgcp` is a Python utility designed for downloading and reprojecting GOES-R satellite data. This script leverages the `google.cloud` library for accessing data from the Google Cloud Platform (GCP) and `pyproj` for reprojecting data to EPSG:4326, as well as cropping it to a user-defined bounding box.
+
+## Features
+
+- **Download GOES-R satellite data**: Supports GOES-16 and GOES-17.
+- **Reprojection and cropping**: Reprojects data to EPSG:4326 and crops to a specified bounding box.
+- **Flexible command-line interface**: Customize download options, variables, channels, time range, and output format.
+- **Efficient processing**: Handles large datasets with optimized performance.
 
 ## Installation
 
-You can install the necessary dependencies using `pip`:
+Install the necessary dependencies via `pip`:
 
 ```bash
 pip install goesgcp
 ```
 
+
 ## Usage
 
 ### Command-Line Arguments
@@ -74,19 +88,31 @@ goesgcp [OPTIONS]
 | `--bt_minute`         | Minute of the hour to download data (default: [0, 15, 30, 45]).           |
 | `--save_format`       | Format for saving output files (default: `by_date`).                      |
 
+#### Available GOES Products
+A comprehensive list of available GOES products can be found at the following link: [https://console.cloud.google.com/storage/browser/gcp-public-data-goes-16](https://console.cloud.google.com/storage/browser/gcp-public-data-goes-16)
+
 ### Examples
 
-To download most 3 recent data for the GOES-16 satellite, ABI-L2-CMIPF product, variable CMI, and channel 13, run the following command:
+#### Download Recent Data
+In the example below, the command downloads the 3 most recent files from the GOES-16 satellite for the product ABI-L2-CMIPF. It focuses on the variable CMI (Cloud and Moisture Imagery) from channel 13, which is commonly used for infrared observations. The downloaded files are saved to the specified output directory output/.
 
 ```bash
-goesgcp --satellite goes16 --product ABI-L2-CMIPF --var_name CMI --channel 13 --recent 3 --output "output/"
+goesgcp --satellite goes-16 --product ABI-L2-CMIPF --recent 3 --output "output/"
 ```
 
-To download data for a specific date range, use the `--start` and `--end` options:
+#### Download Data for a Specific Time Range
+This command retrieves GOES-16 satellite data for the product ABI-L2-CMIPF within the date range 2022-12-15 00:00:00 to 2022-12-20 10:00:00, focusing on hours 5:00 and 6:00 AM. The data is cropped to the geographic bounds of -35° to 5° latitude and -80° to -30° longitude, reprojected with a resolution of 0.045 degrees, and saved in a by_date format for easy organization.
 
 ```bash
-goesgcp --start '2022-12-15 00:00:00' --end '2022-12-20 10:00:00' --bt_hour 5 6 --save_format by_date --resolution 0.045 --lat_min -35 --lat_max 5 --lon_min -80 --lon_max -30
+goesgcp --satellite goes-16 --product ABI-L2-CMIPF --start '2022-12-15 00:00:00' --end '2022-12-20 10:00:00' --bt_hour 5 6 --save_format by_date --resolution 0.045 --lat_min -35 --lat_max 5 --lon_min -80 --lon_max -30
 ```
 
+### Contributing
+Contributions are welcome! If you encounter issues or have suggestions for improvements, please submit them via GitHub issues or pull requests.
+
 ### Credits
-And this is a otimization by Helvecio Neto - 2025
+This project was developed and optimized by Helvecio Neto (2025).
+It builds upon NOAA GOES-R data and leverages resources provided by the Google Cloud Platform.
+
+### License
+This project is licensed under the MIT License. 

goesgcp-2.0.2.dist-info/RECORD

@@ -0,0 +1,8 @@
+goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
+goesgcp/main.py,sha256=tDkn46WP0Nwet_3EwW6Rx-A-ASipuc8X251qZOpeeMU,18851
+goesgcp-2.0.2.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
+goesgcp-2.0.2.dist-info/METADATA,sha256=mab7nr7wRRptYsLU_vZykjWZKqXyIT72aV5RiL-CBmQ,6119
+goesgcp-2.0.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+goesgcp-2.0.2.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
+goesgcp-2.0.2.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
+goesgcp-2.0.2.dist-info/RECORD,,

goesgcp-2.0.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
-goesgcp/main.py,sha256=5vCAcSuMgFRzTLKZL4IUGgXVBxM3Buw19l-2MrpGR9M,16837
-goesgcp-2.0.0.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
-goesgcp-2.0.0.dist-info/METADATA,sha256=fgCGPkdI51BKwv7kpgiZgHP9WZbXhSdWcxkPhaZ2fKk,4066
-goesgcp-2.0.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-goesgcp-2.0.0.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
-goesgcp-2.0.0.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
-goesgcp-2.0.0.dist-info/RECORD,,