goesgcp 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

goesgcp/main.py

@@ -5,9 +5,10 @@ import argparse
 import sys
 import tqdm
 from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import Pool
 from google.cloud import storage
 from datetime import datetime, timedelta, timezone
-from pyproj import CRS
+from pyproj import CRS, Transformer
 
 
 
@@ -64,71 +65,120 @@ def get_recent_files(connection, bucket_name, base_prefix, pattern, min_files):
     # Return only the names of the most recent files, according to the minimum requested
     return [file[0] for file in files[:min_files]]
 
-def download_file(connection, bucket_name, blob_name, local_path):
-    """Downloads a file from a GCP bucket."""
-    bucket = connection.bucket(bucket_name)
-    blob = bucket.blob(blob_name)
-    blob.download_to_filename(local_path)
 
-def crop_reproject(file, output):
+def crop_reproject(args):
     """
     Crops and reprojects a GOES-16 file to EPSG:4326.
     """
 
+    file, output = args
+
+    # Open the file
+    ds = xr.open_dataset(file, engine='netcdf4')
 
-    ds = xr.open_dataset(file)
     # Select only var_name and goes_imager_projection
     ds = ds[[var_name, "goes_imager_projection"]]
+
     # Get projection
     sat_height = ds["goes_imager_projection"].attrs["perspective_point_height"]
     ds = ds.assign_coords({
                 "x": ds["x"].values * sat_height,
                 "y": ds["y"].values * sat_height,
             })
-    # Set CRS
+    # Set CRS from goes_imager_projection
     crs = CRS.from_cf(ds["goes_imager_projection"].attrs)
     ds = ds.rio.write_crs(crs)
 
-    # Reproject to EPSG:4326 using parallel processing
-    ds = ds.rio.reproject(dst_crs="EPSG:4326",
-                          resolution=(resolution, resolution),
-                          num_threads=-1)
+    # Try to reduce the size of the dataset
+    try:
+        # Create a transformer
+        transformer = Transformer.from_crs(CRS.from_epsg(4326), crs)
+        # Calculate the margin
+        margin_ratio = 0.40  # 40% margin
+
+        # Get the bounding box
+        min_x, min_y = transformer.transform(lat_min, lon_min)
+        max_x, max_y = transformer.transform(lat_max, lon_max)
+
+        # Calculate the range
+        x_range = abs(max_x - min_x)
+        y_range = abs(max_y - min_y)
+
+        margin_x = x_range * margin_ratio
+        margin_y = y_range * margin_ratio
+
+        # Expand the bounding box
+        min_x -= margin_x
+        max_x += margin_x
+        min_y -= margin_y
+        max_y += margin_y
+
+        # Select the region
+        if ds["y"].values[0] > ds["y"].values[-1]:  # Eixo y decrescente
+            ds_ = ds.sel(x=slice(min_x, max_x), y=slice(max_y, min_y))
+        else:  # Eixo y crescente
+            ds_ = ds.sel(x=slice(min_x, max_x), y=slice(min_y, max_y))
+        # Sort by y
+        if ds_["y"].values[0] > ds_["y"].values[-1]:
+            ds_ = ds_.sortby("y")
+        # Assign to ds
+        ds = ds_
+    except:
+        pass
+
+    # Reproject to EPSG:4326
+    ds = ds.rio.reproject("EPSG:4326", resolution=resolution)
 
     # Rename lat/lon coordinates
     ds = ds.rename({"x": "lon", "y": "lat"})
 
-    # # Crop using lat/lon coordinates, in parallel
-    ds = ds.rio.clip_box(minx=lon_min, miny=lat_min, maxx=lon_max, maxy=lat_max)
+    # Add resolution to attributes
+    ds[var_name].attrs['resolution'] = "x={:.2f} y={:.2f} degree".format(resolution, resolution) 
 
-    # Remove any previous file
-    if pathlib.Path(f'{output}{file.split("/")[-1]}.nc').exists():
-        pathlib.Path(f'{output}{file.split("/")[-1]}.nc').unlink()
+    # Crop using lat/lon coordinates, in parallel
+    ds = ds.rio.clip_box(minx=lon_min, miny=lat_min, maxx=lon_max, maxy=lat_max)
 
     # Add comments
-    ds[var_name].attrs['comments'] = 'Cropped and reprojected to EPSG:4326 by helvecioblneto@gmail.com'
+    ds[var_name].attrs['comments'] = 'Cropped and reprojected to EPSG:4326 by goesgcp'
 
-    # # Save as netcdf
-    ds.to_netcdf(f'{output}{file.split("/")[-1]}')
+    # Add global metadata comments
+    ds.attrs['comments'] = "Data processed by goesgcp, author: Helvecio B. L. Neto (helvecioblneto@gmail.com)"
+        
+    # Save as netcdf overwriting the original file
+    ds.to_netcdf(f'{output}{file.split("/")[-1]}', mode='w', format='NETCDF4_CLASSIC')
 
-    # Remove original file
-    pathlib.Path(file).unlink()
+    # Close the dataset
+    ds.close()
 
     return
 
 
 
+def download_file(args):
+    """Downloads a file from a GCP bucket."""
+
+    bucket_name, blob_name, local_path = args
+
+    # Create a client
+    bucket = storage_client.bucket(bucket_name)
+    blob = bucket.blob(blob_name)
+
+    # Download the file
+    blob.download_to_filename(local_path, timeout=120)
+
+
 def main():
 
     global output_path, var_name, \
           lat_min, lat_max, lon_min, lon_max, \
-          max_attempts, parallel, recent, resolution
+          max_attempts, parallel, recent, resolution, storage_client
 
     epilog = """
     Example usage:
     
-    - To download recent files from the GOES-16 satellite for the ABI-L2-CMIPF product, extracting the CMI variable from channel 13, in the last 30 minutes:
+    - To download recent 10 files from the GOES-16 satellite for the ABI-L2-CMIPF product:
 
-    goesgcp --satellite goes16 --product ABI-L2-CMIP --domain F --var_name CMI --channel 13 --recent 10 --output_path "output/"
+    goesgcp --satellite goes16 --product ABI-L2-CMIP --recent 10 --output_path "output/"
     """
 
 
@@ -146,11 +196,11 @@ def main():
     parser.add_argument('--recent', type=int, default=3, help='Number of recent files to download')
 
     # Geographic bounding box
-    parser.add_argument('--lat_min', type=float, default=-56, help='Minimum latitude of the bounding box')
-    parser.add_argument('--lat_max', type=float, default=35, help='Maximum latitude of the bounding box')
-    parser.add_argument('--lon_min', type=float, default=-116, help='Minimum longitude of the bounding box')
-    parser.add_argument('--lon_max', type=float, default=-25, help='Maximum longitude of the bounding box')
-    parser.add_argument('--resolution', type=float, default=0.045, help='Resolution of the output file')
+    parser.add_argument('--lat_min', type=float, default=-81.3282, help='Minimum latitude of the bounding box')
+    parser.add_argument('--lat_max', type=float, default=81.3282, help='Maximum latitude of the bounding box')
+    parser.add_argument('--lon_min', type=float, default=-156.2995, help='Minimum longitude of the bounding box')
+    parser.add_argument('--lon_max', type=float, default=6.2995, help='Maximum longitude of the bounding box')
+    parser.add_argument('--resolution', type=float, default=0.03208, help='Resolution of the output file')
     parser.add_argument('--output', type=str, default='output/', help='Path for saving output files')
 
     # Other settings
@@ -205,28 +255,36 @@ def main():
     if not recent_files:
         print(f"No files found with the pattern {pattern}. Exiting...")
         sys.exit(1)
-    print('Downloading files...')
-    # Loading bar
+
+    # Create a temporary directory
+    pathlib.Path('tmp/').mkdir(parents=True, exist_ok=True)
+
+    # Download files
+    print(f"Downloading {len(recent_files)} files...")
     loading_bar = tqdm.tqdm(total=len(recent_files), ncols=100, position=0, leave=True,
                         bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} + \
                         [Elapsed:{elapsed} Remaining:<{remaining}]')
     
-    # Create a temporary directory
-    pathlib.Path('tmp/').mkdir(parents=True, exist_ok=True)
-
     # Download all files to a temporary directory
     with ThreadPoolExecutor(max_workers=args.processes) as executor:
         for file in recent_files:
-            download_file(storage_client, bucket_name, file, f'tmp/{file.split("/")[-1]}')
+            local_path = f"tmp/{file.split('/')[-1]}"
+            executor.submit(download_file, (bucket_name, file, local_path))
             loading_bar.update(1)
     loading_bar.close()
 
-    print('Cropping and reprojecting files...')
-    # Crop and reproject all files in serial mode
-    for file in recent_files:
-        crop_reproject(f'tmp/{file.split("/")[-1]}', output_path)
-        loading_bar.update(1)
-    loading_bar.close()
+    # Process files
+    print(f"\nProcessing {len(recent_files)} files...")
+    load_bar2 = tqdm.tqdm(total=len(recent_files), ncols=100, position=0, leave=True,
+                        bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} + \
+                        [Elapsed:{elapsed} Remaining:<{remaining}]')
+
+    
+    # Process files in parallel
+    with Pool(processes=args.processes) as pool:
+        for _ in pool.imap_unordered(crop_reproject, [(f"tmp/{file.split('/')[-1]}", output_path) for file in recent_files]):
+            load_bar2.update(1)
+    load_bar2.close()
 
     # Remove temporary directory
     shutil.rmtree('tmp/')

{goesgcp-1.0.4.dist-info → goesgcp-1.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: goesgcp
-Version: 1.0.4
+Version: 1.0.6
 Summary: A package to download and process GOES-16/17 data
 Home-page: https://github.com/helvecioneto/goesgcp
 Author: Helvecio B. L. Neto
@@ -46,28 +46,26 @@ The script uses the `argparse` module for handling command-line arguments. Below
 goesgcp [OPTIONS]
 ```
 
-| Option               | Description                                                                 |
-|----------------------|-----------------------------------------------------------------------------|
-| `--satellite`         | Name of the satellite (e.g., goes16).                                      |
-| `--product`           | Name of the satellite product (e.g., ABI-L2-CMIPF). |
-| `--var_name`          | Variable name to extract (e.g., CMI).                                       |
-| `--channel`           | Channel to use (e.g., 13).                                                   |                      |
-| `--between_minutes`   | Filter data between these minutes (default: `[0, 60]`).                    |
-| `--output_path`       | Path for saving output files (default: `output/`).                           |                  |
-| `--lat_min`           | Minimum latitude of the bounding box (default: `-56`).                      |
-| `--lat_max`           | Maximum latitude of the bounding box (default: `35`).                       |
-| `--lon_min`           | Minimum longitude of the bounding box (default: `-116`).                    |
-| `--lon_max`           | Maximum longitude of the bounding box (default: `-25`).                     |
-| `--max_attempts`      | Number of attempts to download a file before logging a failure (default: `3`).|
+| Option               | Description                                                                |
+|----------------------|----------------------------------------------------------------------------|
+| `--satellite`         | Name of the satellite (e.g., goes16).                                     |
+| `--product`           | Name of the satellite product (e.g., ABI-L2-CMIPF).                       |
+| `--var_name`          | Variable name to extract (e.g., CMI).                                     |
+| `--channel`           | Channel to use (e.g., 13).                                                |
+| `--output`            | Path for saving output files (default: `output/`).                        | 
+| `--lat_min`           | Minimum latitude of the bounding box (default: `-56`).                    |
+| `--lat_max`           | Maximum latitude of the bounding box (default: `35`).                     |
+| `--lon_min`           | Minimum longitude of the bounding box (default: `-116`).                  |
+| `--lon_max`           | Maximum longitude of the bounding box (default: `-25`).                   |
+| `--resolution`        | Set the reprojet data resolution in degree (default: `-0.045`).           |
 
 ### Examples
 
-To download and process recent data for the GOES-16 satellite, ABI-L2-CMIPF product, variable CMI, and channel 13, run the following command:
+To download most 3 recent data for the GOES-16 satellite, ABI-L2-CMIPF product, variable CMI, and channel 13, run the following command:
 
 ```bash
-goesgcp --satellite goes16 --product ABI-L2-CMIPF --var_name CMI --channel 13 --recent 3 --output_path "output/"
+goesgcp --satellite goes16 --product ABI-L2-CMIPF --var_name CMI --channel 13 --recent 3 --output "output/"
 ```
 
 ### Credits
-All the credit goes to the original author of the **goes2go** library.
 And this is a otimization by Helvecio Neto - 2025

goesgcp-1.0.6.dist-info/RECORD

@@ -0,0 +1,8 @@
+goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
+goesgcp/main.py,sha256=3cvmzeFyMh7smI3jkMnY0PUrird9i3lntkFt9c_xwZo,10462
+goesgcp-1.0.6.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
+goesgcp-1.0.6.dist-info/METADATA,sha256=-3huDGZYV4-ZFxmKQcU-1avXQJWPQW_RrIQ841XrlGI,2993
+goesgcp-1.0.6.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+goesgcp-1.0.6.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
+goesgcp-1.0.6.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
+goesgcp-1.0.6.dist-info/RECORD,,

goesgcp-1.0.4.dist-info/RECORD

@@ -1,8 +0,0 @@
-goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
-goesgcp/main.py,sha256=4ATwrk6E_mzyeiEDZ11f6z_1tbAHDvTHQa4Jvr4CqIQ,8745
-goesgcp-1.0.4.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
-goesgcp-1.0.4.dist-info/METADATA,sha256=6CjGqrBkRDVWISzTiaMZRL2O7NXInZtjNq4vXhH59mY,3219
-goesgcp-1.0.4.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-goesgcp-1.0.4.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
-goesgcp-1.0.4.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
-goesgcp-1.0.4.dist-info/RECORD,,