goesgcp 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

goesgcp/main.py

@@ -4,6 +4,8 @@ import xarray as xr
 import argparse
 import sys
 import tqdm
+from distutils.util import strtobool
+from concurrent.futures import ThreadPoolExecutor
 from multiprocessing import Pool
 from google.cloud import storage
 from datetime import datetime, timedelta, timezone
@@ -65,10 +67,13 @@ def get_recent_files(connection, bucket_name, base_prefix, pattern, min_files):
     return [file[0] for file in files[:min_files]]
 
 
-def crop_reproject(file, output):
+def crop_reproject(args):
     """
     Crops and reprojects a GOES-16 file to EPSG:4326.
     """
+
+    file, output = args
+
     # Open the file
     ds = xr.open_dataset(file, engine='netcdf4')
 
@@ -162,13 +167,6 @@ def download_file(args):
     # Download the file
     blob.download_to_filename(local_path, timeout=120)
 
-    # Crop and reproject the file
-    crop_reproject(local_path, output_path)
-
-    # Remove the file
-    pathlib.Path(local_path).unlink()
-
-
 
 def main():
 
@@ -207,16 +205,16 @@ def main():
     parser.add_argument('--output', type=str, default='output/', help='Path for saving output files')
 
     # Other settings
-    parser.add_argument('--parallel', type=bool, default=True, help='Use parallel processing')
+    parser.add_argument('--parallel', type=lambda x: bool(strtobool(x)), default=True, help='Use parallel processing')
     parser.add_argument('--processes', type=int, default=4, help='Number of processes for parallel execution')
     parser.add_argument('--max_attempts', type=int, default=3, help='Number of attempts to download a file')
 
     # Parse arguments
     args = parser.parse_args()
 
-    if len(sys.argv) == 1:
-        parser.print_help(sys.stderr)
-        sys.exit(1)
+    # if len(sys.argv) == 1:
+    #     parser.print_help(sys.stderr)
+    #     sys.exit(1)
 
     # Set global variables
     output_path = args.output
@@ -262,18 +260,42 @@ def main():
     # Create a temporary directory
     pathlib.Path('tmp/').mkdir(parents=True, exist_ok=True)
 
-    print(f"Downloading and processing {len(recent_files)} files...")
-
-    # Process files in parallel
+    # Download files
+    print(f"Downloading {len(recent_files)} files...")
     loading_bar = tqdm.tqdm(total=len(recent_files), ncols=100, position=0, leave=True,
                         bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} + \
                         [Elapsed:{elapsed} Remaining:<{remaining}]')
     
-    # Download all files to a temporary directory
-    with Pool(processes=args.processes) as pool:
-        for _ in pool.imap_unordered(download_file, [(bucket_name, 
-                                                      file, f'tmp/{file.split("/")[-1]}') for file in recent_files]):
+    if parallel:
+        # Download all files to a temporary directory
+        with ThreadPoolExecutor(max_workers=args.processes) as executor:
+            for file in recent_files:
+                local_path = f"tmp/{file.split('/')[-1]}"
+                executor.submit(download_file, (bucket_name, file, local_path))
+                loading_bar.update(1)
+        loading_bar.close()
+
+        # Process files
+        print(f"\nProcessing {len(recent_files)} files...")
+        load_bar2 = tqdm.tqdm(total=len(recent_files), ncols=100, position=0, leave=True,
+                            bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} + \
+                            [Elapsed:{elapsed} Remaining:<{remaining}]')
+
+        
+        # Process files in parallel
+        with Pool(processes=args.processes) as pool:
+            for _ in pool.imap_unordered(crop_reproject, [(f"tmp/{file.split('/')[-1]}", output_path) for file in recent_files]):
+                load_bar2.update(1)
+        load_bar2.close()
+    else:
+        for file in recent_files:
+            local_path = f"tmp/{file.split('/')[-1]}"
+            download_file((bucket_name, file, local_path))
+            crop_reproject((local_path, output_path))
             loading_bar.update(1)
+        loading_bar.close()
+
+
 
     # Remove temporary directory
     shutil.rmtree('tmp/')

{goesgcp-1.0.5.dist-info → goesgcp-1.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: goesgcp
-Version: 1.0.5
+Version: 1.0.7
 Summary: A package to download and process GOES-16/17 data
 Home-page: https://github.com/helvecioneto/goesgcp
 Author: Helvecio B. L. Neto

goesgcp-1.0.7.dist-info/RECORD

@@ -0,0 +1,8 @@
+goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
+goesgcp/main.py,sha256=zCrAfAjc9Vzs9-7klATCCp6hhe1w4tsu_FiMBuBAafc,10893
+goesgcp-1.0.7.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
+goesgcp-1.0.7.dist-info/METADATA,sha256=eXzz34_yexZ4zx4EmaDRrpwiHFoBtJBzJ_hnVLQD-4Q,2993
+goesgcp-1.0.7.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+goesgcp-1.0.7.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
+goesgcp-1.0.7.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
+goesgcp-1.0.7.dist-info/RECORD,,

goesgcp-1.0.5.dist-info/RECORD

@@ -1,8 +0,0 @@
-goesgcp/__init__.py,sha256=MigXIT7A1M9YZuH2MyjKReSziFwzbZX2boVYsLosR6s,22
-goesgcp/main.py,sha256=mSHN0R0-xIJZkKo0U82dUWgFfX56KNqgiNv_YefdwVs,9967
-goesgcp-1.0.5.dist-info/LICENSE,sha256=AHeZifD4UyBZI61Ug5lETXgX3Anp_XfAvFXQqrW9AnU,1078
-goesgcp-1.0.5.dist-info/METADATA,sha256=LL90blorVD0KmRNCVYnycaAcTgGoiPgyVtJVkAhTPwI,2993
-goesgcp-1.0.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-goesgcp-1.0.5.dist-info/entry_points.txt,sha256=6afMW51WnUR9VZ_xvDoiB8JQb2OFiLuzRtV6dPL__OQ,46
-goesgcp-1.0.5.dist-info/top_level.txt,sha256=C-C3vipI0AwEDW9nWFkJ6D0TkcKkIYlyyM15LMskUEc,8
-goesgcp-1.0.5.dist-info/RECORD,,