ngiab-data-preprocess 4.0.2__py3-none-any.whl → 4.0.4__py3-none-any.whl

data_sources/source_validation.py

@@ -3,26 +3,30 @@ import os
 import tarfile
 import warnings
 import json
-from concurrent.futures import ThreadPoolExecutor
 import requests
 from data_processing.file_paths import file_paths
 from tqdm import TqdmExperimentalWarning
-from tqdm.rich import tqdm
 from time import sleep
+import boto3
+from botocore.exceptions import ClientError
+from boto3.s3.transfer import TransferConfig
 from rich.console import Console
 from rich.prompt import Prompt
-from rich.progress import Progress, BarColumn, TextColumn, TimeElapsedColumn, SpinnerColumn, TimeRemainingColumn, DownloadColumn, TransferSpeedColumn
-import threading
+from rich.progress import Progress, TextColumn, TimeElapsedColumn, SpinnerColumn
 import psutil
 
 warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
 
 console = Console()
-
+S3_BUCKET = "communityhydrofabric"
+S3_KEY = "hydrofabrics/community/conus_nextgen.tar.gz"
+S3_REGION = "us-east-1"
+hydrofabric_url = f"https://{S3_BUCKET}.s3.{S3_REGION}.amazonaws.com/{S3_KEY}"
 
 def decompress_gzip_tar(file_path, output_dir):
     # use rich to display "decompressing" message with a progress bar that just counts down from 30s
     # actually measuring this is hard and it usually takes ~20s to decompress
+    console.print("Decompressing Hydrofabric...", style="bold green")
     progress = Progress(
         SpinnerColumn(),
         TextColumn("[progress.description]{task.description}"),
@@ -40,79 +44,71 @@ def decompress_gzip_tar(file_path, output_dir):
     progress.stop()
 
 
-def download_chunk(url, start, end, index, save_path):
-    headers = {"Range": f"bytes={start}-{end}"}
-    response = requests.get(url, headers=headers, stream=True)
-    chunk_path = f"{save_path}.part{index}"
-    # store the response in memory rather than streaming to disk
-    # OSX has a limit of 256 open files so this is a workaround
-    response_bytes = bytes()
-    for chunk in response.iter_content(chunk_size=8 * 1024):
-        response_bytes += chunk
-    with open(chunk_path, "wb") as f_out:
-        f_out.write(response_bytes)
-    return chunk_path
-
-def download_progress_estimate(progress, task, total_size):
-    network_bytes_start = psutil.net_io_counters().bytes_recv
-    # make a new progress bar that will be updated by a separate thread
-    progress.start()
-    interval = 0.5
-    while not progress.finished:
-        current_downloaded = psutil.net_io_counters().bytes_recv
-        total_downloaded = current_downloaded - network_bytes_start
-        progress.update(task, completed=total_downloaded)
-        sleep(interval)
-        if total_downloaded >= total_size or progress.finished:
-            break
-    progress.stop()
-
-
-def download_file(url, save_path, num_threads=150):
+def download_from_s3(save_path, bucket=S3_BUCKET, key=S3_KEY, region=S3_REGION):
+    """Download file from S3 with optimal multipart configuration"""
     if not os.path.exists(os.path.dirname(save_path)):
         os.makedirs(os.path.dirname(save_path))
 
-    response = requests.head(url)
-    total_size = int(response.headers.get("content-length", 0))
-    chunk_size = total_size // num_threads
-
-    progress = Progress(
-            TextColumn("[progress.description]{task.description}"),
-            BarColumn(),
-            DownloadColumn(),
-            TransferSpeedColumn(),
-            TextColumn(" Elapsed Time:"),
-            TimeElapsedColumn(),
-            TextColumn(" Remaining Time:"),
-            TimeRemainingColumn(),
-        )
-    task = progress.add_task("Downloading", total=total_size)
-
-    download_progress_thread = threading.Thread(target=download_progress_estimate, args=(progress, task ,total_size))
-    download_progress_thread.start()
-
-    with ThreadPoolExecutor(max_workers=num_threads) as executor:
-        futures = []
-        for i in range(num_threads):
-            start = i * chunk_size
-            end = start + chunk_size - 1 if i < num_threads - 1 else total_size - 1
-            futures.append(executor.submit(download_chunk, url, start, end, i, save_path))
+    # Check if file already exists
+    if os.path.exists(save_path):
+        console.print(f"File already exists: {save_path}", style="bold yellow")
+        os.remove(save_path)
 
-        chunk_paths = [
-            future.result() for future in futures
-        ]
-
-    with open(save_path, "wb") as f_out:
-        for chunk_path in chunk_paths:
-            with open(chunk_path, "rb") as f_in:
-                f_out.write(f_in.read())
-            os.remove(chunk_path)
+    # Initialize S3 client
+    s3_client = boto3.client(
+        "s3", aws_access_key_id="", aws_secret_access_key="", region_name=region
+    )
+    # Disable request signing for public buckets
+    s3_client._request_signer.sign = lambda *args, **kwargs: None
 
-    progress.update(task, completed=total_size)
-    download_progress_thread.join()
+    # Get object size
+    try:
+        response = s3_client.head_object(Bucket=bucket, Key=key)
+        total_size = int(response.get("ContentLength", 0))
+    except ClientError as e:
+        console.print(f"Error getting object info: {e}", style="bold red")
+        return False
+
+    # Configure transfer settings for maximum speed
+    # Use more CPU cores for parallel processing
+    cpu_count = os.cpu_count() or 8
+    max_threads = cpu_count * 4
+
+    # Optimize chunk size based on file size and available memory
+    memory = psutil.virtual_memory()
+    available_mem_mb = memory.available / (1024 * 1024)
+
+    # Calculate optimal chunk size (min 8MB, max 100MB)
+    # Larger files get larger chunks for better throughput
+    optimal_chunk_mb = min(max(8, total_size / (50 * 1024 * 1024)), 100)
+    # Ensure we don't use too much memory
+    optimal_chunk_mb = min(optimal_chunk_mb, available_mem_mb / (max_threads * 2))
+
+    # Create transfer config
+    config = TransferConfig(
+        # multipart_threshold=8 * 1024 * 1024,  # 8MB
+        max_concurrency=max_threads,
+        multipart_chunksize=int(optimal_chunk_mb * 1024 * 1024),
+        use_threads=True,
+    )
 
+    console.print(f"Downloading {key} to {save_path}...", style="bold green")
+    console.print(
+        f"The file downloads faster with no progress indicator, this should take around 30s",
+        style="bold yellow",
+    )
+    console.print(
+        f"Please use network monitoring on your computer if you wish to track the download",
+        style="green",
+    )
 
-hydrofabric_url = "https://communityhydrofabric.s3.us-east-1.amazonaws.com/hydrofabrics/community/conus_nextgen.tar.gz"
+    try:
+        # Download file using optimized transfer config
+        s3_client.download_file(Bucket=bucket, Key=key, Filename=save_path, Config=config)
+        return True
+    except Exception as e:
+        console.print(f"Error downloading file: {e}", style="bold red")
+        return False
 
 
 def get_headers():
@@ -126,7 +122,11 @@ def get_headers():
 
 
 def download_and_update_hf():
-    download_file(hydrofabric_url, file_paths.conus_hydrofabric.with_suffix(".tar.gz"))
+    download_from_s3(
+        file_paths.conus_hydrofabric.with_suffix(".tar.gz"),
+        bucket="communityhydrofabric",
+        key="hydrofabrics/community/conus_nextgen.tar.gz",
+    )
     status, headers = get_headers()
 
     if status == 200:

ngiab_data_cli/__main__.py

@@ -194,12 +194,12 @@ def main() -> None:
                 num_partitions = int(f.read())
 
             try:
-                subprocess.run("docker pull joshcu/ngiab", shell=True)
+                subprocess.run("docker pull awiciroh/ciroh-ngen-image:latest", shell=True)
             except:
                 logging.error("Docker is not running, please start Docker and try again.")
             try:
-                command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
-                # command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest-x86 /ngen/ngen/data/ auto {num_partitions}'
+                #command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" joshcu/ngiab /ngen/ngen/data/ auto {num_partitions} local'
+                command = f'docker run --rm -it -v "{str(paths.subset_dir)}:/ngen/ngen/data" awiciroh/ciroh-ngen-image:latest /ngen/ngen/data/ auto {num_partitions} local'
                 subprocess.run(command, shell=True)
                 logging.info("Next Gen run complete.")
             except:

{ngiab_data_preprocess-4.0.2.dist-info → ngiab_data_preprocess-4.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ngiab_data_preprocess
-Version: 4.0.2
+Version: 4.0.4
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
@@ -32,6 +32,7 @@ Requires-Dist: tqdm==4.66.4
 Requires-Dist: rich==13.7.1
 Requires-Dist: colorama==0.4.6
 Requires-Dist: bokeh==3.5.1
+Requires-Dist: boto3
 Provides-Extra: eval
 Requires-Dist: ngiab_eval; extra == "eval"
 Provides-Extra: plot

{ngiab_data_preprocess-4.0.2.dist-info → ngiab_data_preprocess-4.0.4.dist-info}/RECORD

@@ -15,7 +15,7 @@ data_sources/em-realization-template.json,sha256=DJvB7N8lCeS2vLFenmbTzysBDR-xPaJ
 data_sources/forcing_template.nc,sha256=uRuVAqX3ngdlougZINavtwl_wC2VLD8fHqG7_CLim1s,85284
 data_sources/ngen-routing-template.yaml,sha256=RV28MAbyQNx9U8FAYmZhD2Fv8Yu6o_08Ekoc77KNdH4,4622
 data_sources/noah-owp-modular-init.namelist.input,sha256=Vb7mp40hFpJogruOrXrDHwVW1bKi9h1ciDNyDvTzn20,3045
-data_sources/source_validation.py,sha256=0PZts0wUr7nnR1cc0xOpoFMYj-8MCGO2vYc79NU3T_c,8052
+data_sources/source_validation.py,sha256=9mC1LiL8TXh0mtHFwOlFkT2jN1DEorgw_K7qccrdFfA,8038
 data_sources/template.sql,sha256=ZnFqAqleEq9wgmAhNO90Wue_L9k0JAn8KF99DYtcxgs,10457
 data_sources/triggers.sql,sha256=G0d_175eNsamKAFhsbphPATvzMPuPL_iCleIhlToduQ,14906
 map_app/__init__.py,sha256=OarJao9X98kcbLyiwewN4ObWNAYkKDichcxbuWywTsA,818
@@ -32,13 +32,13 @@ map_app/static/resources/light-style.json,sha256=DaE52qKpAkjiWSKY_z7LxreqA2rW4Zy
 map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
 map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
 map_app/templates/index.html,sha256=OAPh-EMlk3cAQqFwhSw5oPjUu6PS-H0oLfhyHmnI0_4,6629
-ngiab_data_cli/__main__.py,sha256=2pn-U6G62dud3s8h7N5U_tQ36u7mfwDwwISzD8pAp2M,10143
+ngiab_data_cli/__main__.py,sha256=_m0bhAXdyLeZBuh03PKNXlJ6BUJnZDR-xZIbFyRH8iE,10164
 ngiab_data_cli/arguments.py,sha256=7ptImy8tLM1XvjfDr13tZszkjGVtenXo0KqllJeE3Mw,4372
 ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
 ngiab_data_cli/forcing_cli.py,sha256=lkcqWDk5H8IPyGv0DwLIZMQldqTUXpfwSX0C_RIuIJ8,3890
-ngiab_data_preprocess-4.0.2.dist-info/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
-ngiab_data_preprocess-4.0.2.dist-info/METADATA,sha256=gXvFqgKRdeVmYmsKSOM4zV9G21TikmI0T9fjcT_m4bE,9390
-ngiab_data_preprocess-4.0.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ngiab_data_preprocess-4.0.2.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
-ngiab_data_preprocess-4.0.2.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
-ngiab_data_preprocess-4.0.2.dist-info/RECORD,,
+ngiab_data_preprocess-4.0.4.dist-info/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
+ngiab_data_preprocess-4.0.4.dist-info/METADATA,sha256=Rgmbfo8XC_T9Vw6zUfHGFl2du6HBmL2WCR8S0ds5T2k,9411
+ngiab_data_preprocess-4.0.4.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+ngiab_data_preprocess-4.0.4.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
+ngiab_data_preprocess-4.0.4.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
+ngiab_data_preprocess-4.0.4.dist-info/RECORD,,

{ngiab_data_preprocess-4.0.2.dist-info → ngiab_data_preprocess-4.0.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any