ngiab-data-preprocess 3.2.0__tar.gz → 3.2.2__tar.gz

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ngiab_data_preprocess
-Version: 3.2.0
+Version: 3.2.2
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/modules/data_processing/create_realization.py

@@ -77,7 +77,7 @@ def make_cfe_config(
             slope=row["mean.slope"],
             smcmax=row["mean.smcmax_soil_layers_stag=2"],
             smcwlt=row["mean.smcwlt_soil_layers_stag=2"],
-            max_gw_storage=row["mean.Zmax"] if row["mean.Zmax"] is not None else "0.011[m]",
+            max_gw_storage=row["mean.Zmax"]/1000 if row["mean.Zmax"] is not None else "0.011[m]", # mean.Zmax is in mm!
             gw_Coeff=row["mean.Coeff"] if row["mean.Coeff"] is not None else "0.0018[m h-1]",
             gw_Expon=row["mode.Expon"],
             gw_storage="{:.5}".format(gw_storage_ratio),

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/modules/data_processing/forcings.py

@@ -10,6 +10,8 @@ from math import ceil
 from multiprocessing import shared_memory
 from pathlib import Path
 
+from dask.distributed import Client, LocalCluster
+
 import geopandas as gpd
 import numpy as np
 import pandas as pd
@@ -68,10 +70,12 @@ def get_cell_weights(raster, gdf, wkt):
 
 
 def add_APCP_SURFACE_to_dataset(dataset: xr.Dataset) -> xr.Dataset:
-    # precip rate is mm/s
-    # cfe says input m/h
-    dataset["APCP_surface"] = dataset["precip_rate"] * 3600 / 1000
-    # technically should be kg/m^2/h, at 1kg = 1l it equates to mm/h
+    # precip_rate is mm/s
+    # cfe says input atmosphere_water__liquid_equivalent_precipitation_rate is mm/h
+    # nom says prcpnonc input is mm/s
+    # technically should be kg/m^2/s at 1kg = 1l it equates to mm/s
+    # nom says qinsur output is m/s, hopefully qinsur is converted to mm/h by ngen
+    dataset["APCP_surface"] = dataset["precip_rate"] * 3600
     return dataset
 
 
@@ -79,7 +83,9 @@ def get_index_chunks(data: xr.DataArray) -> list[tuple[int, int]]:
     # takes a data array and calculates the start and end index for each chunk
     # based on the available memory.
     array_memory_usage = data.nbytes
-    free_memory = psutil.virtual_memory().available * 0.8 # 80% of available memory
+    # free_memory = psutil.virtual_memory().available * 0.8 # 80% of available memory
+    # limit the chunk to 20gb, makes things more stable
+    free_memory = min(free_memory, 20 * 1024 * 1024 * 1024)
     num_chunks = ceil(array_memory_usage / free_memory)
     max_index = data.shape[0]
     stride = max_index // num_chunks
@@ -233,8 +239,15 @@ def compute_zonal_stats(
 
 def write_outputs(forcings_dir, variables):
 
-    # Combine all variables into a single dataset
-    results = [xr.open_dataset(file) for file in forcings_dir.glob("*.nc")]
+    # start a dask cluster if there isn't one already running
+    try:
+        client = Client.current()
+    except ValueError:
+        cluster = LocalCluster()
+        client = Client(cluster)
+
+    # Combine all variables into a single dataset using dask
+    results = [xr.open_dataset(file, chunks="auto") for file in forcings_dir.glob("*.nc")]
     final_ds = xr.merge(results)
 
     output_folder = forcings_dir / "by_catchment"

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/modules/data_processing/s3fs_utils.py

@@ -36,7 +36,7 @@ class S3ParallelFileSystem(S3FileSystem):
             # Fall back to single request if HEAD fails
             return await self._download_chunk(bucket, key, {}, version_kw)
 
-        CHUNK_SIZE = 1 * 1024 * 1024  # 1MB chunks
+        CHUNK_SIZE = 5 * 1024 * 1024  # 1MB chunks
         if obj_size <= CHUNK_SIZE:
             return await self._download_chunk(bucket, key, {}, version_kw)
 

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/modules/ngiab_data_preprocess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ngiab_data_preprocess
-Version: 3.2.0
+Version: 3.2.2
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/modules/ngiab_data_preprocess.egg-info/SOURCES.txt

@@ -42,6 +42,7 @@ modules/ngiab_data_cli/custom_logging.py
 modules/ngiab_data_preprocess.egg-info/PKG-INFO
 modules/ngiab_data_preprocess.egg-info/SOURCES.txt
 modules/ngiab_data_preprocess.egg-info/dependency_links.txt
+modules/ngiab_data_preprocess.egg-info/entry_points.txt
 modules/ngiab_data_preprocess.egg-info/requires.txt
 modules/ngiab_data_preprocess.egg-info/top_level.txt
 output/.gitkeep

ngiab_data_preprocess-3.2.2/modules/ngiab_data_preprocess.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+cli = ngiab_data_cli.__main__:main
+map = map_app.__main__:main

{ngiab_data_preprocess-3.2.0 → ngiab_data_preprocess-3.2.2}/pyproject.toml

@@ -12,7 +12,7 @@ exclude = ["tests*"]
 
 [project]
 name = "ngiab_data_preprocess"
-version = "v3.2.0"
+version = "v3.2.2"
 authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
 description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"
@@ -54,6 +54,10 @@ plot = ["ngiab_eval[plot]"]
 Homepage = "https://github.com/CIROH-UA/NGIAB_data_preprocess"
 Issues = "https://github.com/CIROH-UA/NGIAB_data_preprocess/issues"
 
+[project.scripts]
+cli = "ngiab_data_cli.__main__:main"
+map = "map_app.__main__:main"
+
 [build-system]
 # scm adds files tracked by git to the package
 requires = ["setuptools>=69.0", "setuptools-scm>=8.0"]