ngiab-data-preprocess 3.2.1__tar.gz → 3.2.3__tar.gz

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ngiab_data_preprocess
-Version: 3.2.1
+Version: 3.2.3
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/modules/data_processing/forcings.py

@@ -83,7 +83,7 @@ def get_index_chunks(data: xr.DataArray) -> list[tuple[int, int]]:
     # takes a data array and calculates the start and end index for each chunk
     # based on the available memory.
     array_memory_usage = data.nbytes
-    # free_memory = psutil.virtual_memory().available * 0.8 # 80% of available memory
+    free_memory = psutil.virtual_memory().available * 0.8  # 80% of available memory
     # limit the chunk to 20gb, makes things more stable
     free_memory = min(free_memory, 20 * 1024 * 1024 * 1024)
     num_chunks = ceil(array_memory_usage / free_memory)
@@ -223,7 +223,12 @@ def compute_zonal_stats(
             concatenated_da.to_dataset(name=variable).to_netcdf(forcings_dir/ "temp" / f"{variable}_{i}.nc")
         # Merge the chunks back together
         datasets = [xr.open_dataset(forcings_dir / "temp" / f"{variable}_{i}.nc") for i in range(len(time_chunks))]
-        xr.concat(datasets, dim="time").to_netcdf(forcings_dir / f"{variable}.nc")
+        result = xr.concat(datasets, dim="time")
+        result.to_netcdf(forcings_dir / f"{variable}.nc")
+        # close the datasets
+        result.close()
+        _ = [dataset.close() for dataset in datasets]
+
         for file in forcings_dir.glob("temp/*.nc"):
             file.unlink()
         progress.remove_task(chunk_task)
@@ -237,6 +242,7 @@ def compute_zonal_stats(
     )
     write_outputs(forcings_dir, variables)
 
+
 def write_outputs(forcings_dir, variables):
 
     # start a dask cluster if there isn't one already running
@@ -289,9 +295,9 @@ def write_outputs(forcings_dir, variables):
     final_ds["Time"].attrs["epoch_start"] = "01/01/1970 00:00:00" # not needed but suppresses the ngen warning
 
     final_ds.to_netcdf(output_folder / "forcings.nc", engine="netcdf4")
-    # delete the individual variable files
-    for file in forcings_dir.glob("*.nc"):
-        file.unlink()
+    # close the datasets
+    _ = [result.close() for result in results]
+    final_ds.close()
 
 
 def setup_directories(cat_id: str) -> file_paths:

ngiab_data_preprocess-3.2.3/modules/map_app/__init__.py

@@ -0,0 +1,28 @@
+from flask import Flask
+import logging
+from map_app.views import main, intra_module_db
+from data_sources.source_validation import validate_all
+
+with open("app.log", "w") as f:
+    f.write("")
+    f.write("Starting Application!\n")
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(name)-12s: %(levelname)s - %(message)s",
+    filename="app.log",
+    filemode="a",
+)  # Append mode
+# Example: Adding a console handler to root logger (optional)
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)  # Or any other level
+formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s")
+console_handler.setFormatter(formatter)
+logging.getLogger("").addHandler(console_handler)
+
+validate_all()
+
+app = Flask(__name__)
+app.register_blueprint(main)
+
+intra_module_db["app"] = app

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/modules/map_app/__main__.py

@@ -5,36 +5,8 @@ import webbrowser
 from threading import Timer
 
 from data_processing.file_paths import file_paths
-from data_sources.source_validation import validate_all
 from data_processing.graph_utils import get_graph
-from flask import Flask
-
-from .views import intra_module_db, main
-
-validate_all()
-
-with open("app.log", "w") as f:
-    f.write("")
-    f.write("Starting Application!\n")
-
-
-app = Flask(__name__)
-app.register_blueprint(main)
-
-intra_module_db["app"] = app
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(name)-12s: %(levelname)s - %(message)s",
-    filename="app.log",
-    filemode="a",
-)  # Append mode
-# Example: Adding a console handler to root logger (optional)
-console_handler = logging.StreamHandler()
-console_handler.setLevel(logging.INFO)  # Or any other level
-formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s")
-console_handler.setFormatter(formatter)
-logging.getLogger("").addHandler(console_handler)
+from map_app import app, console_handler
 
 
 def open_browser():
@@ -59,8 +31,7 @@ def set_logs_to_warning():
     console_handler.setLevel(logging.DEBUG)
 
 
-if __name__ == "__main__":
-
+def main():
     # call this once to cache the graph
     Timer(1, get_graph).start()
 
@@ -74,3 +45,6 @@ if __name__ == "__main__":
         with open("app.log", "a") as f:
             f.write("Running in production mode\n")
         app.run(host="0.0.0.0", port="0")
+
+if __name__ == "__main__":
+    main()

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/modules/map_app/views.py

@@ -1,6 +1,7 @@
 import json
 import logging
 from datetime import datetime
+from pathlib import Path
 
 from data_processing.create_realization import create_realization
 from data_processing.file_paths import file_paths
@@ -43,7 +44,7 @@ def subset_selection():
     subset_name = cat_ids[0]
     run_paths = file_paths(subset_name)
     subset(cat_ids, output_gpkg_path=run_paths.geopackage_path)
-    return run_paths.geopackage_path, 200
+    return str(run_paths.geopackage_path), 200
 
 
 @main.route("/subset_to_file", methods=["POST"])
@@ -65,7 +66,9 @@ def subset_to_file():
 def get_forcings():
     # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}),
     data = json.loads(request.data.decode("utf-8"))
-    cat_id = data.get("forcing_dir").split("/")[-1]
+    subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1])
+    output_folder = subset_gpkg.parent.parent.stem
+
     start_time = data.get("start_time")
     end_time = data.get("end_time")
     # get the forcings
@@ -77,7 +80,7 @@ def get_forcings():
     app.debug = False
     logger.info(f"get_forcings() disabled debug mode at {datetime.now()}")
     try:
-        create_forcings(start_time, end_time, cat_id)
+        create_forcings(start_time, end_time, output_folder)
     except Exception as e:
         logger.info(f"get_forcings() failed with error: {str(e)}")
         return jsonify({"error": str(e)}), 500
@@ -90,13 +93,14 @@ def get_forcings():
 def get_realization():
     # body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}),
     data = json.loads(request.data.decode("utf-8"))
-    cat_id = data.get("forcing_dir").split("/")[-1]
+    subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1])
+    output_folder = subset_gpkg.parent.parent.stem
     start_time = data.get("start_time")
     end_time = data.get("end_time")
     # get the forcings
     start_time = datetime.strptime(start_time, "%Y-%m-%dT%H:%M")
     end_time = datetime.strptime(end_time, "%Y-%m-%dT%H:%M")
-    create_realization(cat_id, start_time, end_time)
+    create_realization(output_folder, start_time, end_time)
     return "success", 200
 
 

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/modules/ngiab_data_preprocess.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ngiab_data_preprocess
-Version: 3.2.1
+Version: 3.2.3
 Summary: Graphical Tools for creating Next Gen Water model input data.
 Author-email: Josh Cunningham <jcunningham8@ua.edu>
 Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/modules/ngiab_data_preprocess.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ modules/data_sources/noah-owp-modular-init.namelist.input
 modules/data_sources/source_validation.py
 modules/data_sources/template.sql
 modules/data_sources/triggers.sql
+modules/map_app/__init__.py
 modules/map_app/__main__.py
 modules/map_app/views.py
 modules/map_app/static/css/console.css
@@ -42,6 +43,7 @@ modules/ngiab_data_cli/custom_logging.py
 modules/ngiab_data_preprocess.egg-info/PKG-INFO
 modules/ngiab_data_preprocess.egg-info/SOURCES.txt
 modules/ngiab_data_preprocess.egg-info/dependency_links.txt
+modules/ngiab_data_preprocess.egg-info/entry_points.txt
 modules/ngiab_data_preprocess.egg-info/requires.txt
 modules/ngiab_data_preprocess.egg-info/top_level.txt
 output/.gitkeep

ngiab_data_preprocess-3.2.3/modules/ngiab_data_preprocess.egg-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+cli = ngiab_data_cli.__main__:main
+map_app = map_app.__main__:main

{ngiab_data_preprocess-3.2.1 → ngiab_data_preprocess-3.2.3}/pyproject.toml

@@ -12,7 +12,7 @@ exclude = ["tests*"]
 
 [project]
 name = "ngiab_data_preprocess"
-version = "v3.2.1"
+version = "v3.2.3"
 authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
 description = "Graphical Tools for creating Next Gen Water model input data."
 readme = "README.md"
@@ -54,6 +54,10 @@ plot = ["ngiab_eval[plot]"]
 Homepage = "https://github.com/CIROH-UA/NGIAB_data_preprocess"
 Issues = "https://github.com/CIROH-UA/NGIAB_data_preprocess/issues"
 
+[project.scripts]
+cli = "ngiab_data_cli.__main__:main"
+map_app = "map_app.__main__:main"
+
 [build-system]
 # scm adds files tracked by git to the package
 requires = ["setuptools>=69.0", "setuptools-scm>=8.0"]