ngiab-data-preprocess 4.5.0__tar.gz → 4.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/PKG-INFO +1 -1
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/create_realization.py +10 -8
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_preprocess.egg-info/PKG-INFO +1 -1
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/pyproject.toml +1 -1
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/.github/workflows/build_only.yml +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/.github/workflows/publish.yml +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/.gitignore +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/LICENSE +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/README.md +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/ciroh-bgsafe.png +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/dask_utils.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/dataset_utils.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/datasets.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/file_paths.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/forcings.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/gpkg_utils.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/graph_utils.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/s3fs_utils.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/subset.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/cfe-nowpm-realization-template.json +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/cfe-template.ini +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/forcing_template.nc +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/lstm-catchment-template.yml +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/lstm-realization-template.json +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/ngen-routing-template.yaml +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/noah-owp-modular-init.namelist.input +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/source_validation.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/template.sql +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/triggers.sql +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/__init__.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/__main__.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/css/console.css +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/css/main.css +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/css/toggle.css +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/js/console.js +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/js/data_processing.js +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/js/main.js +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/resources/loading.gif +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/resources/screenshot.jpg +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/templates/index.html +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/views.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/__main__.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/arguments.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/custom_logging.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/forcing_cli.py +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_preprocess.egg-info/SOURCES.txt +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_preprocess.egg-info/dependency_links.txt +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_preprocess.egg-info/entry_points.txt +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_preprocess.egg-info/requires.txt +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_preprocess.egg-info/top_level.txt +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/setup.cfg +0 -0
- {ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/tests/test_nan_impute.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ngiab_data_preprocess
|
|
3
|
-
Version: 4.5.
|
|
3
|
+
Version: 4.5.1
|
|
4
4
|
Summary: Graphical Tools for creating Next Gen Water model input data.
|
|
5
5
|
Author-email: Josh Cunningham <jcunningham8@ua.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
3
|
import multiprocessing
|
|
4
|
+
import os
|
|
4
5
|
import shutil
|
|
5
6
|
import sqlite3
|
|
6
7
|
from datetime import datetime
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import Dict, Optional
|
|
9
|
-
import psutil
|
|
10
|
-
import os
|
|
11
10
|
|
|
12
11
|
import numpy as np
|
|
13
12
|
import pandas
|
|
13
|
+
import psutil
|
|
14
14
|
import requests
|
|
15
15
|
import s3fs
|
|
16
16
|
import xarray as xr
|
|
@@ -169,7 +169,7 @@ def make_lstm_config(
|
|
|
169
169
|
lat=row["latitude"],
|
|
170
170
|
lon=row["longitude"],
|
|
171
171
|
slope_mean=row["mean_slope_mpkm"],
|
|
172
|
-
elevation_mean=row["mean.elevation"] /
|
|
172
|
+
elevation_mean=row["mean.elevation"] / 100, # convert cm in hf to m
|
|
173
173
|
)
|
|
174
174
|
)
|
|
175
175
|
|
|
@@ -180,14 +180,16 @@ def configure_troute(
|
|
|
180
180
|
with open(file_paths.template_troute_config, "r") as file:
|
|
181
181
|
troute_template = file.read()
|
|
182
182
|
time_step_size = 300
|
|
183
|
-
gpkg_file_path=f"{config_dir}/{cat_id}_subset.gpkg"
|
|
183
|
+
gpkg_file_path = f"{config_dir}/{cat_id}_subset.gpkg"
|
|
184
184
|
nts = (end_time - start_time).total_seconds() / time_step_size
|
|
185
185
|
with sqlite3.connect(gpkg_file_path) as conn:
|
|
186
186
|
ncats_df = pandas.read_sql_query("SELECT COUNT(id) FROM 'divides';", conn)
|
|
187
|
-
ncats = ncats_df[
|
|
187
|
+
ncats = ncats_df["COUNT(id)"][0]
|
|
188
188
|
|
|
189
|
-
est_bytes_required = nts * ncats * 45
|
|
190
|
-
local_ram_available =
|
|
189
|
+
est_bytes_required = nts * ncats * 45 # extremely rough calculation based on about 3 tests :)
|
|
190
|
+
local_ram_available = (
|
|
191
|
+
0.8 * psutil.virtual_memory().available
|
|
192
|
+
) # buffer to not accidentally explode machine
|
|
191
193
|
|
|
192
194
|
if est_bytes_required > local_ram_available:
|
|
193
195
|
max_loop_size = nts // (est_bytes_required // local_ram_available)
|
|
@@ -210,7 +212,7 @@ def configure_troute(
|
|
|
210
212
|
start_datetime=start_time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
211
213
|
nts=nts,
|
|
212
214
|
max_loop_size=max_loop_size,
|
|
213
|
-
binary_nexus_file_folder_comment=binary_nexus_file_folder_comment
|
|
215
|
+
binary_nexus_file_folder_comment=binary_nexus_file_folder_comment,
|
|
214
216
|
)
|
|
215
217
|
|
|
216
218
|
with open(config_dir / "troute.yaml", "w") as file:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ngiab_data_preprocess
|
|
3
|
-
Version: 4.5.
|
|
3
|
+
Version: 4.5.1
|
|
4
4
|
Summary: Graphical Tools for creating Next Gen Water model input data.
|
|
5
5
|
Author-email: Josh Cunningham <jcunningham8@ua.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
|
|
@@ -19,7 +19,7 @@ filterwarnings = [
|
|
|
19
19
|
]
|
|
20
20
|
[project]
|
|
21
21
|
name = "ngiab_data_preprocess"
|
|
22
|
-
version = "v4.5.
|
|
22
|
+
version = "v4.5.1"
|
|
23
23
|
authors = [{ name = "Josh Cunningham", email = "jcunningham8@ua.edu" }]
|
|
24
24
|
description = "Graphical Tools for creating Next Gen Water model input data."
|
|
25
25
|
readme = "README.md"
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/.github/workflows/build_only.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/dask_utils.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/dataset_utils.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/datasets.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/file_paths.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/forcings.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/gpkg_utils.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/graph_utils.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/s3fs_utils.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_processing/subset.py
RENAMED
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/cfe-template.ini
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/forcing_template.nc
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/template.sql
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/data_sources/triggers.sql
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/css/console.css
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/css/main.css
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/css/toggle.css
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/js/console.js
RENAMED
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/static/js/main.js
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/map_app/templates/index.html
RENAMED
|
File without changes
|
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/__main__.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/arguments.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/custom_logging.py
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.0 → ngiab_data_preprocess-4.5.1}/modules/ngiab_data_cli/forcing_cli.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|