ngiab-data-preprocess 4.5.1__py3-none-any.whl → 4.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_processing/create_realization.py +29 -14
- data_processing/dataset_utils.py +4 -86
- data_processing/file_paths.py +12 -5
- data_processing/forcings.py +70 -5
- data_processing/gpkg_utils.py +13 -15
- data_processing/graph_utils.py +5 -5
- data_processing/subset.py +4 -4
- data_sources/lstm-rust-realization-template.json +47 -0
- data_sources/source_validation.py +57 -45
- map_app/__main__.py +3 -2
- map_app/static/css/main.css +41 -0
- map_app/static/css/toggle.css +30 -1
- map_app/static/js/data_processing.js +101 -37
- map_app/static/js/main.js +51 -15
- map_app/templates/index.html +16 -1
- map_app/views.py +83 -34
- ngiab_data_cli/__main__.py +73 -47
- ngiab_data_cli/arguments.py +16 -3
- {ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/METADATA +1 -1
- ngiab_data_preprocess-4.6.0.dist-info/RECORD +43 -0
- ngiab_data_preprocess-4.5.1.dist-info/RECORD +0 -42
- {ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/WHEEL +0 -0
- {ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/entry_points.txt +0 -0
- {ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/licenses/LICENSE +0 -0
- {ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/top_level.txt +0 -0
map_app/views.py
CHANGED
|
@@ -2,12 +2,14 @@ import json
|
|
|
2
2
|
import logging
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from pathlib import Path
|
|
5
|
+
import os
|
|
6
|
+
import threading
|
|
5
7
|
|
|
6
8
|
import geopandas as gpd
|
|
7
9
|
from data_processing.create_realization import create_realization
|
|
8
10
|
from data_processing.dataset_utils import save_and_clip_dataset
|
|
9
11
|
from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
|
|
10
|
-
from data_processing.file_paths import
|
|
12
|
+
from data_processing.file_paths import FilePaths
|
|
11
13
|
from data_processing.forcings import create_forcings
|
|
12
14
|
from data_processing.graph_utils import get_upstream_cats, get_upstream_ids
|
|
13
15
|
from data_processing.subset import subset
|
|
@@ -23,24 +25,35 @@ logger = logging.getLogger(__name__)
|
|
|
23
25
|
def index():
|
|
24
26
|
return render_template("index.html")
|
|
25
27
|
|
|
26
|
-
|
|
28
|
+
# this subset does not include the downstream nexus
|
|
27
29
|
@main.route("/get_upstream_catids", methods=["POST"])
|
|
28
30
|
def get_upstream_catids():
|
|
29
31
|
cat_id = json.loads(request.data.decode("utf-8"))
|
|
30
32
|
# give wb_id to get_upstream_cats because the graph search is 1000x faster
|
|
31
33
|
wb_id = "wb-" + cat_id.split("-")[-1]
|
|
32
|
-
upstream_cats =
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
upstream_cats = get_upstream_ids(wb_id, include_outlet=False)
|
|
35
|
+
cleaned_upstreams = set()
|
|
36
|
+
for id in upstream_cats:
|
|
37
|
+
if id.startswith("wb-"):
|
|
38
|
+
cleaned_upstreams.add("cat-" + id.split("-")[-1])
|
|
39
|
+
if cat_id in cleaned_upstreams:
|
|
40
|
+
cleaned_upstreams.remove(cat_id)
|
|
41
|
+
return list(cleaned_upstreams), 200
|
|
42
|
+
|
|
43
|
+
# this subset includes the downstream nexus
|
|
38
44
|
@main.route("/get_upstream_wbids", methods=["POST"])
|
|
39
45
|
def get_upstream_wbids():
|
|
40
46
|
cat_id = json.loads(request.data.decode("utf-8"))
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
47
|
+
# give wb_id to get_upstream_cats because the graph search is 1000x faster
|
|
48
|
+
wb_id = "wb-" + cat_id.split("-")[-1]
|
|
49
|
+
upstream_cats = get_upstream_ids(wb_id)
|
|
50
|
+
cleaned_upstreams = set()
|
|
51
|
+
for id in upstream_cats:
|
|
52
|
+
if id.startswith("wb-"):
|
|
53
|
+
cleaned_upstreams.add("cat-" + id.split("-")[-1])
|
|
54
|
+
if cat_id in cleaned_upstreams:
|
|
55
|
+
cleaned_upstreams.remove(cat_id)
|
|
56
|
+
return list(cleaned_upstreams), 200
|
|
44
57
|
|
|
45
58
|
|
|
46
59
|
@main.route("/subset_check", methods=["POST"])
|
|
@@ -48,20 +61,28 @@ def subset_check():
|
|
|
48
61
|
cat_ids = list(json.loads(request.data.decode("utf-8")))
|
|
49
62
|
logger.info(cat_ids)
|
|
50
63
|
subset_name = cat_ids[0]
|
|
51
|
-
run_paths =
|
|
64
|
+
run_paths = FilePaths(subset_name)
|
|
52
65
|
if run_paths.geopackage_path.exists():
|
|
53
|
-
return
|
|
66
|
+
return str(run_paths.geopackage_path), 409
|
|
54
67
|
else:
|
|
55
|
-
return "
|
|
68
|
+
return "no conflict",200
|
|
56
69
|
|
|
57
70
|
|
|
58
71
|
@main.route("/subset", methods=["POST"])
|
|
59
72
|
def subset_selection():
|
|
60
|
-
|
|
73
|
+
#body: JSON.stringify({ 'cat_id': [cat_id], 'subset_type': subset_type})
|
|
74
|
+
data = json.loads(request.data.decode("utf-8"))
|
|
75
|
+
cat_ids = data.get("cat_id")
|
|
76
|
+
subset_type = data.get("subset_type")
|
|
61
77
|
logger.info(cat_ids)
|
|
78
|
+
logger.info(subset_type)
|
|
62
79
|
subset_name = cat_ids[0]
|
|
63
|
-
|
|
64
|
-
|
|
80
|
+
|
|
81
|
+
run_paths = FilePaths(subset_name)
|
|
82
|
+
if subset_type == "nexus":
|
|
83
|
+
subset(cat_ids, output_gpkg_path=run_paths.geopackage_path, override_gpkg=True)
|
|
84
|
+
else:
|
|
85
|
+
subset(cat_ids, output_gpkg_path=run_paths.geopackage_path, include_outlet=False, override_gpkg=True)
|
|
65
86
|
return str(run_paths.geopackage_path), 200
|
|
66
87
|
|
|
67
88
|
|
|
@@ -72,21 +93,57 @@ def subset_to_file():
|
|
|
72
93
|
logger.info(cat_ids)
|
|
73
94
|
subset_name = cat_ids[0]
|
|
74
95
|
total_subset = get_upstream_ids(cat_ids)
|
|
75
|
-
subset_paths =
|
|
96
|
+
subset_paths = FilePaths(subset_name)
|
|
76
97
|
output_file = subset_paths.subset_dir / "subset.txt"
|
|
77
98
|
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
78
99
|
with open(output_file, "w") as f:
|
|
79
100
|
f.write("\n".join(total_subset))
|
|
80
101
|
return str(subset_paths.subset_dir), 200
|
|
81
102
|
|
|
82
|
-
|
|
103
|
+
@main.route("/make_forcings_progress_file", methods=["POST"])
|
|
104
|
+
def make_forcings_progress_file():
|
|
105
|
+
data = json.loads(request.data.decode("utf-8"))
|
|
106
|
+
subset_gpkg = Path(data.split("subset to ")[-1])
|
|
107
|
+
paths = FilePaths(subset_gpkg.stem.split("_")[0])
|
|
108
|
+
paths.forcing_progress_file.parent.mkdir(parents=True, exist_ok=True)
|
|
109
|
+
with open(paths.forcing_progress_file, "w") as f:
|
|
110
|
+
json.dump({"total_steps": 0, "steps_completed": 0}, f)
|
|
111
|
+
return str(paths.forcing_progress_file), 200
|
|
112
|
+
|
|
113
|
+
@main.route("/forcings_progress", methods=["POST"])
|
|
114
|
+
def forcings_progress_endpoint():
|
|
115
|
+
progress_file = Path(json.loads(request.data.decode("utf-8")))
|
|
116
|
+
with open(progress_file, "r") as f:
|
|
117
|
+
forcings_progress = json.load(f)
|
|
118
|
+
forcings_progress_all = forcings_progress['total_steps']
|
|
119
|
+
forcings_progress_completed = forcings_progress['steps_completed']
|
|
120
|
+
try:
|
|
121
|
+
percent = int((forcings_progress_completed / forcings_progress_all) * 100)
|
|
122
|
+
except ZeroDivisionError:
|
|
123
|
+
percent = "NaN"
|
|
124
|
+
return str(percent), 200
|
|
125
|
+
|
|
126
|
+
def download_forcings(data_source, start_time, end_time, paths):
|
|
127
|
+
if data_source == "aorc":
|
|
128
|
+
raw_data = load_aorc_zarr(start_time.year, end_time.year)
|
|
129
|
+
elif data_source == "nwm":
|
|
130
|
+
raw_data = load_v3_retrospective_zarr()
|
|
131
|
+
else:
|
|
132
|
+
raise ValueError(f"Unknown data source: {data_source}")
|
|
133
|
+
gdf = gpd.read_file(paths.geopackage_path, layer="divides")
|
|
134
|
+
cached_data = save_and_clip_dataset(raw_data, gdf, start_time, end_time, paths.cached_nc_file)
|
|
135
|
+
return cached_data
|
|
136
|
+
|
|
137
|
+
def compute_forcings(cached_data, paths):
|
|
138
|
+
create_forcings(cached_data, paths.output_dir.stem) # type: ignore
|
|
139
|
+
|
|
83
140
|
@main.route("/forcings", methods=["POST"])
|
|
84
141
|
def get_forcings():
|
|
85
142
|
# body: JSON.stringify({'forcing_dir': forcing_dir, 'start_time': start_time, 'end_time': end_time}),
|
|
86
143
|
data = json.loads(request.data.decode("utf-8"))
|
|
87
144
|
subset_gpkg = Path(data.get("forcing_dir").split("subset to ")[-1])
|
|
88
145
|
output_folder = Path(subset_gpkg.parent.parent)
|
|
89
|
-
paths =
|
|
146
|
+
paths = FilePaths(output_dir=output_folder)
|
|
90
147
|
|
|
91
148
|
start_time = data.get("start_time")
|
|
92
149
|
end_time = data.get("end_time")
|
|
@@ -102,22 +159,14 @@ def get_forcings():
|
|
|
102
159
|
app.debug = False
|
|
103
160
|
logger.debug(f"get_forcings() disabled debug mode at {datetime.now()}")
|
|
104
161
|
logger.debug(f"forcing_dir: {output_folder}")
|
|
105
|
-
try:
|
|
106
|
-
if data_source == "aorc":
|
|
107
|
-
data = load_aorc_zarr(start_time.year, end_time.year)
|
|
108
|
-
elif data_source == "nwm":
|
|
109
|
-
data = load_v3_retrospective_zarr()
|
|
110
|
-
gdf = gpd.read_file(paths.geopackage_path, layer="divides")
|
|
111
|
-
cached_data = save_and_clip_dataset(data, gdf, start_time, end_time, paths.cached_nc_file)
|
|
112
|
-
|
|
113
|
-
create_forcings(cached_data, paths.output_dir.stem) # type: ignore
|
|
114
|
-
except Exception as e:
|
|
115
|
-
logger.info(f"get_forcings() failed with error: {str(e)}")
|
|
116
|
-
return jsonify({"error": str(e)}), 500
|
|
117
162
|
app.debug = debug_enabled
|
|
118
163
|
|
|
119
|
-
|
|
120
|
-
|
|
164
|
+
cached_data = download_forcings(data_source, start_time, end_time, paths)
|
|
165
|
+
# threading implemented so that main process can periodically poll progress file
|
|
166
|
+
thread = threading.Thread(target=compute_forcings,
|
|
167
|
+
args=(cached_data, paths))
|
|
168
|
+
thread.start()
|
|
169
|
+
return "started", 200
|
|
121
170
|
|
|
122
171
|
@main.route("/realization", methods=["POST"])
|
|
123
172
|
def get_realization():
|
ngiab_data_cli/__main__.py
CHANGED
|
@@ -15,7 +15,7 @@ with rich.status.Status("loading") as status:
|
|
|
15
15
|
from data_processing.dask_utils import shutdown_cluster
|
|
16
16
|
from data_processing.dataset_utils import save_and_clip_dataset
|
|
17
17
|
from data_processing.datasets import load_aorc_zarr, load_v3_retrospective_zarr
|
|
18
|
-
from data_processing.file_paths import
|
|
18
|
+
from data_processing.file_paths import FilePaths
|
|
19
19
|
from data_processing.forcings import create_forcings
|
|
20
20
|
from data_processing.gpkg_utils import get_cat_from_gage_id, get_catid_from_point
|
|
21
21
|
from data_processing.graph_utils import get_upstream_cats
|
|
@@ -23,57 +23,63 @@ with rich.status.Status("loading") as status:
|
|
|
23
23
|
from data_sources.source_validation import validate_hydrofabric, validate_output_dir
|
|
24
24
|
from ngiab_data_cli.arguments import parse_arguments
|
|
25
25
|
from ngiab_data_cli.custom_logging import set_logging_to_critical_only, setup_logging
|
|
26
|
-
|
|
27
26
|
|
|
28
27
|
|
|
29
28
|
def validate_input(args: argparse.Namespace) -> Tuple[str, str]:
|
|
30
29
|
"""Validate input arguments."""
|
|
31
30
|
|
|
31
|
+
feature_name = None
|
|
32
|
+
output_folder = None
|
|
33
|
+
|
|
32
34
|
if args.vpu:
|
|
33
35
|
if not args.output_name:
|
|
34
36
|
args.output_name = f"vpu-{args.vpu}"
|
|
35
37
|
validate_output_dir()
|
|
36
38
|
return args.vpu, args.output_name
|
|
37
39
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
40
|
+
if args.input_feature:
|
|
41
|
+
input_feature = args.input_feature.replace("_", "-")
|
|
42
|
+
|
|
43
|
+
if args.gage and not input_feature.startswith("gage-"):
|
|
44
|
+
input_feature = "gage-" + input_feature
|
|
45
|
+
|
|
46
|
+
# look at the prefix for autodetection, if -g or -l is used then there is no prefix
|
|
47
|
+
if len(input_feature.split("-")) > 1:
|
|
48
|
+
prefix = input_feature.split("-")[0]
|
|
49
|
+
if prefix.lower() == "gage":
|
|
50
|
+
args.gage = True
|
|
51
|
+
elif prefix.lower() == "wb":
|
|
52
|
+
logging.warning("Waterbody IDs are no longer supported!")
|
|
53
|
+
logging.warning(f"Automatically converting {input_feature} to catid")
|
|
54
|
+
time.sleep(2)
|
|
55
|
+
|
|
56
|
+
# always add or replace the prefix with cat if it is not a lat lon or gage
|
|
57
|
+
if not args.latlon and not args.gage:
|
|
58
|
+
input_feature = "cat-" + input_feature.split("-")[-1]
|
|
59
|
+
|
|
60
|
+
if args.latlon and args.gage:
|
|
61
|
+
raise ValueError("Cannot use both --latlon and --gage options at the same time.")
|
|
62
|
+
|
|
63
|
+
if args.latlon:
|
|
64
|
+
validate_hydrofabric()
|
|
65
|
+
feature_name = get_cat_id_from_lat_lon(input_feature)
|
|
66
|
+
logging.info(f"Found {feature_name} from {input_feature}")
|
|
67
|
+
elif args.gage:
|
|
68
|
+
validate_hydrofabric()
|
|
69
|
+
feature_name = get_cat_from_gage_id(input_feature)
|
|
70
|
+
logging.info(f"Found {feature_name} from {input_feature}")
|
|
71
|
+
else:
|
|
72
|
+
feature_name = input_feature
|
|
73
|
+
|
|
74
|
+
if args.output_name:
|
|
75
|
+
output_folder = args.output_name
|
|
76
|
+
validate_output_dir()
|
|
77
|
+
elif args.gage:
|
|
78
|
+
output_folder = input_feature
|
|
79
|
+
validate_output_dir()
|
|
80
|
+
else:
|
|
81
|
+
output_folder = feature_name
|
|
82
|
+
validate_output_dir()
|
|
77
83
|
|
|
78
84
|
return feature_name, output_folder
|
|
79
85
|
|
|
@@ -87,7 +93,7 @@ def get_cat_id_from_lat_lon(input_feature: str) -> str:
|
|
|
87
93
|
raise ValueError("Lat Lon input must be comma separated e.g. -l 54.33,-69.4")
|
|
88
94
|
|
|
89
95
|
|
|
90
|
-
def set_dependent_flags(args, paths:
|
|
96
|
+
def set_dependent_flags(args, paths: FilePaths):
|
|
91
97
|
# if validate is set, run everything that is missing
|
|
92
98
|
if args.validate:
|
|
93
99
|
logging.info("Running all missing steps required to run ngiab.")
|
|
@@ -109,7 +115,7 @@ def set_dependent_flags(args, paths: file_paths):
|
|
|
109
115
|
return args
|
|
110
116
|
|
|
111
117
|
|
|
112
|
-
def validate_run_directory(args, paths:
|
|
118
|
+
def validate_run_directory(args, paths: FilePaths):
|
|
113
119
|
# checks the folder that is going to be run, enables steps that are needed to populate the folder
|
|
114
120
|
if not paths.subset_dir.exists():
|
|
115
121
|
logging.info("Subset folder does not exist, enabling subset, forcings, and realization.")
|
|
@@ -134,8 +140,23 @@ def main() -> None:
|
|
|
134
140
|
args = parse_arguments()
|
|
135
141
|
if args.debug:
|
|
136
142
|
logging.getLogger("data_processing").setLevel(logging.DEBUG)
|
|
143
|
+
|
|
144
|
+
if args.output_root:
|
|
145
|
+
with open(FilePaths.config_file, "w") as config_file:
|
|
146
|
+
config_file.write(args.output_root)
|
|
147
|
+
logging.info(
|
|
148
|
+
f"Changed default directory where outputs are stored to {args.output_root}"
|
|
149
|
+
)
|
|
150
|
+
|
|
137
151
|
feature_to_subset, output_folder = validate_input(args)
|
|
138
|
-
|
|
152
|
+
|
|
153
|
+
if (feature_to_subset, output_folder) == (
|
|
154
|
+
None,
|
|
155
|
+
None,
|
|
156
|
+
): # in case someone just passes an argument to change default output dir
|
|
157
|
+
return
|
|
158
|
+
|
|
159
|
+
paths = FilePaths(output_folder)
|
|
139
160
|
args = set_dependent_flags(args, paths) # --validate
|
|
140
161
|
if feature_to_subset:
|
|
141
162
|
logging.info(f"Processing {feature_to_subset} in {paths.output_dir}")
|
|
@@ -186,9 +207,14 @@ def main() -> None:
|
|
|
186
207
|
gage_id = None
|
|
187
208
|
if args.gage:
|
|
188
209
|
gage_id = args.input_feature
|
|
189
|
-
|
|
210
|
+
if not gage_id.startswith("gage-"):
|
|
211
|
+
gage_id = "gage-" + gage_id
|
|
212
|
+
if args.lstm or args.lstm_rust:
|
|
190
213
|
create_lstm_realization(
|
|
191
|
-
output_folder,
|
|
214
|
+
output_folder,
|
|
215
|
+
start_time=args.start_date,
|
|
216
|
+
end_time=args.end_date,
|
|
217
|
+
use_rust=args.lstm_rust,
|
|
192
218
|
)
|
|
193
219
|
else:
|
|
194
220
|
create_realization(
|
|
@@ -202,7 +228,7 @@ def main() -> None:
|
|
|
202
228
|
|
|
203
229
|
if args.run:
|
|
204
230
|
logging.info("Running Next Gen using NGIAB...")
|
|
205
|
-
|
|
231
|
+
|
|
206
232
|
try:
|
|
207
233
|
subprocess.run("docker pull awiciroh/ciroh-ngen-image:latest", shell=True)
|
|
208
234
|
except:
|
ngiab_data_cli/arguments.py
CHANGED
|
@@ -11,7 +11,14 @@ def parse_arguments() -> argparse.Namespace:
|
|
|
11
11
|
parser = argparse.ArgumentParser(
|
|
12
12
|
description="Subsetting hydrofabrics, forcing generation, and realization creation"
|
|
13
13
|
)
|
|
14
|
-
group = parser.add_mutually_exclusive_group(required=
|
|
14
|
+
group = parser.add_mutually_exclusive_group(required=False)
|
|
15
|
+
|
|
16
|
+
parser.add_argument(
|
|
17
|
+
"--output_root",
|
|
18
|
+
type=str,
|
|
19
|
+
help="Path to new default directory where outputs in the future will be stored",
|
|
20
|
+
)
|
|
21
|
+
|
|
15
22
|
group.add_argument(
|
|
16
23
|
"-i",
|
|
17
24
|
"--input_feature",
|
|
@@ -97,7 +104,7 @@ def parse_arguments() -> argparse.Namespace:
|
|
|
97
104
|
"-o",
|
|
98
105
|
"--output_name",
|
|
99
106
|
type=str,
|
|
100
|
-
help="
|
|
107
|
+
help="Custom data output folder name in lieu of the default, which is the ID of the input feature",
|
|
101
108
|
)
|
|
102
109
|
parser.add_argument(
|
|
103
110
|
"-D",
|
|
@@ -105,11 +112,17 @@ def parse_arguments() -> argparse.Namespace:
|
|
|
105
112
|
action="store_true",
|
|
106
113
|
help="enable debug logging",
|
|
107
114
|
)
|
|
108
|
-
parser.
|
|
115
|
+
lstm_group = parser.add_mutually_exclusive_group(required=False)
|
|
116
|
+
lstm_group.add_argument(
|
|
109
117
|
"--lstm",
|
|
110
118
|
action="store_true",
|
|
111
119
|
help="enable LSTM model realization and forcings",
|
|
112
120
|
)
|
|
121
|
+
lstm_group.add_argument(
|
|
122
|
+
"--lstm_rust",
|
|
123
|
+
action="store_true",
|
|
124
|
+
help="enable experimental high speed Rust bindings of LSTM model realization and forcings",
|
|
125
|
+
)
|
|
113
126
|
parser.add_argument(
|
|
114
127
|
"--nwm_gw",
|
|
115
128
|
action="store_true",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ngiab_data_preprocess
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.6.0
|
|
4
4
|
Summary: Graphical Tools for creating Next Gen Water model input data.
|
|
5
5
|
Author-email: Josh Cunningham <jcunningham8@ua.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/CIROH-UA/NGIAB_data_preprocess
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
data_processing/create_realization.py,sha256=b1a9Wuld9saJ-zzVDPY_kba3-ZOVbRuobyR-QiHLXyY,11550
|
|
2
|
+
data_processing/dask_utils.py,sha256=A2IP94WAz8W9nek3etXKEKTOxGPf0NWSFLh8cZ5S-xU,2454
|
|
3
|
+
data_processing/dataset_utils.py,sha256=0FhspUlGUs0iJLwYY0dl_1fjt0_zwhe9Hl-jikIgrpk,8312
|
|
4
|
+
data_processing/datasets.py,sha256=_EJ1uZSWTU1HWpvF7TQSikneJqWZFikTrdo9usCV8A0,4665
|
|
5
|
+
data_processing/file_paths.py,sha256=IcDcQ8tdpvhUfB2YSor85dFqGCQWWyd5nY0Gj6OQ6-c,4967
|
|
6
|
+
data_processing/forcings.py,sha256=Vy9P6ABBddMbUyCP4Q0aLrt446todgc2f7U6m35trqQ,21532
|
|
7
|
+
data_processing/gpkg_utils.py,sha256=VoQG53zaGcdmkU9M_7kRrk_vygRGJV9E-wX-T48Rzj8,20576
|
|
8
|
+
data_processing/graph_utils.py,sha256=4D72wMSiCRKCPC7JUz7XCoaISRGLuqDx6wpeO_VP8fk,8301
|
|
9
|
+
data_processing/s3fs_utils.py,sha256=ki1EmA0ezV0r26re6dRWIGzL5FudGdwF9Qw1eVLR0Bc,2747
|
|
10
|
+
data_processing/subset.py,sha256=EwDii-EsOiJBaDMjvYWfz7V9wQSXb887w_fk28u8dlw,3789
|
|
11
|
+
data_sources/cfe-nowpm-realization-template.json,sha256=8an6q1drWD8wU1ocvdPab-GvZDvlQ-0di_-NommH3QI,3528
|
|
12
|
+
data_sources/cfe-template.ini,sha256=6e5-usqjWtm3MWVvtm8CTeZTJJMxO1ZswkOXq0L9mnc,2033
|
|
13
|
+
data_sources/forcing_template.nc,sha256=uRuVAqX3ngdlougZINavtwl_wC2VLD8fHqG7_CLim1s,85284
|
|
14
|
+
data_sources/lstm-catchment-template.yml,sha256=LtknqvxbWrtLLZIXxFgTfbQmM4x8XnHBDFvRIh2EIFI,965
|
|
15
|
+
data_sources/lstm-realization-template.json,sha256=ndz3h5NGhtUSnsZwscgNuXYBG9mlAuz7Lxx7iCw22UY,1270
|
|
16
|
+
data_sources/lstm-rust-realization-template.json,sha256=l6JsJc8bp8MTGV-ED4oI4pvZTa4uReqDRDBHu3Uj1b8,1338
|
|
17
|
+
data_sources/ngen-routing-template.yaml,sha256=wM5v6jj0kwcJBVatLFuy2big6g8nlSXxzc8a23nwI5s,4655
|
|
18
|
+
data_sources/noah-owp-modular-init.namelist.input,sha256=Vb7mp40hFpJogruOrXrDHwVW1bKi9h1ciDNyDvTzn20,3045
|
|
19
|
+
data_sources/source_validation.py,sha256=DMCTo-Tad9QSXFqSDB5guI8hMzY_kQE6aW78-5nqwlU,9457
|
|
20
|
+
data_sources/template.sql,sha256=ZnFqAqleEq9wgmAhNO90Wue_L9k0JAn8KF99DYtcxgs,10457
|
|
21
|
+
data_sources/triggers.sql,sha256=G0d_175eNsamKAFhsbphPATvzMPuPL_iCleIhlToduQ,14906
|
|
22
|
+
map_app/__init__.py,sha256=OarJao9X98kcbLyiwewN4ObWNAYkKDichcxbuWywTsA,818
|
|
23
|
+
map_app/__main__.py,sha256=5qJGypfesfdWDWRsbTQiMs3uL5zPZKRZQq31l7qXqCc,1649
|
|
24
|
+
map_app/views.py,sha256=AxLsXL8ZSnRJzg5zVYDGfVFA_6amgae41AO_E1G7W58,7923
|
|
25
|
+
map_app/static/css/console.css,sha256=xN6G2MMFyKc9YW9HEVpUUTUjx2o2nokBR4nCX5c18UM,803
|
|
26
|
+
map_app/static/css/main.css,sha256=gntWJL4x7Lm3lRDnv0AEmZ18u92C69rSj1HbF_9__f8,7107
|
|
27
|
+
map_app/static/css/toggle.css,sha256=sYxDi381TWNFwDD8wXlJ5n8yAlF9aIR3gkI0QwC8oKM,2555
|
|
28
|
+
map_app/static/js/console.js,sha256=BnG0pED5B9d563sLWshDNt_q-SyoTY48sETvVoOVJkU,1377
|
|
29
|
+
map_app/static/js/data_processing.js,sha256=sfeXvfPtiOvEN1kDFFtibLDOjwyudOpeOjmfXuTcR7Y,8583
|
|
30
|
+
map_app/static/js/main.js,sha256=8vWWO56vc8LEhyp3AeRiJw1tsqebZtkG9IOS65b154w,11063
|
|
31
|
+
map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
|
|
32
|
+
map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
|
|
33
|
+
map_app/templates/index.html,sha256=vgcefTU3GmRDNL6P5Xi284nO2vdLxxIW8c5xp3SHsTg,8664
|
|
34
|
+
ngiab_data_cli/__main__.py,sha256=SRNt7DTJuPW3nMDk30tmde5rc0kSIyLQ0fMwFeHT9tg,11254
|
|
35
|
+
ngiab_data_cli/arguments.py,sha256=97BFzlj68tt9tRCR3rTPnK6o7zd7dUCdPg8Cypa2aKA,4782
|
|
36
|
+
ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
|
|
37
|
+
ngiab_data_cli/forcing_cli.py,sha256=eIWRxRWUwPqR16fihFDEIV4VzGlNuvcD6lJW5VYjkPU,3635
|
|
38
|
+
ngiab_data_preprocess-4.6.0.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
|
|
39
|
+
ngiab_data_preprocess-4.6.0.dist-info/METADATA,sha256=CHJUKYW6WdXYNGxBLBG5vCNQsDGrPrO3BIXf237EHGo,13344
|
|
40
|
+
ngiab_data_preprocess-4.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
41
|
+
ngiab_data_preprocess-4.6.0.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
|
|
42
|
+
ngiab_data_preprocess-4.6.0.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
|
|
43
|
+
ngiab_data_preprocess-4.6.0.dist-info/RECORD,,
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
data_processing/create_realization.py,sha256=deRP3p4zfV9PF_vUikYKRiKA994bgj2GtPGCjQDSsTE,10795
|
|
2
|
-
data_processing/dask_utils.py,sha256=A2IP94WAz8W9nek3etXKEKTOxGPf0NWSFLh8cZ5S-xU,2454
|
|
3
|
-
data_processing/dataset_utils.py,sha256=AJOxE2nRfZnWYon_qqGcfkpRZuRW8Yy8YI86SxVDU3M,11168
|
|
4
|
-
data_processing/datasets.py,sha256=_EJ1uZSWTU1HWpvF7TQSikneJqWZFikTrdo9usCV8A0,4665
|
|
5
|
-
data_processing/file_paths.py,sha256=MFUShBB1g9IGi9MaJwrl6fKIcsrhbmcYEdTHtmnphZo,4667
|
|
6
|
-
data_processing/forcings.py,sha256=k-JhBncTnXcdjSieam1Q2cDx5Xt9hH5Aywv0gDY4O2U,19010
|
|
7
|
-
data_processing/gpkg_utils.py,sha256=tSSIMlHeqqgxTJQyF3X9tPmunQTJYx0xrCNHqUBQxkg,20590
|
|
8
|
-
data_processing/graph_utils.py,sha256=qvHw6JlzQxLi--eMsGgC_rUBP4nDatl6X9mSa03Xxyo,8306
|
|
9
|
-
data_processing/s3fs_utils.py,sha256=ki1EmA0ezV0r26re6dRWIGzL5FudGdwF9Qw1eVLR0Bc,2747
|
|
10
|
-
data_processing/subset.py,sha256=XoojOgWCwxOi5Q4KXHXARNQeoZlobJp-mqhIIvTRtTw,3793
|
|
11
|
-
data_sources/cfe-nowpm-realization-template.json,sha256=8an6q1drWD8wU1ocvdPab-GvZDvlQ-0di_-NommH3QI,3528
|
|
12
|
-
data_sources/cfe-template.ini,sha256=6e5-usqjWtm3MWVvtm8CTeZTJJMxO1ZswkOXq0L9mnc,2033
|
|
13
|
-
data_sources/forcing_template.nc,sha256=uRuVAqX3ngdlougZINavtwl_wC2VLD8fHqG7_CLim1s,85284
|
|
14
|
-
data_sources/lstm-catchment-template.yml,sha256=LtknqvxbWrtLLZIXxFgTfbQmM4x8XnHBDFvRIh2EIFI,965
|
|
15
|
-
data_sources/lstm-realization-template.json,sha256=ndz3h5NGhtUSnsZwscgNuXYBG9mlAuz7Lxx7iCw22UY,1270
|
|
16
|
-
data_sources/ngen-routing-template.yaml,sha256=wM5v6jj0kwcJBVatLFuy2big6g8nlSXxzc8a23nwI5s,4655
|
|
17
|
-
data_sources/noah-owp-modular-init.namelist.input,sha256=Vb7mp40hFpJogruOrXrDHwVW1bKi9h1ciDNyDvTzn20,3045
|
|
18
|
-
data_sources/source_validation.py,sha256=RmvyPLjuDetpuNOUqCclgDfe8zd_Ojr7pfbUoUya2pQ,9498
|
|
19
|
-
data_sources/template.sql,sha256=ZnFqAqleEq9wgmAhNO90Wue_L9k0JAn8KF99DYtcxgs,10457
|
|
20
|
-
data_sources/triggers.sql,sha256=G0d_175eNsamKAFhsbphPATvzMPuPL_iCleIhlToduQ,14906
|
|
21
|
-
map_app/__init__.py,sha256=OarJao9X98kcbLyiwewN4ObWNAYkKDichcxbuWywTsA,818
|
|
22
|
-
map_app/__main__.py,sha256=Uj7__cJUyPQkZo2tNQ2x2g6rwizsyg1DcNtJkQniHzY,1650
|
|
23
|
-
map_app/views.py,sha256=ajU_QSd-Oa7UrRQEZPX4rmOlaKwo76Q8UPQNXtt-e2k,5622
|
|
24
|
-
map_app/static/css/console.css,sha256=xN6G2MMFyKc9YW9HEVpUUTUjx2o2nokBR4nCX5c18UM,803
|
|
25
|
-
map_app/static/css/main.css,sha256=HmRIfhWeHTrNLOCHGpaKuzwGj05LkkUiQy538D-ZRLY,6464
|
|
26
|
-
map_app/static/css/toggle.css,sha256=Ep6tXT7gCrPRRITuEMpXyisuiTQgiLIEKFFTWRmC82o,1913
|
|
27
|
-
map_app/static/js/console.js,sha256=BnG0pED5B9d563sLWshDNt_q-SyoTY48sETvVoOVJkU,1377
|
|
28
|
-
map_app/static/js/data_processing.js,sha256=wXv0p_bPmNOrSpU_p6Yqtfd17vqOFRJFAmLdUUWLF7s,5486
|
|
29
|
-
map_app/static/js/main.js,sha256=_Yq1tuzyREqWU24rFQJSh5zIaXtAXEGlfZPo36QLHvI,9690
|
|
30
|
-
map_app/static/resources/loading.gif,sha256=ggdkZf1AD7rSwIpSJwfiIqANgmVV1WHlxGuKxQKv7uY,72191
|
|
31
|
-
map_app/static/resources/screenshot.jpg,sha256=Ia358aX-OHM9BP4B8lX05cLnguF2fHUIimno9bnFLYw,253730
|
|
32
|
-
map_app/templates/index.html,sha256=Jy2k1Ob2_et--BPpfmTYO22Yin3vrG6IOeNlwzUoEqY,7878
|
|
33
|
-
ngiab_data_cli/__main__.py,sha256=io9YbZY65tQC66gpcP02ECRnGpM-fnjLxQHa1EKDKzc,10269
|
|
34
|
-
ngiab_data_cli/arguments.py,sha256=qS8RupcT3Ax7ZRT0uKKzFdUvkDdVugBlYyuzljY_bxo,4290
|
|
35
|
-
ngiab_data_cli/custom_logging.py,sha256=iS2XozaxudcxQj17qAsrCgbVK9LJAYAPmarJuVWJo1k,1280
|
|
36
|
-
ngiab_data_cli/forcing_cli.py,sha256=eIWRxRWUwPqR16fihFDEIV4VzGlNuvcD6lJW5VYjkPU,3635
|
|
37
|
-
ngiab_data_preprocess-4.5.1.dist-info/licenses/LICENSE,sha256=6dMSprwwnsRzEm02mEDbKHD9dUbL8bPIt9Vhrhb0Ulk,1081
|
|
38
|
-
ngiab_data_preprocess-4.5.1.dist-info/METADATA,sha256=Sa-C8mvOi-1Qn5u27X2Dz5wYeMPLnjle-nuC_VJ2kZo,13344
|
|
39
|
-
ngiab_data_preprocess-4.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
40
|
-
ngiab_data_preprocess-4.5.1.dist-info/entry_points.txt,sha256=spwlhKEJ3ZnNETQsJGeTjD7Vwy8O_zGHb9GdX8ACCtw,128
|
|
41
|
-
ngiab_data_preprocess-4.5.1.dist-info/top_level.txt,sha256=CjhYAUZrdveR2fOK6rxffU09VIN2IuPD7hk4V3l3pV0,52
|
|
42
|
-
ngiab_data_preprocess-4.5.1.dist-info/RECORD,,
|
|
File without changes
|
{ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{ngiab_data_preprocess-4.5.1.dist-info → ngiab_data_preprocess-4.6.0.dist-info}/top_level.txt
RENAMED
|
File without changes
|