ffmtools 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ffmclr/__init__.py +25 -0
- ffmclr/__main__.py +5 -0
- ffmclr/cli.py +37 -0
- ffmclr/core.py +191 -0
- ffmgen/__init__.py +32 -0
- ffmgen/__main__.py +5 -0
- ffmgen/cli.py +99 -0
- ffmgen/core.py +203 -0
- ffmgen/gui.py +315 -0
- ffmgen/templates/STAMFORD_basis.mdu +138 -0
- ffmgen/timecalc.py +107 -0
- ffmtools-0.1.0.dist-info/METADATA +106 -0
- ffmtools-0.1.0.dist-info/RECORD +17 -0
- ffmtools-0.1.0.dist-info/WHEEL +5 -0
- ffmtools-0.1.0.dist-info/entry_points.txt +3 -0
- ffmtools-0.1.0.dist-info/licenses/LICENSE +21 -0
- ffmtools-0.1.0.dist-info/top_level.txt +2 -0
ffmclr/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""ffmclr - clean files generated by a 3Di (subgrid) model run.
|
|
2
|
+
|
|
3
|
+
Programmatic use::
|
|
4
|
+
|
|
5
|
+
import ffmclr
|
|
6
|
+
ffmclr.clean('path/to/model') # dry run: list only
|
|
7
|
+
ffmclr.clean('path/to/model', dry_run=False) # delete
|
|
8
|
+
ffmclr.clean('path/to/model', dry_run=False, move_to='_old_run')
|
|
9
|
+
|
|
10
|
+
Command line: ``ffmclr [MODEL_DIR] [--yes] [--keep-table] [-r] [--move-to DIR]``
|
|
11
|
+
(or ``python -m ffmclr``).
|
|
12
|
+
"""
|
|
13
|
+
from .core import (OUTPUT_PATTERNS, PROTECTED_EXT, CleanResult, is_model_dir,
|
|
14
|
+
read_mdu_references, find_generated, clean, human)
|
|
15
|
+
|
|
16
|
+
__version__ = '0.1.0'
|
|
17
|
+
|
|
18
|
+
__all__ = ['OUTPUT_PATTERNS', 'PROTECTED_EXT', 'CleanResult', 'is_model_dir',
|
|
19
|
+
'read_mdu_references', 'find_generated', 'clean', 'human', 'main', '__version__']
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main(argv=None):
|
|
23
|
+
"""Console entry point (see ffmclr.cli)."""
|
|
24
|
+
from .cli import main as _main
|
|
25
|
+
return _main(argv)
|
ffmclr/__main__.py
ADDED
ffmclr/cli.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Command line interface.
|
|
2
|
+
|
|
3
|
+
ffmclr dry run in the current folder: list what would be removed
|
|
4
|
+
ffmclr --yes actually delete
|
|
5
|
+
ffmclr --yes --keep-table delete but keep the table file (faster restart)
|
|
6
|
+
ffmclr PATH\\TO\\MODEL --yes clean another model folder
|
|
7
|
+
ffmclr --move-to _old_run --yes move instead of delete
|
|
8
|
+
"""
|
|
9
|
+
import argparse
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .core import clean
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main(argv=None):
|
|
17
|
+
ap = argparse.ArgumentParser(prog='ffmclr', description="Clean files generated by a 3Di model run.")
|
|
18
|
+
ap.add_argument('--version', action='version', version='%(prog)s ' + __version__)
|
|
19
|
+
ap.add_argument("model_dir", nargs="?", default='.',
|
|
20
|
+
help="model folder (default: current folder)")
|
|
21
|
+
ap.add_argument("--yes", "-y", action="store_true", help="actually delete (default is dry run)")
|
|
22
|
+
ap.add_argument("--keep-table", action="store_true", help="keep the TableDataFile")
|
|
23
|
+
ap.add_argument("--recursive", "-r", action="store_true", help="also scan sub-folders")
|
|
24
|
+
ap.add_argument("--move-to", metavar="DIR",
|
|
25
|
+
help="move files into DIR (relative to the model folder) instead of deleting")
|
|
26
|
+
a = ap.parse_args(argv)
|
|
27
|
+
try:
|
|
28
|
+
res = clean(a.model_dir, dry_run=not a.yes, keep_table=a.keep_table,
|
|
29
|
+
recursive=a.recursive, move_to=a.move_to)
|
|
30
|
+
except ValueError as e:
|
|
31
|
+
print(e, file=sys.stderr)
|
|
32
|
+
return 2
|
|
33
|
+
return 1 if res.failed else 0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if __name__ == '__main__':
|
|
37
|
+
sys.exit(main())
|
ffmclr/core.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Find and remove files generated by a 3Di (subgrid) model run.
|
|
2
|
+
|
|
3
|
+
What counts as "generated":
|
|
4
|
+
* <name>_his.nc, <name>_map.nc history / map output
|
|
5
|
+
* *_rst.nc, *.rst restart files
|
|
6
|
+
* *.dia diagnostic log (e.g. subgridf90.dia)
|
|
7
|
+
* *_numlimdt.xyz, *_timings.txt solver diagnostics
|
|
8
|
+
* .test empty marker file written at start-up
|
|
9
|
+
* the TableDataFile named in the .mdu files (e.g. "l") - the pre-computed
|
|
10
|
+
subgrid table; 3Di rebuilds it on the next run (keep_table=True keeps it)
|
|
11
|
+
|
|
12
|
+
Safety:
|
|
13
|
+
* Any file referenced as an input in an .mdu or .ext file is never removed
|
|
14
|
+
(except the TableDataFile, see above).
|
|
15
|
+
* Input extensions (.mdu .ini .ext .pli .pol .xyn .tim .tif .asc .rgb ...)
|
|
16
|
+
are never removed, whatever their name.
|
|
17
|
+
* Only the model root folder is scanned unless recursive=True.
|
|
18
|
+
* The move_to folder (if inside the model folder) is never scanned, so a
|
|
19
|
+
second run does not pick up files moved there earlier.
|
|
20
|
+
"""
|
|
21
|
+
import fnmatch
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
__all__ = ['OUTPUT_PATTERNS', 'PROTECTED_EXT', 'CleanResult',
|
|
28
|
+
'is_model_dir', 'read_mdu_references', 'find_generated', 'clean', 'human']
|
|
29
|
+
|
|
30
|
+
OUTPUT_PATTERNS = [
|
|
31
|
+
"*_his.nc", "*_map.nc", "*_rst.nc", "*.rst",
|
|
32
|
+
"*.dia", "*_numlimdt.xyz", "*_timings.txt", ".test",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
PROTECTED_EXT = {
|
|
36
|
+
".mdu", ".ini", ".ext", ".pli", ".pol", ".xyn", ".tim", ".tif", ".tiff",
|
|
37
|
+
".asc", ".rgb", ".zrgb", ".ldb", ".xyz_input", ".dat", ".cfg", ".py",
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
KEY_VALUE = re.compile(r"^\s*([A-Za-z0-9_]+)\s*=\s*([^#]*)")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def is_model_dir(model_dir):
|
|
44
|
+
"""True if the folder contains at least one .mdu file."""
|
|
45
|
+
p = Path(model_dir)
|
|
46
|
+
return p.is_dir() and any(p.glob("*.mdu"))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def read_mdu_references(model_dir):
|
|
50
|
+
"""Return (set of input paths referenced, set of table-data paths).
|
|
51
|
+
|
|
52
|
+
Scans the .mdu files in ``model_dir`` and every .ext file below it.
|
|
53
|
+
"""
|
|
54
|
+
model_dir = Path(model_dir)
|
|
55
|
+
inputs, tables = set(), set()
|
|
56
|
+
for cfg in list(model_dir.glob("*.mdu")) + list(model_dir.rglob("*.ext")):
|
|
57
|
+
try:
|
|
58
|
+
text = cfg.read_text(errors="ignore")
|
|
59
|
+
except OSError:
|
|
60
|
+
continue
|
|
61
|
+
for line in text.splitlines():
|
|
62
|
+
m = KEY_VALUE.match(line)
|
|
63
|
+
if not m:
|
|
64
|
+
continue
|
|
65
|
+
key, value = m.group(1).lower(), m.group(2).strip()
|
|
66
|
+
if not value:
|
|
67
|
+
continue
|
|
68
|
+
# value may be "a b" for some keys; check every token as a file name
|
|
69
|
+
for token in value.split():
|
|
70
|
+
p = (cfg.parent / token).resolve() if cfg.suffix.lower() == ".ext" \
|
|
71
|
+
else (model_dir / token).resolve()
|
|
72
|
+
if key == "tabledatafile":
|
|
73
|
+
tables.add(p)
|
|
74
|
+
elif p.exists() and p.is_file():
|
|
75
|
+
inputs.add(p)
|
|
76
|
+
return inputs, tables
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _is_within(path, folder):
|
|
80
|
+
try:
|
|
81
|
+
path.relative_to(folder)
|
|
82
|
+
return True
|
|
83
|
+
except ValueError:
|
|
84
|
+
return False
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def find_generated(model_dir, recursive=False, keep_table=False, exclude=()):
|
|
88
|
+
"""List generated files as sorted ``[(Path, reason), ...]``.
|
|
89
|
+
|
|
90
|
+
``exclude``: folders whose contents are skipped (e.g. the move_to folder).
|
|
91
|
+
"""
|
|
92
|
+
model_dir = Path(model_dir).resolve()
|
|
93
|
+
excl = [Path(e).resolve() for e in exclude]
|
|
94
|
+
inputs, tables = read_mdu_references(model_dir)
|
|
95
|
+
files = model_dir.rglob("*") if recursive else model_dir.iterdir()
|
|
96
|
+
found = []
|
|
97
|
+
for f in files:
|
|
98
|
+
if not f.is_file():
|
|
99
|
+
continue
|
|
100
|
+
rp = f.resolve()
|
|
101
|
+
if any(_is_within(rp, e) for e in excl):
|
|
102
|
+
continue
|
|
103
|
+
if rp in tables:
|
|
104
|
+
if not keep_table:
|
|
105
|
+
found.append((f, "table data (rebuilt next run)"))
|
|
106
|
+
continue
|
|
107
|
+
if rp in inputs or f.suffix.lower() in PROTECTED_EXT:
|
|
108
|
+
continue
|
|
109
|
+
for pat in OUTPUT_PATTERNS:
|
|
110
|
+
if fnmatch.fnmatch(f.name.lower(), pat.lower()):
|
|
111
|
+
found.append((f, f"matches {pat}"))
|
|
112
|
+
break
|
|
113
|
+
return sorted(found)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def human(n):
|
|
117
|
+
"""Byte count -> '1.5 MB'."""
|
|
118
|
+
for unit in ("B", "KB", "MB", "GB"):
|
|
119
|
+
if n < 1024:
|
|
120
|
+
return f"{n:.1f} {unit}"
|
|
121
|
+
n /= 1024
|
|
122
|
+
return f"{n:.1f} TB"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass
|
|
126
|
+
class CleanResult:
|
|
127
|
+
model_dir: Path
|
|
128
|
+
targets: list # [(Path, reason)] found
|
|
129
|
+
removed: list = field(default_factory=list) # Paths deleted / moved
|
|
130
|
+
failed: list = field(default_factory=list) # [(Path, error message)]
|
|
131
|
+
dest: Path = None # move_to folder, None = deleted
|
|
132
|
+
dry_run: bool = True
|
|
133
|
+
total_bytes: int = 0 # size of all targets
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _resolve_dest(model_dir, move_to):
|
|
137
|
+
if not move_to:
|
|
138
|
+
return None
|
|
139
|
+
dest = Path(move_to)
|
|
140
|
+
return (dest if dest.is_absolute() else model_dir / dest).resolve()
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def clean(model_dir='.', dry_run=True, keep_table=False, recursive=False,
|
|
144
|
+
move_to=None, verbose=True):
|
|
145
|
+
"""Find generated files and (unless ``dry_run``) delete them or move them to ``move_to``.
|
|
146
|
+
|
|
147
|
+
``move_to`` may be relative to ``model_dir``. Raises ValueError if
|
|
148
|
+
``model_dir`` is not a folder containing an .mdu file. Returns a CleanResult.
|
|
149
|
+
"""
|
|
150
|
+
log = print if verbose else (lambda *a, **k: None)
|
|
151
|
+
model_dir = Path(model_dir).resolve()
|
|
152
|
+
if not model_dir.is_dir():
|
|
153
|
+
raise ValueError(f"Not a folder: {model_dir}")
|
|
154
|
+
if not is_model_dir(model_dir):
|
|
155
|
+
raise ValueError(f"No .mdu file in {model_dir} - is this a 3Di model folder?")
|
|
156
|
+
|
|
157
|
+
dest = _resolve_dest(model_dir, move_to)
|
|
158
|
+
targets = find_generated(model_dir, recursive, keep_table, exclude=[dest] if dest else ())
|
|
159
|
+
res = CleanResult(model_dir=model_dir, targets=targets, dest=dest, dry_run=dry_run)
|
|
160
|
+
if not targets:
|
|
161
|
+
log("Nothing to clean.")
|
|
162
|
+
return res
|
|
163
|
+
|
|
164
|
+
sizes = {f: f.stat().st_size for f, _ in targets}
|
|
165
|
+
res.total_bytes = sum(sizes.values())
|
|
166
|
+
log(f"Model folder: {model_dir}")
|
|
167
|
+
for f, why in targets:
|
|
168
|
+
log(f" {f.relative_to(model_dir)!s:<30} {human(sizes[f]):>10} ({why})")
|
|
169
|
+
log(f"{len(targets)} file(s), {human(res.total_bytes)}")
|
|
170
|
+
|
|
171
|
+
if dry_run:
|
|
172
|
+
log("\nDry run - nothing removed. Re-run with --yes to clean.")
|
|
173
|
+
return res
|
|
174
|
+
|
|
175
|
+
if dest:
|
|
176
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
177
|
+
for f, _ in targets:
|
|
178
|
+
try:
|
|
179
|
+
if dest:
|
|
180
|
+
target = dest / f.relative_to(model_dir)
|
|
181
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
182
|
+
shutil.move(str(f), str(target))
|
|
183
|
+
else:
|
|
184
|
+
f.unlink()
|
|
185
|
+
res.removed.append(f)
|
|
186
|
+
except OSError as e:
|
|
187
|
+
res.failed.append((f, str(e)))
|
|
188
|
+
log(f" ! could not remove {f.name}: {e}")
|
|
189
|
+
action = f"moved to {dest}" if dest else "deleted"
|
|
190
|
+
log(f"{len(res.removed)} file(s) {action}." + (f" {len(res.failed)} failed." if res.failed else ""))
|
|
191
|
+
return res
|
ffmgen/__init__.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""ffmgen - build 3Di subgrid .mdu files from xlsx/mdu parameter sets.
|
|
2
|
+
|
|
3
|
+
Programmatic use::
|
|
4
|
+
|
|
5
|
+
import ffmgen
|
|
6
|
+
data = ffmgen.ensure_end_rows(ffmgen.read_any('mdu_prep.xlsx'))
|
|
7
|
+
ffmgen.update_ntimesteps(data)
|
|
8
|
+
ffmgen.write_mdu(data, out_dir='.', name='STAMFORD_basis')
|
|
9
|
+
|
|
10
|
+
Desktop GUI: ``ffmgen`` (or ``ffmgen gui``, ``python -m ffmgen``).
|
|
11
|
+
"""
|
|
12
|
+
from .core import (PAR_LIST, DEFAULT_NAME, EXE_DEFAULT, OMP_DEFAULT,
|
|
13
|
+
default_mdu_text, default_data,
|
|
14
|
+
parse_mdu, read_mdu, read_xlsx, write_xlsx, read_any,
|
|
15
|
+
check_groups, to_mdu_text, bat_text, name_from_path, write_mdu)
|
|
16
|
+
from .timecalc import ensure_end_rows, update_ntimesteps
|
|
17
|
+
|
|
18
|
+
__version__ = '0.1.0'
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
'PAR_LIST', 'DEFAULT_NAME', 'EXE_DEFAULT', 'OMP_DEFAULT',
|
|
22
|
+
'default_mdu_text', 'default_data',
|
|
23
|
+
'parse_mdu', 'read_mdu', 'read_xlsx', 'write_xlsx', 'read_any',
|
|
24
|
+
'check_groups', 'to_mdu_text', 'bat_text', 'name_from_path', 'write_mdu',
|
|
25
|
+
'ensure_end_rows', 'update_ntimesteps', 'main', '__version__',
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(argv=None):
|
|
30
|
+
"""Console entry point (see ffmgen.cli)."""
|
|
31
|
+
from .cli import main as _main
|
|
32
|
+
return _main(argv)
|
ffmgen/__main__.py
ADDED
ffmgen/cli.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Command line interface.
|
|
2
|
+
|
|
3
|
+
ffmgen open the GUI (loads ./mdu_prep.xlsx if present)
|
|
4
|
+
ffmgen gui [SRC] open the GUI with SRC (.xlsx or .mdu)
|
|
5
|
+
ffmgen generate SRC [options] write <name>.mdu (+ _run_<name>.bat) without the GUI
|
|
6
|
+
ffmgen convert SRC DST .mdu <-> .xlsx
|
|
7
|
+
ffmgen template [DST] write the built-in template .mdu (stdout if no DST)
|
|
8
|
+
"""
|
|
9
|
+
import argparse
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .core import (DEFAULT_NAME, EXE_DEFAULT, OMP_DEFAULT, check_groups, default_data,
|
|
14
|
+
default_mdu_text, name_from_path, read_any, to_mdu_text, write_mdu, write_xlsx)
|
|
15
|
+
from .timecalc import ensure_end_rows, update_ntimesteps
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _build_parser():
|
|
19
|
+
p = argparse.ArgumentParser(prog='ffmgen', description='Build 3Di subgrid .mdu files.')
|
|
20
|
+
p.add_argument('--version', action='version', version='%(prog)s ' + __version__)
|
|
21
|
+
sub = p.add_subparsers(dest='cmd')
|
|
22
|
+
|
|
23
|
+
g = sub.add_parser('gui', help='open the desktop GUI')
|
|
24
|
+
g.add_argument('src', nargs='?', help='.xlsx or .mdu to load at start (default ./mdu_prep.xlsx)')
|
|
25
|
+
g.add_argument('-o', '--out-dir', default='.', help='initial output folder')
|
|
26
|
+
|
|
27
|
+
gen = sub.add_parser('generate', help='write .mdu (+ .bat) from an .xlsx/.mdu')
|
|
28
|
+
gen.add_argument('src', nargs='?', help='.xlsx or .mdu (omit to use the built-in template)')
|
|
29
|
+
gen.add_argument('-o', '--out-dir', default='.', help='output folder (default .)')
|
|
30
|
+
gen.add_argument('-n', '--name', help='MDU name (default: derived from SRC)')
|
|
31
|
+
gen.add_argument('--exe', default=EXE_DEFAULT, help='executable in the .bat (default %(default)s)')
|
|
32
|
+
gen.add_argument('-t', '--threads', type=int, default=OMP_DEFAULT, help='OMP_NUM_THREADS (default %(default)s)')
|
|
33
|
+
gen.add_argument('--no-bat', action='store_true', help='do not write _run_<name>.bat')
|
|
34
|
+
gen.add_argument('--keep-ntimesteps', action='store_true',
|
|
35
|
+
help='do not recompute NTimesteps from EndDate/EndTime')
|
|
36
|
+
gen.add_argument('--dry-run', action='store_true', help='print the .mdu instead of writing files')
|
|
37
|
+
|
|
38
|
+
c = sub.add_parser('convert', help='convert between .mdu and .xlsx')
|
|
39
|
+
c.add_argument('src')
|
|
40
|
+
c.add_argument('dst', help='output .xlsx or .mdu')
|
|
41
|
+
|
|
42
|
+
t = sub.add_parser('template', help='write the built-in template .mdu')
|
|
43
|
+
t.add_argument('dst', nargs='?')
|
|
44
|
+
return p
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _generate(a):
|
|
48
|
+
if a.src:
|
|
49
|
+
data, name = read_any(a.src), name_from_path(a.src)
|
|
50
|
+
else:
|
|
51
|
+
data, name = default_data(), DEFAULT_NAME
|
|
52
|
+
data = ensure_end_rows(data)
|
|
53
|
+
check_groups(data.keys())
|
|
54
|
+
if not a.keep_ntimesteps:
|
|
55
|
+
update_ntimesteps(data)
|
|
56
|
+
if a.dry_run:
|
|
57
|
+
sys.stdout.write(to_mdu_text(data))
|
|
58
|
+
return 0
|
|
59
|
+
write_mdu(data, out_dir=a.out_dir, name=a.name or name,
|
|
60
|
+
write_bat=not a.no_bat, threads=a.threads, exe=a.exe)
|
|
61
|
+
return 0
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _convert(a):
|
|
65
|
+
data = read_any(a.src)
|
|
66
|
+
if a.dst.lower().endswith('.xlsx'):
|
|
67
|
+
write_xlsx(ensure_end_rows(data), a.dst)
|
|
68
|
+
elif a.dst.lower().endswith('.mdu'):
|
|
69
|
+
with open(a.dst, 'wt', encoding='utf-8', newline='') as f:
|
|
70
|
+
f.write(to_mdu_text(data))
|
|
71
|
+
else:
|
|
72
|
+
raise SystemExit('dst must end with .xlsx or .mdu')
|
|
73
|
+
print('Wrote', a.dst)
|
|
74
|
+
return 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def main(argv=None):
|
|
78
|
+
a = _build_parser().parse_args(argv)
|
|
79
|
+
if a.cmd in (None, 'gui'):
|
|
80
|
+
from .gui import run # tkinter imported only when needed
|
|
81
|
+
run(src=getattr(a, 'src', None), out_dir=getattr(a, 'out_dir', '.'))
|
|
82
|
+
return 0
|
|
83
|
+
if a.cmd == 'generate':
|
|
84
|
+
return _generate(a)
|
|
85
|
+
if a.cmd == 'convert':
|
|
86
|
+
return _convert(a)
|
|
87
|
+
if a.cmd == 'template':
|
|
88
|
+
if a.dst:
|
|
89
|
+
with open(a.dst, 'wt', encoding='utf-8', newline='') as f:
|
|
90
|
+
f.write(default_mdu_text())
|
|
91
|
+
print('Wrote', a.dst)
|
|
92
|
+
else:
|
|
93
|
+
sys.stdout.write(default_mdu_text())
|
|
94
|
+
return 0
|
|
95
|
+
return 1
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == '__main__':
|
|
99
|
+
sys.exit(main())
|
ffmgen/core.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""Read, write and convert 3Di subgrid .mdu parameter sets.
|
|
2
|
+
|
|
3
|
+
A parameter set is an ``OrderedDict`` mapping group name -> list of rows:
|
|
4
|
+
|
|
5
|
+
* ``'info'`` rows are ``[text1, text2, text3]`` (header comment lines).
|
|
6
|
+
* every other group has rows ``[on, key, value, comment]`` where ``on`` is a
|
|
7
|
+
bool; ``on=False`` rows are written commented out with a leading ``#``.
|
|
8
|
+
|
|
9
|
+
The xlsx layout (one sheet per group, first row = header) is the same as the
|
|
10
|
+
``mdu_prep.xlsx`` workbooks used by mdugen v5.
|
|
11
|
+
"""
|
|
12
|
+
import os
|
|
13
|
+
from collections import OrderedDict
|
|
14
|
+
from importlib import resources
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
'PAR_LIST', 'DEFAULT_NAME', 'EXE_DEFAULT', 'OMP_DEFAULT',
|
|
20
|
+
'default_mdu_text', 'default_data',
|
|
21
|
+
'parse_mdu', 'read_mdu', 'read_xlsx', 'write_xlsx', 'read_any',
|
|
22
|
+
'check_groups', 'to_mdu_text', 'bat_text', 'name_from_path', 'write_mdu',
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
PAR_LIST = ['info', 'model', 'geometry',
|
|
26
|
+
'grid', 'initialization', 'defaults',
|
|
27
|
+
'numerics', 'physics', 'hydrology',
|
|
28
|
+
'time', 'external forcing',
|
|
29
|
+
'output', 'display', 'colors']
|
|
30
|
+
|
|
31
|
+
DEFAULT_NAME = 'STAMFORD_basis' # output name when the built-in template is used
|
|
32
|
+
EXE_DEFAULT = r'..\bin\subgridf90.exe'
|
|
33
|
+
OMP_DEFAULT = 3
|
|
34
|
+
_TEMPLATE = 'STAMFORD_basis.mdu'
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def default_mdu_text():
|
|
38
|
+
"""Text of the built-in template .mdu shipped with the package."""
|
|
39
|
+
return resources.files('ffmgen').joinpath('templates', _TEMPLATE).read_text(encoding='utf-8')
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def default_data():
|
|
43
|
+
"""Built-in template parsed into the group/rows structure."""
|
|
44
|
+
return parse_mdu(default_mdu_text())
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _clean(x):
|
|
48
|
+
"""Cell value -> str, NaN/NaT/None -> ''."""
|
|
49
|
+
if x is None:
|
|
50
|
+
return ''
|
|
51
|
+
try:
|
|
52
|
+
if pd.isna(x):
|
|
53
|
+
return ''
|
|
54
|
+
except (TypeError, ValueError):
|
|
55
|
+
pass
|
|
56
|
+
return str(x)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parse_mdu(text):
|
|
60
|
+
"""Parse .mdu text (as written by mdugen) into the group/rows structure."""
|
|
61
|
+
data = OrderedDict([('info', [])])
|
|
62
|
+
group = 'info'
|
|
63
|
+
for line in text.splitlines():
|
|
64
|
+
s = line.strip()
|
|
65
|
+
if not s:
|
|
66
|
+
continue
|
|
67
|
+
if s.startswith('[') and s.endswith(']'):
|
|
68
|
+
group = s[1:-1].strip()
|
|
69
|
+
data.setdefault(group, [])
|
|
70
|
+
continue
|
|
71
|
+
if group == 'info':
|
|
72
|
+
data['info'].append([s.lstrip('#').strip(), '', ''])
|
|
73
|
+
continue
|
|
74
|
+
on = not s.startswith('#')
|
|
75
|
+
body = s if on else s[1:]
|
|
76
|
+
if '=' not in body:
|
|
77
|
+
continue
|
|
78
|
+
key, rest = body.split('=', 1)
|
|
79
|
+
val, com = (rest.split('#', 1) + [''])[:2]
|
|
80
|
+
data[group].append([on, key.strip(), val.strip(), com.strip()])
|
|
81
|
+
return data
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def read_mdu(path):
|
|
85
|
+
"""Read an existing .mdu file (utf-8, falls back to cp950 / latin-1)."""
|
|
86
|
+
for enc in ('utf-8-sig', 'cp950', 'latin-1'):
|
|
87
|
+
try:
|
|
88
|
+
with open(path, encoding=enc) as f:
|
|
89
|
+
return parse_mdu(f.read())
|
|
90
|
+
except UnicodeDecodeError:
|
|
91
|
+
continue
|
|
92
|
+
raise ValueError('Could not decode %s' % path) # unreachable: latin-1 always decodes
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def read_xlsx(path):
|
|
96
|
+
"""Read an mdu_prep-style workbook: one sheet per group, first row = header."""
|
|
97
|
+
xf = pd.ExcelFile(path)
|
|
98
|
+
data = OrderedDict()
|
|
99
|
+
for sht in xf.sheet_names:
|
|
100
|
+
df = pd.read_excel(xf, sheet_name=sht, index_col=None, dtype=object)
|
|
101
|
+
rows = []
|
|
102
|
+
for r in df.itertuples(index=False):
|
|
103
|
+
r = list(r) + [None] * 4
|
|
104
|
+
if sht == 'info':
|
|
105
|
+
rows.append([_clean(x).strip() for x in r[:3]])
|
|
106
|
+
else:
|
|
107
|
+
try:
|
|
108
|
+
on = float(r[0]) == 1
|
|
109
|
+
except (TypeError, ValueError):
|
|
110
|
+
on = False
|
|
111
|
+
rows.append([on, _clean(r[1]).strip(), _clean(r[2]).strip(), _clean(r[3]).strip()])
|
|
112
|
+
data[sht] = rows
|
|
113
|
+
return data
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def read_any(path):
|
|
117
|
+
"""Read a .mdu or an .xlsx parameter file, chosen by extension."""
|
|
118
|
+
return read_mdu(path) if str(path).lower().endswith('.mdu') else read_xlsx(path)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def write_xlsx(data, path):
|
|
122
|
+
"""Write the parameters to a workbook with the mdu_prep sheet layout."""
|
|
123
|
+
with pd.ExcelWriter(path, engine='openpyxl') as xw:
|
|
124
|
+
for g, rows in data.items():
|
|
125
|
+
if g == 'info':
|
|
126
|
+
df = pd.DataFrame(rows, columns=['col1', 'col2', 'col3'])
|
|
127
|
+
else:
|
|
128
|
+
df = pd.DataFrame([[1 if on else 0, k, v, c] for on, k, v, c in rows],
|
|
129
|
+
columns=['use', 'parameter', 'value', 'description'])
|
|
130
|
+
df.to_excel(xw, sheet_name=g[:31], index=False)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def check_groups(names):
|
|
134
|
+
"""Print whether the parameter groups match PAR_LIST. Returns (extra, missing)."""
|
|
135
|
+
names = set(names)
|
|
136
|
+
extra = sorted(names.difference(PAR_LIST))
|
|
137
|
+
missing = sorted(set(PAR_LIST).difference(names))
|
|
138
|
+
if not extra and not missing:
|
|
139
|
+
print('The number of parameter groups is correct.')
|
|
140
|
+
if extra:
|
|
141
|
+
print('Unknown parameter group {0} exist in the inputfile.'.format(extra))
|
|
142
|
+
if missing:
|
|
143
|
+
print('Parameter group {0} missing in the inputfile.'.format(missing))
|
|
144
|
+
return extra, missing
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def to_mdu_text(data):
|
|
148
|
+
"""Render the parameter set as .mdu text (mdugen v5 line format)."""
|
|
149
|
+
lines = []
|
|
150
|
+
for g, rows in data.items():
|
|
151
|
+
if g == 'info':
|
|
152
|
+
for r in rows:
|
|
153
|
+
txt = ' '.join(str(x).strip() for x in r if str(x).strip())
|
|
154
|
+
if txt:
|
|
155
|
+
lines.append('# ' + txt)
|
|
156
|
+
lines.append('')
|
|
157
|
+
continue
|
|
158
|
+
lines.append('[' + g + ']')
|
|
159
|
+
for on, key, val, com in rows:
|
|
160
|
+
key = str(key).strip()
|
|
161
|
+
if not key:
|
|
162
|
+
continue # empty parameter name -> skipped
|
|
163
|
+
prefix = '' if on else '#'
|
|
164
|
+
lines.append('{0}{1:30s} = {2:45s} # {3}'.format(prefix, key, str(val).strip(), str(com).strip()))
|
|
165
|
+
lines.append('')
|
|
166
|
+
return '\n'.join(lines) + '\n'
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def bat_text(mdu_name, threads=OMP_DEFAULT, exe=EXE_DEFAULT):
|
|
170
|
+
"""Content of the _run_<name>.bat launcher."""
|
|
171
|
+
return 'set OMP_NUM_THREADS={0}\n{1} {2}\n'.format(threads, exe, mdu_name)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def name_from_path(path):
|
|
175
|
+
"""mdu_prep.xlsx -> 'prep' (strip a leading 'mdu_' and the extension)."""
|
|
176
|
+
base = os.path.splitext(os.path.basename(path))[0]
|
|
177
|
+
return base[4:] if base.lower().startswith('mdu_') else base
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def write_mdu(data, out_dir='.', name='model', write_bat=True,
|
|
181
|
+
threads=OMP_DEFAULT, exe=EXE_DEFAULT):
|
|
182
|
+
"""Write <out_dir>/<name>.mdu and, optionally, _run_<name>.bat.
|
|
183
|
+
|
|
184
|
+
``data`` is written as given; call :func:`ffmgen.update_ntimesteps` first
|
|
185
|
+
if NTimesteps should be derived from EndDate/EndTime.
|
|
186
|
+
Returns ``(mdu_path, bat_path_or_None)``.
|
|
187
|
+
"""
|
|
188
|
+
name = (name or 'model').strip()
|
|
189
|
+
if name.lower().endswith('.mdu'):
|
|
190
|
+
name = name[:-4]
|
|
191
|
+
out_dir = out_dir or '.'
|
|
192
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
193
|
+
mdu_file = os.path.join(out_dir, name + '.mdu')
|
|
194
|
+
with open(mdu_file, 'wt', encoding='utf-8', newline='') as f:
|
|
195
|
+
f.write(to_mdu_text(data))
|
|
196
|
+
print('Writing input file done ->', os.path.abspath(mdu_file))
|
|
197
|
+
bat_file = None
|
|
198
|
+
if write_bat:
|
|
199
|
+
bat_file = os.path.join(out_dir, '_run_' + name + '.bat')
|
|
200
|
+
with open(bat_file, 'wt', encoding='utf-8', newline='') as f:
|
|
201
|
+
f.write(bat_text(name + '.mdu', threads, exe))
|
|
202
|
+
print('Run script ->', os.path.abspath(bat_file))
|
|
203
|
+
return mdu_file, bat_file
|