makepst 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {makepst-0.1.0 → makepst-0.2.0}/CHANGELOG.md +25 -0
- {makepst-0.1.0 → makepst-0.2.0}/CITATION.cff +1 -1
- {makepst-0.1.0 → makepst-0.2.0}/MANIFEST.in +2 -1
- {makepst-0.1.0 → makepst-0.2.0}/PKG-INFO +67 -9
- {makepst-0.1.0 → makepst-0.2.0}/README.md +66 -8
- {makepst-0.1.0 → makepst-0.2.0}/makepst/__init__.py +4 -2
- {makepst-0.1.0 → makepst-0.2.0}/makepst/checks.py +114 -2
- {makepst-0.1.0 → makepst-0.2.0}/makepst/cli.py +220 -5
- {makepst-0.1.0 → makepst-0.2.0}/makepst/excel.py +193 -36
- makepst-0.2.0/makepst/model_files.py +181 -0
- makepst-0.2.0/makepst/rules.py +403 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst.egg-info/PKG-INFO +67 -9
- {makepst-0.1.0 → makepst-0.2.0}/makepst.egg-info/SOURCES.txt +4 -0
- {makepst-0.1.0 → makepst-0.2.0}/pyproject.toml +1 -1
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_checks.py +20 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_fixture.py +35 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_makepst.py +2 -1
- makepst-0.2.0/tests/test_model_files.py +207 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_provenance.py +3 -5
- makepst-0.2.0/tests/test_rules.py +194 -0
- {makepst-0.1.0 → makepst-0.2.0}/LICENSE +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/control.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/flow.ins +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/flow.out +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/heads.ins +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/heads.out +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/io.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/model.in +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/model.par +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/model.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/model.res +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/model.tpl +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/obs.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/par.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/examples/minimal/pargp.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/__main__.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/diff.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/phi.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/provenance.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/pst.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/pyemu_bridge.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/reader.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/sections.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/starter.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst/writer.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst.egg-info/dependency_links.txt +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst.egg-info/entry_points.txt +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst.egg-info/requires.txt +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/makepst.egg-info/top_level.txt +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/setup.cfg +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.3.obs.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.3.par.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.par +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.phi.actual.csv +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.pst +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.res +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/data/demo.xlsx +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/make_fixture.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_diff.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_examples.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_phi.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_pyemu.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_starter.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_v2.py +0 -0
- {makepst-0.1.0 → makepst-0.2.0}/tests/test_version.py +0 -0
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.0 (2026-09-19)
|
|
4
|
+
|
|
5
|
+
- `update`: the xlwings backend reads only headers, name and target columns, writes contiguous
|
|
6
|
+
cells in one call per run, suspends events and calculation while writing and recalculates
|
|
7
|
+
once before saving (1,068 parameters over eight sheets of a 27-sheet workbook: ~9 s, was
|
|
8
|
+
minutes); `--par` with `--pst` no longer rewrites the observation sheets
|
|
9
|
+
- `tempchek`: check a template file or write model input files from parameter values, with
|
|
10
|
+
PEST's number writer (maximum precision in the space, `PRECIS` / `DPOINT`, one word per
|
|
11
|
+
parameter, right-justified; checked against `tempchek.exe`); for a template + `.par` file
|
|
12
|
+
or for a whole case
|
|
13
|
+
- `inschek`: check an instruction file or read a model output file with it into an `.obf`
|
|
14
|
+
file, for one file or a whole case
|
|
15
|
+
- `validate`: values that cannot be written into their template space are errors; a line
|
|
16
|
+
advance not at the start of an instruction line and `t` / fixed instructions moving left
|
|
17
|
+
are errors (INSCHEK's rules); three more pestchek warnings (empty observation group,
|
|
18
|
+
`ICOV`/`ICOR`/`IEIG` with more than 300 adjustable parameters, PEST_HP-only variables —
|
|
19
|
+
the last as a note); an all-zero-weight observation group is now a note, not a warning
|
|
20
|
+
- `update`: cells inside array formulas or dynamic-array spill ranges are left alone (Excel
|
|
21
|
+
refuses them; xlwings no longer shows a dialog); names present on only one side are reported
|
|
22
|
+
with counts and examples
|
|
23
|
+
- `validate`: pestchek's rules added (parameter data, groups, observations, prior information,
|
|
24
|
+
~60 control-variable ranges and consistency rules, template and instruction syntax), derived
|
|
25
|
+
from PEST 17's `pestchek.F` / `cheksub.F`; file paths resolved from the directory PEST runs
|
|
26
|
+
in, not only the control file's folder
|
|
27
|
+
|
|
3
28
|
## 0.1.0 (2026-09-19)
|
|
4
29
|
|
|
5
30
|
First packaged release; replaces the single-file `makePst.py` script.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
include LICENSE README.md CHANGELOG.md CITATION.cff
|
|
2
2
|
recursive-include tests *.py *.xlsx *.pst *.par *.res *.csv
|
|
3
|
-
recursive-include examples *.csv *.py *.tpl *.ins *.in *.out *.par *.res
|
|
3
|
+
recursive-include examples/minimal *.csv *.py *.tpl *.ins *.in *.out *.par *.res
|
|
4
|
+
prune examples/mnnrd
|
|
4
5
|
global-exclude __pycache__ *.py[cod] *.manifest.json
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: makepst
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Build PEST / PEST++ / PEST_HP control files from spreadsheet tables, read them back, and push results into the workbook
|
|
5
5
|
Author-email: Michael Ou <ougengxin@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -120,6 +120,10 @@ makepst build model.pst regul --set_ctl_csv control.csv --add_pargp_csv pargp.cs
|
|
|
120
120
|
# 2. check it against the template / instruction / output files, like pestchek
|
|
121
121
|
makepst validate model.pst --outputs
|
|
122
122
|
|
|
123
|
+
# 2b. what PEST will do with them: write the model input file, read the model output files
|
|
124
|
+
makepst tempchek model.pst --dry_run
|
|
125
|
+
makepst inschek model.pst
|
|
126
|
+
|
|
123
127
|
# 3. control file -> workbook (CONTROL / PARGP / PAR / OBS / PRIOR / IO sheets + a BUILD sheet)
|
|
124
128
|
makepst dump model.pst model.xlsx
|
|
125
129
|
|
|
@@ -154,8 +158,10 @@ makepst validate model.pst --outputs
|
|
|
154
158
|
makepst validate book.xlsm --strict
|
|
155
159
|
```
|
|
156
160
|
|
|
157
|
-
See [Validation](#validation). Template and instruction paths are resolved
|
|
158
|
-
|
|
161
|
+
See [Validation](#validation). Template and instruction paths are resolved the way PEST
|
|
162
|
+
resolves them, from the directory PEST is run in: makePst tries the control file's folder,
|
|
163
|
+
the current directory and the control file's parent, uses the one where the files are, and
|
|
164
|
+
says which; `--base_dir` overrides that. `--strict` also fails on warnings,
|
|
159
165
|
`--quiet` hides the informational lines.
|
|
160
166
|
|
|
161
167
|
### build — tables → .pst
|
|
@@ -225,10 +231,19 @@ makepst update book.xlsm --obs_csv run.3.obs.csv --pst run.pst # IES simulated
|
|
|
225
231
|
|
|
226
232
|
Every worksheet whose header row contains `PARNME` (or `OBSNME`) is matched row by row on the
|
|
227
233
|
name, so tables spread over several sheets are all found; only the requested columns are
|
|
228
|
-
written and only for names present in the
|
|
229
|
-
|
|
234
|
+
written and only for names present in both: names in the results with no row in the workbook,
|
|
235
|
+
and workbook rows absent from the results, are left alone and reported as a warning with
|
|
236
|
+
counts and examples. Cells that hold formulas are left alone unless `--overwrite_formulas`;
|
|
237
|
+
cells inside an array formula or a dynamic-array spill range are never written (Excel would
|
|
238
|
+
refuse), only counted. `--sheet` (glob, case-insensitive) restricts the worksheets;
|
|
230
239
|
`--group` restricts the parameter / observation groups (a `.par` file carries no groups, so
|
|
231
|
-
add `--pst` to supply them).
|
|
240
|
+
add `--pst` to supply them). `--pst` on its own writes the control file's parameter and
|
|
241
|
+
observation columns; next to `--par`, `--res` or `--obs_csv` it only supplies groups and
|
|
242
|
+
measured values, and the observation sheets are not touched by a parameter update.
|
|
243
|
+
|
|
244
|
+
The xlwings backend reads only each sheet's header, name column and target columns, writes
|
|
245
|
+
contiguous runs of cells in one call each and recalculates once before saving: the
|
|
246
|
+
demonstration workbook (27 sheets, 1,068 parameters over eight sheets) updates in about 9 s.
|
|
232
247
|
|
|
233
248
|
PESTPP-IES ensembles (`case.N.par.csv`, `case.N.obs.csv`) are recognised by their
|
|
234
249
|
`real_name` header. `--real` picks the realization: a name (`base`, `17`) or `best` — the
|
|
@@ -268,6 +283,41 @@ to that workbook; other relative names resolve against the workbook's folder. No
|
|
|
268
283
|
route rebuilds from the workbook's *current* state, so any edits made since the original
|
|
269
284
|
control file was written are picked up too.
|
|
270
285
|
|
|
286
|
+
### tempchek — templates → model input files
|
|
287
|
+
|
|
288
|
+
```
|
|
289
|
+
makepst tempchek model.tpl # check the template, list its parameters
|
|
290
|
+
makepst tempchek model.tpl model.in [model.par] # write model.in (values from model.par by default)
|
|
291
|
+
makepst tempchek model.pst [--par run.par] [--real best] [--dry_run] # every template of the case
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
Like PEST's TEMPCHEK: a template is checked, then its model input file is written with
|
|
295
|
+
parameter values from a `.par` file (or, given a control file / workbook, from `PARVAL1`
|
|
296
|
+
with `SCALE` and `OFFSET` applied, or from `--par` — a `.par` file or an IES realization).
|
|
297
|
+
Numbers are written the way PEST writes them (its `WRTSIG`): into the parameter space with
|
|
298
|
+
as much precision as the space allows, fixed or exponential notation, right-justified, one
|
|
299
|
+
word per parameter sized for its narrowest space, `PRECIS` / `DPOINT` honoured
|
|
300
|
+
(`double`, `nopoint`). Checked against `tempchek.exe` on a grid of values and widths from 3
|
|
301
|
+
to 23 characters: the same fields are refused, and the written value is never further from
|
|
302
|
+
the true value than PEST's. A value that cannot be written (`1e-10` in a five-character
|
|
303
|
+
space, say) is an error and nothing is written; parameters in the `.par` file that the
|
|
304
|
+
template does not cite are warnings. `--dry_run` checks without writing.
|
|
305
|
+
|
|
306
|
+
### inschek — model output files → observation values
|
|
307
|
+
|
|
308
|
+
```
|
|
309
|
+
makepst inschek heads.ins # check the instruction file, list its observations
|
|
310
|
+
makepst inschek heads.ins heads.out [--out heads.obf]
|
|
311
|
+
makepst inschek model.pst [--out model.obf] # every instruction file of the case, one .obf
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
Like PEST's INSCHEK: the instruction file is checked (the syntax rules in the validation
|
|
315
|
+
table below, including that a line advance can only start an instruction line and that fixed
|
|
316
|
+
/ tab instructions must move right), then read against the model output file with the
|
|
317
|
+
interpreter that `validate --outputs` uses, and the values go to an observation value file
|
|
318
|
+
(`name value` per line, INSCHEK's `.obf`). With a control file, all instruction files are
|
|
319
|
+
read and observations the control file lists but no file produced are reported.
|
|
320
|
+
|
|
271
321
|
### diff — what changed
|
|
272
322
|
|
|
273
323
|
```
|
|
@@ -358,6 +408,12 @@ makepst validate tr13.xlsm # a workbook with a BUILD sheet is built f
|
|
|
358
408
|
| malformed prior equation; non-numeric right-hand side; `log()` used on a non-log parameter or vice versa | error |
|
|
359
409
|
| `DERCOM` beyond the number of model command lines | error |
|
|
360
410
|
| name longer than PEST's limit (12 parameter / 20 observation / 12 group; PEST++ allows 200) | warning |
|
|
411
|
+
| **pestchek's rules** (from PEST 17's `pestchek.F` / `cheksub.F`): `PARCHGLIM` vocabulary and `absolute(n)` needing `absparmax(n)=`; log parameters factor-limited with positive values; factor-limited bounds of one sign and non-zero; `SCALE ≠ 0`; tied to itself, tied or parent with initial value 0; group `none` reserved; all parameters fixed/tied; `INCTYP` / `FORCEN` / `DERMTHD` vocabularies and their compatibility; split-column ranges; duplicate groups; `dum` as an observation name; prior weights, labels (≤ 20, not an observation name), duplicated parameters, all-zero factors, group `predict`; regularisation needs a `regul*` group, prediction exactly one `predict` observation; ~60 control-variable range and consistency rules (`RLAMFAC`, `PHIRATSUF`, `FACPARMAX`, `NPHISTP`, `RELPARSTP < RELPARMAX`, `EIGTHRESH`, `PHIMLIM < PHIMACCEPT < 1.2·PHIMLIM`, `WFMIN ≤ WFINIT ≤ WFMAX`, `UPTESTMIN`/`UPTESTLIM`, SVD vs AUI vs LSQR exclusivity, …) | error |
|
|
412
|
+
| pestchek's warnings: derivative increment larger than a third of the range; initial value 0 with a relative increment; group with only fixed/tied parameters; log parameters with a non-relative increment; observation group listed but empty (dropped on write); `MAXSING` above the adjustable-parameter count; more than 300 adjustable parameters with `ICOV`/`ICOR`/`IEIG` on; `NOPTMAX 0` | warning |
|
|
413
|
+
| PEST_HP-only control variables present (`WIN_MRUN_HOURS`, `UPTESTMIN`, `RRFSAVE`, …; plain PEST needs `/hpstart`); observation group whose weights are all zero | note |
|
|
414
|
+
| template: parameter space narrower than 3 characters or containing a tab | error |
|
|
415
|
+
| template: a value that cannot be written into its space as PEST writes numbers (`PRECIS` / `DPOINT` honoured), e.g. `1e-10` in five characters | error |
|
|
416
|
+
| instruction file syntax: `l` / `t` integers, marker delimiters, `!` balance, `[name]n1:n2` order, `dum` in a fixed field, continuation `&` placement, `l` only at the start of a line, `t` and fixed columns moving left to right, unknown instructions — checked statically, without model output | error |
|
|
361
417
|
| section in the `.pst` that makePst drops on read | warning |
|
|
362
418
|
| template or instruction file missing; bad `ptf` / `pif` line | error |
|
|
363
419
|
| parameter cited in no template; template citing an unknown parameter | error |
|
|
@@ -383,7 +439,7 @@ from?" after the fact:
|
|
|
383
439
|
|
|
384
440
|
```json
|
|
385
441
|
{
|
|
386
|
-
"makepst": "0.
|
|
442
|
+
"makepst": "0.2.0",
|
|
387
443
|
"command": "build",
|
|
388
444
|
"argv": ["build", "tr13.pst", "regul", "--set_ctl_xls", "tr13.xlsm,CONTROL", "..."],
|
|
389
445
|
"created": "2026-09-19T13:51:39-04:00",
|
|
@@ -454,7 +510,7 @@ update_workbook('book.xlsm', par='run.par', res='run.res', out='book_results.xls
|
|
|
454
510
|
python -m pytest tests -q
|
|
455
511
|
```
|
|
456
512
|
|
|
457
|
-
|
|
513
|
+
106 tests, no external data needed: a synthetic workbook in the real project layout
|
|
458
514
|
(`tests/data/demo.xlsx`, generated by `tests/make_fixture.py`) with a golden control file,
|
|
459
515
|
`.par`, `.res` and IES ensemble files. They cover golden-file builds from Excel and CSV, the
|
|
460
516
|
`read → write → read` identity, `dump → build` byte equality, workbook updates (multi-sheet
|
|
@@ -476,11 +532,13 @@ makepst/
|
|
|
476
532
|
writer.py Pst -> .pst text (+ dump.tpl)
|
|
477
533
|
reader.py .pst text -> Pst
|
|
478
534
|
excel.py sheets -> Pst; Pst -> workbook; results -> existing workbook
|
|
479
|
-
cli.py init / validate / build / dump / update / parrep / diff
|
|
535
|
+
cli.py init / validate / build / dump / update / parrep / diff / tempchek / inschek
|
|
480
536
|
diff.py semantic comparison of two Pst objects
|
|
481
537
|
phi.py objective function by group from residuals; IES realization phis
|
|
482
538
|
pyemu_bridge.py to_pyemu / from_pyemu
|
|
483
539
|
checks.py the validate report: table checks, template / instruction cross-checks, instruction interpreter
|
|
540
|
+
rules.py pestchek's rules (parameters, groups, observations, prior information, control variables)
|
|
541
|
+
model_files.py PEST's number writer, template filling, .obf files (tempchek / inschek)
|
|
484
542
|
provenance.py the <output>.manifest.json sidecar
|
|
485
543
|
starter.py the `init` workbook: control-variable descriptions, example rows, drop-downs
|
|
486
544
|
examples/minimal/ the tutorial inputs
|
|
@@ -89,6 +89,10 @@ makepst build model.pst regul --set_ctl_csv control.csv --add_pargp_csv pargp.cs
|
|
|
89
89
|
# 2. check it against the template / instruction / output files, like pestchek
|
|
90
90
|
makepst validate model.pst --outputs
|
|
91
91
|
|
|
92
|
+
# 2b. what PEST will do with them: write the model input file, read the model output files
|
|
93
|
+
makepst tempchek model.pst --dry_run
|
|
94
|
+
makepst inschek model.pst
|
|
95
|
+
|
|
92
96
|
# 3. control file -> workbook (CONTROL / PARGP / PAR / OBS / PRIOR / IO sheets + a BUILD sheet)
|
|
93
97
|
makepst dump model.pst model.xlsx
|
|
94
98
|
|
|
@@ -123,8 +127,10 @@ makepst validate model.pst --outputs
|
|
|
123
127
|
makepst validate book.xlsm --strict
|
|
124
128
|
```
|
|
125
129
|
|
|
126
|
-
See [Validation](#validation). Template and instruction paths are resolved
|
|
127
|
-
|
|
130
|
+
See [Validation](#validation). Template and instruction paths are resolved the way PEST
|
|
131
|
+
resolves them, from the directory PEST is run in: makePst tries the control file's folder,
|
|
132
|
+
the current directory and the control file's parent, uses the one where the files are, and
|
|
133
|
+
says which; `--base_dir` overrides that. `--strict` also fails on warnings,
|
|
128
134
|
`--quiet` hides the informational lines.
|
|
129
135
|
|
|
130
136
|
### build — tables → .pst
|
|
@@ -194,10 +200,19 @@ makepst update book.xlsm --obs_csv run.3.obs.csv --pst run.pst # IES simulated
|
|
|
194
200
|
|
|
195
201
|
Every worksheet whose header row contains `PARNME` (or `OBSNME`) is matched row by row on the
|
|
196
202
|
name, so tables spread over several sheets are all found; only the requested columns are
|
|
197
|
-
written and only for names present in the
|
|
198
|
-
|
|
203
|
+
written and only for names present in both: names in the results with no row in the workbook,
|
|
204
|
+
and workbook rows absent from the results, are left alone and reported as a warning with
|
|
205
|
+
counts and examples. Cells that hold formulas are left alone unless `--overwrite_formulas`;
|
|
206
|
+
cells inside an array formula or a dynamic-array spill range are never written (Excel would
|
|
207
|
+
refuse), only counted. `--sheet` (glob, case-insensitive) restricts the worksheets;
|
|
199
208
|
`--group` restricts the parameter / observation groups (a `.par` file carries no groups, so
|
|
200
|
-
add `--pst` to supply them).
|
|
209
|
+
add `--pst` to supply them). `--pst` on its own writes the control file's parameter and
|
|
210
|
+
observation columns; next to `--par`, `--res` or `--obs_csv` it only supplies groups and
|
|
211
|
+
measured values, and the observation sheets are not touched by a parameter update.
|
|
212
|
+
|
|
213
|
+
The xlwings backend reads only each sheet's header, name column and target columns, writes
|
|
214
|
+
contiguous runs of cells in one call each and recalculates once before saving: the
|
|
215
|
+
demonstration workbook (27 sheets, 1,068 parameters over eight sheets) updates in about 9 s.
|
|
201
216
|
|
|
202
217
|
PESTPP-IES ensembles (`case.N.par.csv`, `case.N.obs.csv`) are recognised by their
|
|
203
218
|
`real_name` header. `--real` picks the realization: a name (`base`, `17`) or `best` — the
|
|
@@ -237,6 +252,41 @@ to that workbook; other relative names resolve against the workbook's folder. No
|
|
|
237
252
|
route rebuilds from the workbook's *current* state, so any edits made since the original
|
|
238
253
|
control file was written are picked up too.
|
|
239
254
|
|
|
255
|
+
### tempchek — templates → model input files
|
|
256
|
+
|
|
257
|
+
```
|
|
258
|
+
makepst tempchek model.tpl # check the template, list its parameters
|
|
259
|
+
makepst tempchek model.tpl model.in [model.par] # write model.in (values from model.par by default)
|
|
260
|
+
makepst tempchek model.pst [--par run.par] [--real best] [--dry_run] # every template of the case
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Like PEST's TEMPCHEK: a template is checked, then its model input file is written with
|
|
264
|
+
parameter values from a `.par` file (or, given a control file / workbook, from `PARVAL1`
|
|
265
|
+
with `SCALE` and `OFFSET` applied, or from `--par` — a `.par` file or an IES realization).
|
|
266
|
+
Numbers are written the way PEST writes them (its `WRTSIG`): into the parameter space with
|
|
267
|
+
as much precision as the space allows, fixed or exponential notation, right-justified, one
|
|
268
|
+
word per parameter sized for its narrowest space, `PRECIS` / `DPOINT` honoured
|
|
269
|
+
(`double`, `nopoint`). Checked against `tempchek.exe` on a grid of values and widths from 3
|
|
270
|
+
to 23 characters: the same fields are refused, and the written value is never further from
|
|
271
|
+
the true value than PEST's. A value that cannot be written (`1e-10` in a five-character
|
|
272
|
+
space, say) is an error and nothing is written; parameters in the `.par` file that the
|
|
273
|
+
template does not cite are warnings. `--dry_run` checks without writing.
|
|
274
|
+
|
|
275
|
+
### inschek — model output files → observation values
|
|
276
|
+
|
|
277
|
+
```
|
|
278
|
+
makepst inschek heads.ins # check the instruction file, list its observations
|
|
279
|
+
makepst inschek heads.ins heads.out [--out heads.obf]
|
|
280
|
+
makepst inschek model.pst [--out model.obf] # every instruction file of the case, one .obf
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
Like PEST's INSCHEK: the instruction file is checked (the syntax rules in the validation
|
|
284
|
+
table below, including that a line advance can only start an instruction line and that fixed
|
|
285
|
+
/ tab instructions must move right), then read against the model output file with the
|
|
286
|
+
interpreter that `validate --outputs` uses, and the values go to an observation value file
|
|
287
|
+
(`name value` per line, INSCHEK's `.obf`). With a control file, all instruction files are
|
|
288
|
+
read and observations the control file lists but no file produced are reported.
|
|
289
|
+
|
|
240
290
|
### diff — what changed
|
|
241
291
|
|
|
242
292
|
```
|
|
@@ -327,6 +377,12 @@ makepst validate tr13.xlsm # a workbook with a BUILD sheet is built f
|
|
|
327
377
|
| malformed prior equation; non-numeric right-hand side; `log()` used on a non-log parameter or vice versa | error |
|
|
328
378
|
| `DERCOM` beyond the number of model command lines | error |
|
|
329
379
|
| name longer than PEST's limit (12 parameter / 20 observation / 12 group; PEST++ allows 200) | warning |
|
|
380
|
+
| **pestchek's rules** (from PEST 17's `pestchek.F` / `cheksub.F`): `PARCHGLIM` vocabulary and `absolute(n)` needing `absparmax(n)=`; log parameters factor-limited with positive values; factor-limited bounds of one sign and non-zero; `SCALE ≠ 0`; tied to itself, tied or parent with initial value 0; group `none` reserved; all parameters fixed/tied; `INCTYP` / `FORCEN` / `DERMTHD` vocabularies and their compatibility; split-column ranges; duplicate groups; `dum` as an observation name; prior weights, labels (≤ 20, not an observation name), duplicated parameters, all-zero factors, group `predict`; regularisation needs a `regul*` group, prediction exactly one `predict` observation; ~60 control-variable range and consistency rules (`RLAMFAC`, `PHIRATSUF`, `FACPARMAX`, `NPHISTP`, `RELPARSTP < RELPARMAX`, `EIGTHRESH`, `PHIMLIM < PHIMACCEPT < 1.2·PHIMLIM`, `WFMIN ≤ WFINIT ≤ WFMAX`, `UPTESTMIN`/`UPTESTLIM`, SVD vs AUI vs LSQR exclusivity, …) | error |
|
|
381
|
+
| pestchek's warnings: derivative increment larger than a third of the range; initial value 0 with a relative increment; group with only fixed/tied parameters; log parameters with a non-relative increment; observation group listed but empty (dropped on write); `MAXSING` above the adjustable-parameter count; more than 300 adjustable parameters with `ICOV`/`ICOR`/`IEIG` on; `NOPTMAX 0` | warning |
|
|
382
|
+
| PEST_HP-only control variables present (`WIN_MRUN_HOURS`, `UPTESTMIN`, `RRFSAVE`, …; plain PEST needs `/hpstart`); observation group whose weights are all zero | note |
|
|
383
|
+
| template: parameter space narrower than 3 characters or containing a tab | error |
|
|
384
|
+
| template: a value that cannot be written into its space as PEST writes numbers (`PRECIS` / `DPOINT` honoured), e.g. `1e-10` in five characters | error |
|
|
385
|
+
| instruction file syntax: `l` / `t` integers, marker delimiters, `!` balance, `[name]n1:n2` order, `dum` in a fixed field, continuation `&` placement, `l` only at the start of a line, `t` and fixed columns moving left to right, unknown instructions — checked statically, without model output | error |
|
|
330
386
|
| section in the `.pst` that makePst drops on read | warning |
|
|
331
387
|
| template or instruction file missing; bad `ptf` / `pif` line | error |
|
|
332
388
|
| parameter cited in no template; template citing an unknown parameter | error |
|
|
@@ -352,7 +408,7 @@ from?" after the fact:
|
|
|
352
408
|
|
|
353
409
|
```json
|
|
354
410
|
{
|
|
355
|
-
"makepst": "0.
|
|
411
|
+
"makepst": "0.2.0",
|
|
356
412
|
"command": "build",
|
|
357
413
|
"argv": ["build", "tr13.pst", "regul", "--set_ctl_xls", "tr13.xlsm,CONTROL", "..."],
|
|
358
414
|
"created": "2026-09-19T13:51:39-04:00",
|
|
@@ -423,7 +479,7 @@ update_workbook('book.xlsm', par='run.par', res='run.res', out='book_results.xls
|
|
|
423
479
|
python -m pytest tests -q
|
|
424
480
|
```
|
|
425
481
|
|
|
426
|
-
|
|
482
|
+
106 tests, no external data needed: a synthetic workbook in the real project layout
|
|
427
483
|
(`tests/data/demo.xlsx`, generated by `tests/make_fixture.py`) with a golden control file,
|
|
428
484
|
`.par`, `.res` and IES ensemble files. They cover golden-file builds from Excel and CSV, the
|
|
429
485
|
`read → write → read` identity, `dump → build` byte equality, workbook updates (multi-sheet
|
|
@@ -445,11 +501,13 @@ makepst/
|
|
|
445
501
|
writer.py Pst -> .pst text (+ dump.tpl)
|
|
446
502
|
reader.py .pst text -> Pst
|
|
447
503
|
excel.py sheets -> Pst; Pst -> workbook; results -> existing workbook
|
|
448
|
-
cli.py init / validate / build / dump / update / parrep / diff
|
|
504
|
+
cli.py init / validate / build / dump / update / parrep / diff / tempchek / inschek
|
|
449
505
|
diff.py semantic comparison of two Pst objects
|
|
450
506
|
phi.py objective function by group from residuals; IES realization phis
|
|
451
507
|
pyemu_bridge.py to_pyemu / from_pyemu
|
|
452
508
|
checks.py the validate report: table checks, template / instruction cross-checks, instruction interpreter
|
|
509
|
+
rules.py pestchek's rules (parameters, groups, observations, prior information, control variables)
|
|
510
|
+
model_files.py PEST's number writer, template filling, .obf files (tempchek / inschek)
|
|
453
511
|
provenance.py the <output>.manifest.json sidecar
|
|
454
512
|
starter.py the `init` workbook: control-variable descriptions, example rows, drop-downs
|
|
455
513
|
examples/minimal/ the tutorial inputs
|
|
@@ -3,9 +3,10 @@ try: # the installed package's metadata (
|
|
|
3
3
|
from importlib.metadata import version as _dist_version
|
|
4
4
|
__version__ = _dist_version('makepst')
|
|
5
5
|
except Exception: # a checkout run without installing
|
|
6
|
-
__version__ = '0.
|
|
6
|
+
__version__ = '0.2.0'
|
|
7
7
|
|
|
8
8
|
from .excel import load_table, to_workbook, update_workbook
|
|
9
|
+
from .model_files import fill_template, write_number
|
|
9
10
|
from .provenance import Manifest
|
|
10
11
|
from .pst import Pst, read_ensemble, read_obs_ensemble, read_par, read_res
|
|
11
12
|
from .reader import from_text, read_pst
|
|
@@ -22,4 +23,5 @@ def from_pyemu(ppst):
|
|
|
22
23
|
return _from(ppst)
|
|
23
24
|
|
|
24
25
|
__all__ = ['Pst', 'read_pst', 'from_text', 'write_pst', 'to_text',
|
|
25
|
-
'load_table', 'to_workbook', 'update_workbook', 'read_par', 'read_res', 'read_ensemble', 'read_obs_ensemble',
|
|
26
|
+
'load_table', 'to_workbook', 'update_workbook', 'read_par', 'read_res', 'read_ensemble', 'read_obs_ensemble',
|
|
27
|
+
'fill_template', 'write_number', 'Manifest', '__version__', 'to_pyemu', 'from_pyemu']
|
|
@@ -11,8 +11,10 @@ import re
|
|
|
11
11
|
|
|
12
12
|
import pandas as pd
|
|
13
13
|
|
|
14
|
+
from .model_files import fit_problems, scaled_values
|
|
14
15
|
from .pst import ADJUSTABLE, Pst, equation_params, is_number
|
|
15
16
|
from .sections import ALL_SECTIONS
|
|
17
|
+
from .writer import effective_control
|
|
16
18
|
|
|
17
19
|
KNOWN_SECTIONS = {s.name for s in ALL_SECTIONS} | {
|
|
18
20
|
'regularization', 'parameter groups', 'parameter data', 'observation groups', 'observation data',
|
|
@@ -173,9 +175,107 @@ def template_names(path):
|
|
|
173
175
|
return [m.strip().lower() for m in re.findall(f'{d}(.*?){d}', body)]
|
|
174
176
|
|
|
175
177
|
|
|
178
|
+
def template_problems(path):
|
|
179
|
+
"""pestchek's template checks beyond the names: spaces narrower than 3 characters, tabs inside them."""
|
|
180
|
+
with open(path, errors='replace') as f:
|
|
181
|
+
head = f.readline().split()
|
|
182
|
+
body = f.read()
|
|
183
|
+
if len(head) != 2:
|
|
184
|
+
return []
|
|
185
|
+
d = re.escape(head[1])
|
|
186
|
+
out = []
|
|
187
|
+
for m in re.finditer(f'{d}(.*?){d}', body):
|
|
188
|
+
space = m.group(0)
|
|
189
|
+
if '\t' in space:
|
|
190
|
+
out.append(f'tab character inside the parameter space for {m.group(1).strip()!r}')
|
|
191
|
+
elif len(space) < 3:
|
|
192
|
+
out.append(f'parameter space for {m.group(1).strip()!r} is narrower than 3 characters')
|
|
193
|
+
return sorted(set(out))
|
|
194
|
+
|
|
195
|
+
|
|
176
196
|
_INS_NAME = re.compile(r'!([^!\s]+)!|\[([^\]\s]+)\]|\(([^)\s]+)\)')
|
|
177
197
|
|
|
178
198
|
|
|
199
|
+
def instruction_problems(path):
|
|
200
|
+
"""pestchek's static instruction-file checks (syntax only; nothing is read from the model output)."""
|
|
201
|
+
with open(path, errors='replace') as f:
|
|
202
|
+
head = f.readline().split()
|
|
203
|
+
raw = f.read().splitlines()
|
|
204
|
+
if len(head) != 2 or head[0].lower() not in ('pif', 'jif'):
|
|
205
|
+
return ['first line must be "pif" or "jif" followed by the marker delimiter']
|
|
206
|
+
d = head[1]
|
|
207
|
+
if len(d) != 1 or d.isalnum() or d in '!&[]():':
|
|
208
|
+
return [f'illegal marker delimiter {d!r}']
|
|
209
|
+
tok_re = re.compile(re.escape(d) + '[^' + re.escape(d) + ']*' + re.escape(d) + r'|\S+')
|
|
210
|
+
out = []
|
|
211
|
+
first = True
|
|
212
|
+
col = 0 # rightmost column pinned by t / [ ] / ( ) on the current output line
|
|
213
|
+
for li, ln in enumerate(raw, 2):
|
|
214
|
+
s = ln.strip()
|
|
215
|
+
if not s:
|
|
216
|
+
continue
|
|
217
|
+
continued = s.startswith('&')
|
|
218
|
+
if continued:
|
|
219
|
+
if first:
|
|
220
|
+
out.append(f'line {li}: the first instruction line cannot begin with the continuation character "&"')
|
|
221
|
+
s = s[1:].strip()
|
|
222
|
+
else:
|
|
223
|
+
if s.count(d) % 2:
|
|
224
|
+
out.append(f'line {li}: marker delimiter {d!r} not closed')
|
|
225
|
+
lead = s.split()[0].lower()
|
|
226
|
+
if not (lead.startswith(d) or re.fullmatch(r'l\d+', lead)):
|
|
227
|
+
out.append(f'line {li}: an instruction line must begin with "l", a marker or "&"')
|
|
228
|
+
col = 0
|
|
229
|
+
first = False
|
|
230
|
+
for ti, t in enumerate(tok_re.findall(s)):
|
|
231
|
+
low = t.lower()
|
|
232
|
+
if t.startswith(d):
|
|
233
|
+
if len(t) < 3:
|
|
234
|
+
out.append(f'line {li}: marker has zero length')
|
|
235
|
+
elif '\t' in t:
|
|
236
|
+
out.append(f'line {li}: tab character inside marker {t}')
|
|
237
|
+
elif re.fullmatch(r'l\d+', low):
|
|
238
|
+
if int(low[1:]) <= 0:
|
|
239
|
+
out.append(f'line {li}: the integer after "l" must be positive')
|
|
240
|
+
if ti > 0 or continued:
|
|
241
|
+
out.append(f'line {li}: a line advance ({t}) can only occur at the beginning of an instruction line')
|
|
242
|
+
elif re.fullmatch(r't\d+', low):
|
|
243
|
+
if int(low[1:]) <= 0:
|
|
244
|
+
out.append(f'line {li}: the integer after "t" must be positive')
|
|
245
|
+
elif int(low[1:]) < col:
|
|
246
|
+
out.append(f'line {li}: {t} moves backwards; a model output line must be read from left to right')
|
|
247
|
+
else:
|
|
248
|
+
col = int(low[1:])
|
|
249
|
+
elif low == 'w':
|
|
250
|
+
pass
|
|
251
|
+
elif t.startswith('!'):
|
|
252
|
+
if not t.endswith('!') or t.count('!') != 2 or len(t) < 3:
|
|
253
|
+
out.append(f'line {li}: "!" not balanced in {t}')
|
|
254
|
+
elif len(t) - 2 > 20:
|
|
255
|
+
out.append(f'line {li}: observation name longer than 20 characters in {t}')
|
|
256
|
+
elif t[0] in '[(':
|
|
257
|
+
m = re.fullmatch(r'[\[(]([^\])]*)[\])](\d+):(\d+)', t)
|
|
258
|
+
if not m:
|
|
259
|
+
out.append(f'line {li}: fixed / semi-fixed instruction must be [name]n1:n2 or (name)n1:n2, got {t}')
|
|
260
|
+
elif not m.group(1):
|
|
261
|
+
out.append(f'line {li}: missing observation name in {t}')
|
|
262
|
+
elif m.group(1).lower() == 'dum':
|
|
263
|
+
out.append(f'line {li}: "dum" is only allowed for non-fixed (!dum!) observations')
|
|
264
|
+
elif int(m.group(2)) == 0 or int(m.group(3)) < int(m.group(2)):
|
|
265
|
+
out.append(f'line {li}: columns n1:n2 must be positive and increasing in {t}')
|
|
266
|
+
elif len(m.group(1)) > 20:
|
|
267
|
+
out.append(f'line {li}: observation name longer than 20 characters in {t}')
|
|
268
|
+
elif int(m.group(2)) < col:
|
|
269
|
+
out.append(f'line {li}: {t} starts left of column {col}; a model output line must be read from left to right')
|
|
270
|
+
else:
|
|
271
|
+
col = int(m.group(3))
|
|
272
|
+
elif low.startswith('&'):
|
|
273
|
+
out.append(f'line {li}: "&" is only allowed at the beginning of a line')
|
|
274
|
+
else:
|
|
275
|
+
out.append(f'line {li}: illegal instruction {t!r}')
|
|
276
|
+
return out
|
|
277
|
+
|
|
278
|
+
|
|
179
279
|
def instruction_names(path):
|
|
180
280
|
"""Observation names cited in an instruction file (lower-case, 'dum' excluded), in order."""
|
|
181
281
|
with open(path, errors='replace') as f:
|
|
@@ -198,6 +298,9 @@ def check_files(pst: Pst, base_dir='.', outputs=False):
|
|
|
198
298
|
rel = lambda p: os.path.join(base_dir, p) # noqa: E731
|
|
199
299
|
|
|
200
300
|
par_names = set(pst.par['PARNME'])
|
|
301
|
+
ctl = effective_control(pst)
|
|
302
|
+
precis, dpoint = ctl.get('precis', 'single'), ctl.get('dpoint', 'point')
|
|
303
|
+
values = scaled_values(pst.par)
|
|
201
304
|
cited = {}
|
|
202
305
|
for tpl, model_in in pst.tpl:
|
|
203
306
|
if not os.path.exists(rel(tpl)):
|
|
@@ -210,6 +313,12 @@ def check_files(pst: Pst, base_dir='.', outputs=False):
|
|
|
210
313
|
continue
|
|
211
314
|
if not names:
|
|
212
315
|
warn(tpl, 'cites no parameters')
|
|
316
|
+
problems = template_problems(rel(tpl))
|
|
317
|
+
for problem in problems:
|
|
318
|
+
err(tpl, problem)
|
|
319
|
+
if not problems: # as TEMPCHEK would write it: does every value fit its space?
|
|
320
|
+
for problem in fit_problems(rel(tpl), values, precis, dpoint):
|
|
321
|
+
err(tpl, f'value cannot be written into its space: {problem}')
|
|
213
322
|
unknown = sorted(set(names) - par_names)
|
|
214
323
|
if unknown:
|
|
215
324
|
err(tpl, f'cites parameters not in the control file: {_names(unknown)}')
|
|
@@ -236,6 +345,8 @@ def check_files(pst: Pst, base_dir='.', outputs=False):
|
|
|
236
345
|
continue
|
|
237
346
|
if not names:
|
|
238
347
|
warn(ins, 'reads no observations')
|
|
348
|
+
for problem in instruction_problems(rel(ins)):
|
|
349
|
+
err(ins, problem)
|
|
239
350
|
unknown = sorted(set(names) - obs_names)
|
|
240
351
|
if unknown:
|
|
241
352
|
err(ins, f'reads observations not in the control file: {_names(unknown)}')
|
|
@@ -267,7 +378,7 @@ def check_files(pst: Pst, base_dir='.', outputs=False):
|
|
|
267
378
|
for tok in cmd.split()[:2]:
|
|
268
379
|
looks_like_file = ('/' in tok or '\\' in tok or tok.lower().endswith(('.bat', '.exe', '.py', '.sh')))
|
|
269
380
|
if looks_like_file and not os.path.exists(rel(tok)):
|
|
270
|
-
warn('model command line', f'{tok} not found
|
|
381
|
+
warn('model command line', f'{tok} not found in {os.path.abspath(base_dir)}')
|
|
271
382
|
return out
|
|
272
383
|
|
|
273
384
|
|
|
@@ -399,7 +510,8 @@ def _store(values, name, text, li):
|
|
|
399
510
|
|
|
400
511
|
# ---------------------------------------------------------------------- driver
|
|
401
512
|
def validate(pst: Pst, base_dir='.', pst_path=None, outputs=False):
|
|
402
|
-
|
|
513
|
+
from .rules import check_extra
|
|
514
|
+
findings = check_tables(pst) + check_extra(pst)
|
|
403
515
|
if pst_path:
|
|
404
516
|
findings += check_sections(pst_path)
|
|
405
517
|
findings += check_files(pst, base_dir, outputs)
|