goPEST 0.0.11__py3-none-any.whl

gopest/data/goPESTpar.list

@@ -0,0 +1,95 @@
+# This is an example input file for goPESTpar.py 
+#
+# goPESTpar.py uses the content of this file to generate:
+# 1. the * parameter data section for the PEST control file.
+# 2. the template file (.tpl) to be used by PEST
+#
+# In PEST, each line of * parameter data should be:
+# PARNME PARTRANS PARCHGLIM PARVAL1 PARLBND PARUBND PARGP SCALE OFFSET DERCOM PARTIED
+#
+
+[Defaults]
+# default names of parameter data can be set by using 'PARNME = '
+# but to be effective, it has to be exactly two character long,
+# otherwise it will be overwritten with the default short alias 
+# of the parameter type, which is controlled in goPESTpar_def.py
+
+
+
+permeability_1_byrock:
+PARTRANS = 'log'
+PARGP = 'k1'
+PARLBND = 0.01e-15
+PARUBND = 4500.0e-15
+
+permeability_2_byrock:
+PARTRANS = 'log'
+PARGP = 'k2'
+PARLBND = 0.01e-15
+PARUBND = 4500.0e-15
+
+#Vertical permeability
+permeability_3_byrock:
+PARTRANS = 'log'
+PARGP = 'k3'
+PARLBND = 0.01e-15
+PARUBND = 3000.0e-15
+
+#Porosity
+porosity_byrock:
+PARTRANS = 'log'
+PARGP = 'por'
+PARLBND = 0.01
+PARUBND = 0.5
+
+massgener_rate:
+PARTRANS = 'log'
+PARGP = 'upflow'
+PARLBND = 0.01
+PARUBND = 50.0
+
+upflow_rech:
+PARTRANS = 'log'
+PARGP = 'rech'
+PARLBND = 1.0e-8
+PARUBND = 1.0e-5
+
+relative_perm_1:
+PARTRANS = 'log'
+PARGP = 'rpcap'
+PARLBND = 0.1
+PARUBND = 0.9
+
+
+[Param]
+ForEach: permeability_1_byrock,permeability_2_byrock,permeability_3_byrock
+ForEach: Rocktype
+'((?!ATMOS).....)'
+# match all rock types but not ATMOS, using look-around
+
+
+[END]
+
+
+[Param]
+ForEach: porosity_byrock
+ForEach: Rocktype
+'((?!ATMOS).....)'
+# match all rock types but not ATMOS, using look-around
+
+
+[Param]
+ForEach: massgener_rate
+ForEach: Generator
+'SO...'
+
+[Param]
+ForEach: upflow_rech
+ForEach: ConfigRechCoeff
+'.+'
+
+[Param]
+ForEach: relative_perm_1
+ForEach: NA
+'rp1'
+

gopest/make_case_pst.py

@@ -0,0 +1,229 @@
+from gopest.par import generate_params_and_tpl
+from gopest.obs import generate_obses_and_ins
+
+from gopest.common import config
+from gopest.common import runtime
+from gopest.common import check_required
+
+import os
+import re
+import importlib.resources as resources
+import shutil
+
+def replace_section(begin_sec, end_sec, pcf_text, repl):
+    """ returns PEST control file (passed in as a multiline string) with section
+    between begin_sec and end_sec replaced by string repl.  It does not matter
+    if repl has line breaks at the end or start, they will be cleaned up so the
+    final file has no empty lines. """
+    pattern = re.compile(r'\n *%s *\n.*?\n *%s *\n' % (re.escape(begin_sec), re.escape(end_sec)), flags=re.DOTALL)
+    repl = '\n%s\n%s\n%s\n' % (begin_sec, repl.strip("\n"), end_sec)
+    result, cnt = pattern.subn(repl, pcf_text)
+    if cnt < 1:
+        raise Exception("unable to find loaction between section %s and %s" % (begin_sec, end_sec))
+    return result
+
+def get_lines(filename):
+    """ return all non-empty lines froma file as a single string, with a line
+    count. """
+    f = open(filename, 'r')
+    lines, cnt = '', 0
+    for line in f:
+        if line.strip():
+            cnt += 1
+            lines += line
+    f.close()
+    return lines, cnt
+
+def replace_nth_line(longstring, i, repl):
+    """ replace ith line with repl, if repl is a function, it will be called to
+    process the original line. """
+    line_i = re.compile(r'([^\n]*\n){%i}' % i, flags=re.MULTILINE)
+    m = line_i.match(longstring)
+
+    if m:
+        if isinstance(repl, str):
+            real_repl = repl
+        elif hasattr(repl, '__call__'):
+            real_repl = repl(m.group(1))
+        else:
+            # if not a string or callable, try convert it into string
+            real_repl = str(repl)
+        return longstring[:m.start(1)] + real_repl.strip() + '\n' + longstring[m.end(1):]
+    else:
+        raise Exception("replace_nth_line() failed to replace line number %i" % i)
+
+#############################################################################
+
+def fixpcf_modelcmd(fpst):
+    """ update PEST case file (.pst) with model command line and distribution
+    files
+    """
+    fsave = runtime['filename']['save']
+    fincon = runtime['filename']['incon']
+    fdatns = runtime['filename']['dat_seq'][0]
+    flstpr = runtime['filename']['lst_seq'][-1]
+
+    fpst_bk = fpst + '.backup'
+    if not os.path.isfile(fpst):
+        print('Error: %s does not exist.' % fpst)
+        exit(1)
+    if os.path.isfile(fpst_bk):
+        os.remove(fpst_bk)
+    os.rename(fpst, fpst_bk)
+
+    try:
+        with open(fpst_bk,'r') as fin:
+            pcf_text = fin.read()
+
+        model_cmd = 'gopest run-pest-model\n'
+        model_inout = "\n".join([
+            'pest_model.tpl pest_model.dat',
+            'pest_model.ins  pest_model.obf',
+        ])
+        filedist = "\n".join([
+            '2 %s %s %s %s' % (fsave, fincon, fincon, fincon),
+            '1 %s %s.999' % (fsave, fincon),
+            '1 %s %s.999' % (fdatns, fdatns),
+            '1 %s %s.999' % (flstpr, flstpr),
+            '1 pest_model.dat pest_model.dat.999',
+            'command = "gopest save-iter-model"',
+        ])
+
+        pcf_text = replace_section("* model command line", "* model input/output",
+            pcf_text, model_cmd)
+        pcf_text = replace_section("* model input/output", "* prior information",
+            pcf_text, model_inout)
+        pcf_text = replace_section("* distribution files", "# end",
+            pcf_text, filedist)
+
+        with open(fpst, 'w') as fout:
+            fout.write(pcf_text)
+        print('+++ PEST case control file edited, original file saved as %s' % fpst_bk)
+    except Exception as e:
+        print(e)
+        print('update_case_pst.py unable to proceed, restoring.')
+        os.rename(fpst_bk, fpst)
+
+def fixpcf_parobs(fpst, dopar=True, doobs=True):
+    """ update PEST case file (.pst) with parameter and observation data
+    """
+    if dopar is False and doobs is False:
+        return
+
+    fpst_bk = fpst + '.backup'
+    if not os.path.isfile(fpst):
+        print('Error: %s does not exist.' % fpst)
+        exit(1)
+    if os.path.isfile(fpst_bk):
+        os.remove(fpst_bk)
+    os.rename(fpst, fpst_bk)
+
+    try:
+        with open(fpst_bk,'r') as fin:
+            pcf_text = fin.read()
+
+        if dopar:
+            par_data, n_par = get_lines('.pest_par_data')
+            pcf_text = replace_section("* parameter data", "* observation groups",
+                pcf_text, par_data)
+            print('+++ found %i parameters' % n_par)
+
+            # replace the count of parameters
+            def replace_par_cnts(orig):
+                """ first two number is n_par and n_obs """
+                nums = orig.split()
+                return ' '.join([str(n_par)] + nums[1:])
+            pcf_text = replace_nth_line(pcf_text, 4, replace_par_cnts)
+
+        if doobs:
+            obs_data, n_obs = get_lines('.pest_obs_data')
+            pcf_text = replace_section("* observation data", "* model command line",
+                pcf_text, obs_data)
+            print('+++ found %i observations' % n_obs)
+
+            # replace the count of parameters
+            def replace_obs_cnts(orig):
+                """ first two number is n_par and n_obs """
+                nums = orig.split()
+                return ' '.join(nums[:1] + [str(n_obs)] + nums[2:])
+            pcf_text = replace_nth_line(pcf_text, 4, replace_obs_cnts)
+
+        with open(fpst, 'w') as fout:
+            fout.write(pcf_text)
+        print('+++ PEST case control file edited, original file saved as %s' % fpst_bk)
+    except Exception as e:
+        print(e)
+        print('update_case_pst.py unable to proceed, restoring.')
+        os.rename(fpst_bk, fpst)
+
+def copy_model_files():
+    """ user specifies model's original files in [model.original]section
+    These files will be copied to the working directory, with goPEST's internal
+    naming convention.
+    """
+    def copy_to_cwd(filename, newbase):
+        """ copy to working dir and rename, but keeping all extention (to lower case) """
+        newname = newbase + os.path.splitext(filename)[1].lower()
+        print("  copy '%s' -> '%s'" % (filename, newname))
+        shutil.copy2(filename, newname)
+
+    for f in config['model']['original']['geometry-files']:
+        copy_to_cwd(f, './g_real_model')
+    copy_to_cwd(config['model']['original']['incon-file'], './real_model_incon')
+
+    # copy input and output files for the sequence of 'ns', 'pr', etc
+    sequence = config['model']['sequence']
+    for seq in sequence:
+        copy_to_cwd(config['model']['original']['%s-input-file' % seq], './real_model_%s' % seq)
+        if '%s-output-file' % seq in config['model']['original']:
+            copy_to_cwd(config['model']['original']['%s-output-file' % seq], './real_model_%s' % seq)
+    # make a copy of first dat to keep as original (handy for goPESTpar etc)
+    copy_to_cwd(config['model']['original']['%s-input-file' % sequence[0]], './real_model_original')
+
+def make_case_cli(argv=[]):
+    """ runs goPEST to set up par and obs entries """
+    for a in argv[1:]:
+        if a not in ['--no-copy', '--no-par', '--no-obs']:
+            raise Exception('Unrecognised option "%s".' % a)
+    if '--no-copy' in argv:
+        print('+++ use existing model files')
+    else:
+        print('+++ copy from original model files')
+        copy_model_files()
+
+    fgeo = runtime['filename']['geom']
+    fdato = runtime['filename']['dat_orig']
+    fdats = runtime['filename']['dat_seq']
+    fpst = config['pest']['case-name'] + '.pst'
+
+    check_required(fpst, 'PEST Case', fdefault='case.pst')
+
+    dopar = False
+    if '--no-par' not in argv:
+        check_required('goPESTpar.list', 'Parameter list')
+
+        print('+++ running goPEST to get par')
+        print('  gopestpar', fdato, 'pest_model.tpl', '.pest_par_data')
+        generate_params_and_tpl(fdato, 'pest_model.tpl', '.pest_par_data')
+        dopar = True
+
+    doobs = False
+    if '--no-obs' not in argv:
+        check_required('goPESTobs.list', 'Observation list')
+
+        print('+++ running goPEST to get obs')
+        print('  gopestobs', fgeo, fdats[-1], 'pest_model.ins', '.pest_obs_data')
+        generate_obses_and_ins(fgeo, fdats[-1], 'pest_model.ins', '.pest_obs_data')
+        doobs = True
+
+    # unfortunately I need to use 'real_model_original_pr.dat' here because it
+    # has many GENERs that may not exist in natural state, while still being
+    # needed in observations, eg. Production gener's block, hopefully this is
+    # okay because we usually don't need to get any actual values out of the
+    # real_model_original_pr.dat model.
+
+    fixpcf_parobs(fpst, dopar=dopar, doobs=doobs)
+    fixpcf_modelcmd(fpst)
+
+
+

gopest/obs.py

@@ -0,0 +1,297 @@
+import time
+import json
+import inspect
+
+from mulgrids import *
+from t2data import *
+from t2listing import *
+
+from gopest.common import Singleton
+from gopest.common import TwoWayDict
+from gopest.common import readList
+from gopest.common import updateObj
+from gopest.common import merge_dols
+from gopest import obs_def
+
+from gopest.utils.waiwera_listing import wlisting
+from gopest.utils.t2listingh5 import t2listingh5
+
+OBS_USER_FUNC = dict(inspect.getmembers(obs_def,inspect.isfunction))
+OBS_ALIAS = TwoWayDict(obs_def.shortNames)
+
+class PestObsDataName(Singleton):
+    """ remembers a list of observation data and observation data """
+    def __init__(self):
+        self.basenames = set([])
+    def newName(self,basename):
+        apnd = ' 01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+        i = 0
+        newname = (basename + apnd[i]).strip()
+        while newname in self.basenames:
+            i += 1
+            if i >= len(apnd):
+                raise Exception('Unable to find unused obseration name for:' + basename)
+                break
+            newname = (basename + apnd[i]).strip()
+        return newname
+    def add(self,newname):
+        self.basenames.add(newname)
+
+class PestObservData(object):
+    def __init__(self,OBSNME='',OBSVAL=0.0,WEIGHT=1.0,OBGNME=''):
+        self.OBSNME=OBSNME
+        self.OBSVAL=OBSVAL
+        self.WEIGHT=WEIGHT
+        self.OBGNME=OBGNME
+    def __repr__(self):
+        return  self.OBSNME +' '+ str(self.OBSVAL) +' '+ str(self.WEIGHT) +' '+ self.OBGNME
+
+class UserEntryObserv(object):
+    """
+        user defined routines
+    """
+    def __init__(self,obsType,obsInfo,fieldDataFile,customFilter,
+        offsetTime,obsDefault=PestObservData()):
+        self.obsType = obsType # a string
+        self.obsInfo = obsInfo # a list of anything
+        self.fieldDataFile = fieldDataFile # a string
+        self.customFilter = customFilter # an expression string
+        self.offsetTime = offsetTime # a number to offset model time
+        from copy import deepcopy
+        self.obsDefault = deepcopy(obsDefault)
+
+        self.all_obses = None
+        self.all_pst_lines = None # should be a list of strings
+        self.all_ins_lines = None # should be a list of strings
+        self.all_obf_lines = None # should be a list of strings
+
+        # this is expected to be directly written into self, by custom function
+        self.batch_plot_entry = []
+        self.coverage = {} # keyed by OBGNME, each group is a list of model blocks with data
+    def __repr__(self):
+        return '\n'.join([
+            '','[Obs]',
+            self.obsType,
+            ';'.join([str(info) for info in self.obsInfo]),
+            self.fieldDataFile,
+            self.customFilter,
+            str(self.obsDefault),
+            ''])
+    def makeObsDataInsLines(self,geo,dat):
+        """ generate a tuple of two lists:
+            - list of Pest * observation data """
+        # maybe + ('%5s' % str(self.obsInfo[0])) with fill?
+        if self.obsDefault.OBSNME == '':
+            newBaseName = PestObsDataName().newName(OBS_ALIAS[self.obsType])
+            PestObsDataName().add(newBaseName)
+            # user def routines use this as basename
+            self.obsDefault.OBSNME = newBaseName
+        # use obsType if not specified by user
+        if self.obsDefault.OBGNME == '':
+            self.obsDefault.OBGNME = self.obsType
+
+        ### this line does all the work
+        self.all_obses = OBS_USER_FUNC[self.obsType+'_fielddata'](geo,dat,self)
+
+        self.all_pst_lines = []
+        self.all_ins_lines = []
+        for obs in self.all_obses:
+            self.all_ins_lines.append('l1 [%s]21:41' % obs.OBSNME)
+            self.all_pst_lines.append(' %-20s %20.13e %12.5e %s' % (obs.OBSNME,
+                obs.OBSVAL,obs.WEIGHT,obs.OBGNME))
+
+    def makeObfLines(self,geo,dat,lst):
+        """ generate a list of:
+            - values for obf file that PEST requires """
+
+        ### this line does all the work
+        obfValues = OBS_USER_FUNC[self.obsType+'_modelresult'](geo,dat,lst,self)
+        self.all_obf_lines = []
+        for (v,obs) in zip(obfValues,self.all_obses):
+            self.all_obf_lines.append('%-20s %20.13e' % (obs.OBSNME,v))
+
+def readUserObservation(userListName):
+    """ returns a list of UserEntryObserv from reading the file with name
+        userObsListName """
+    userEntries = []
+    f = open(userListName,'r')
+    entryName, entry = readList(f)
+    f.close()
+    # some defaults even if no sections exists:
+    obsDefault = PestObservData()
+    customFilter = 'True'
+    offsetTime = 0.0
+    for i,en in enumerate(entryName):
+        if en == 'ObservationType':
+            obsType = entry[i][0].strip()
+            continue
+        if en == 'DataFilter':
+            if len(entry[i]) == 0:
+                # reset
+                customFilter = 'True'
+            else:
+                customFilter = entry[i][0]
+        if en == 'DataTimeOffset':
+            if len(entry[i]) == 0:
+                # reset
+                offsetTime = 0.0
+            else:
+                offsetTime = float(eval(entry[i][0]))
+        if en == 'Defaults':
+            obsDefault = updateObj(obsDefault,entry[i])
+        # if en == 'Obs':
+        #     if len(entry[i]) < 2: raise Exception
+        #     if ',' in entry[i][0]:
+        #         obsInfo = list(eval(entry[i][0]))
+        #     else:
+        #         obsInfo = [eval(entry[i][0])]
+        #     for fieldDataFile in entry[i][1:]:
+        #         userEntries.append(UserEntryObserv(obsType,obsInfo,
+        #             fieldDataFile.strip(),customFilter,offsetTime,
+        #             obsDefault))
+        if en == 'Obs':
+            if len(entry[i]) < 1:
+                raise Exception("An empty [Obs] entry is found in goPESTobs.list")
+            # pass the list of lines in, user functions to deal with them
+            obsInfo = [s.rstrip('\n') for s in entry[i]]
+            fieldDataFile = ''
+            userEntries.append(UserEntryObserv(obsType,obsInfo,
+                fieldDataFile.strip(),customFilter,offsetTime,
+                obsDefault))
+    return userEntries
+
+def generate_obses_and_ins(fgeo, fdat, insToWrite, fobses, fplts='goPESTobs.json', fcovs='goPESTobs.coverage'):
+    """ reads goPESTobs.list and generate observation data lines and instruction
+    file for PEST """
+    # reset unique obs name
+    obs_def.obsBaseNameCount = {}
+    geo = mulgrid(fgeo)
+    if fdat.endswith('.json'):
+        with open(fdat, 'r') as f:
+            dat = json.load(f)
+    else:
+        dat = t2data(fdat)
+
+    userEntries = readUserObservation('goPESTobs.list')
+    pstLines, insLines, plots, coverage = [], [], [], {}
+    for ue in userEntries:
+        ue.makeObsDataInsLines(geo,dat)
+        pstLines = pstLines + ue.all_pst_lines
+        insLines = insLines + ue.all_ins_lines
+        plots += ue.batch_plot_entry
+        coverage = merge_dols(coverage, ue.coverage)
+
+    f = open(insToWrite, 'w')
+    f.write('pif #\n')
+    for line in insLines:
+        f.write(line+'\n')
+    f.close()
+
+    obs = open(fobses, 'w')
+    for line in pstLines:
+        obs.write(line + '\n')
+    obs.close()
+
+    plt = open(fplts, 'w')
+    json.dump(plots, plt, indent=4, sort_keys=True)
+    plt.close()
+
+    cov = open(fcovs, 'w')
+    json.dump(coverage, cov, indent=4, sort_keys=True)
+    cov.close()
+
+def read_from_real_model(fgeo, fdat, flst, fobf, waiwera=False):
+    """ This reads TOUGH2's results and write in appropriate format into obf
+    file for PEST """
+    # reset unique obs name
+    obs_def.obsBaseNameCount = {}
+    geo = mulgrid(fgeo)
+    if waiwera:
+        with open(fdat, 'r') as f:
+            dat = json.load(f)
+        lst = wlisting(flst, geo, fjson=fdat)
+    else:
+        dat = t2data(fdat)
+        if flst.lower().endswith('.h5'):
+            lst = t2listingh5(flst)
+        else:
+            lst = t2listing(flst)
+
+    userEntries = readUserObservation('goPESTobs.list')
+    obfLines = []
+    for ue in userEntries:
+        ue.makeObsDataInsLines(geo,dat)
+        ue.makeObfLines(geo,dat,lst)
+        obfLines = obfLines + ue.all_obf_lines
+
+    if flst.lower().endswith('.listing'):
+        lst.close()
+
+    f = open(fobf,'w')
+    for line in obfLines:
+        f.write(line+'\n')
+    f.close()
+
+def goPESTobs(argv=[]):
+    START_TIME = time.time()
+
+    userlistname = 'goPESTobs.list'
+
+    if len(argv) not in [4,5]:
+        print('to generate PEST .pst observation section and .ins: ')
+        print('     gopest obs geo dat newPESTins')
+        print('to read Tough2 results and write result file for PEST to read:')
+        print('     gopest obs geo dat lst newPESTobf')
+
+    if len(argv) == 4:
+        fgeo = argv[1]
+        fdat = argv[2]
+        insToWrite = argv[3]
+
+        fobses = 'pest_obs_data'
+        fplts = 'goPESTobs.json'
+        fcovs = 'goPESTobs.coverage'
+        generate_obses_and_ins(fgeo, fdat, insToWrite,
+                               fobses, fplts, fcovs)
+
+    if len(argv) == 5:
+        
+        fgeo = argv[1]
+        fdat = argv[2]
+        flst = argv[3]
+        obfToWrite = argv[4]
+
+        geo = mulgrid(fgeo)
+        if fdat.lower().endswith('.json'):
+            with open(fdat, 'r') as f:
+                dat = json.load(f)
+        else:
+            dat = t2data(fdat)
+        
+        if flst.lower().endswith('.listing'):
+            lst = t2listing(flst)
+        elif flst.lower().endswith('.h5'):
+            if fdat.lower().endswith('.json'):
+                lst = wlisting(flst, geo, fjson=fdat)
+            else:
+                lst = t2listingh5(flst)
+
+
+
+        userEntries = readUserObservation(userlistname)
+        obfLines = []
+        for ue in userEntries:
+            ue.makeObsDataInsLines(geo,dat)
+            ue.makeObfLines(geo,dat,lst)
+            obfLines = obfLines + ue.all_obf_lines
+
+
+        f = open(obfToWrite,'w')
+        for line in obfLines:
+            f.write(line+'\n')
+        f.close()
+        
+    # print('goPESTobs finished after', (time.time() - START_TIME), 'seconds')
+        
+
+