goPEST 0.0.11__py3-none-any.whl

gopest/common.py

@@ -0,0 +1,194 @@
+import os
+import os.path
+import string
+import importlib.resources as resources
+import shutil
+
+# Access nested dictionary items via a list of keys
+from functools import reduce  # forward compatibility for Python 3
+import operator
+
+import tomlkit
+
+def check_required(finput, name, fdefault=None):
+    if not os.path.isfile(finput):
+        print("\nWarning! %s file '%s' is not found in current working directory:" % (name, finput))
+        print("    %s\n" % os.getcwd())
+        ans = input('Do you want goPEST to create a default file? (y/n) ')
+        if 'y' in ans.lower():
+            if fdefault is None:
+                fdefault = finput
+            default_finput = resources.files('gopest.data') / fdefault
+            with resources.as_file(default_finput) as f:
+                shutil.copyfile(f, finput)
+                print("Default %s file '%s' created, please review and re-run " \
+                      "command 'gopest init'." % (name, finput))
+            exit(0)
+        else:
+            print("Error! File '%s' is required for goPEST to work." % finput)
+            print('Existing...')
+            exit(1)
+
+
+check_required('goPESTconfig.toml', 'Configuration')
+
+""" this allows gopest.common.config to be used directly, eg.
+        from gopest.common import config as cfg
+        print(cfg['pest']['executable'])
+        print(cfg['simulator']['executable'])
+"""
+with open('goPESTconfig.toml', 'r') as f:
+    config = tomlkit.load(f)
+
+    if 'silent_slaves' not in config['pest']:
+        config['pest']['silent_slaves'] = True
+
+""" this allows gopest.common.runtime to be used directly, eg.
+        from gopest.common import runtime
+        print(runtime['filename']['fincon'])
+        print(runtime['filename']['fdatns'])
+"""
+runtime = {}
+def runtime_filenames(check=False):
+    """ work out filenames for all internal model files """
+    def getext(fn): return os.path.splitext(fn)[1].lower()
+    all_geoms = ['g_real_model' + getext(f) for f in config['model']['original']['geometry-files']]
+    input_typ = config['simulator']['input-type']
+    output_typ = config['simulator']['output-type']
+    sequence = config['model']['sequence']
+    if input_typ == 'aut2':
+        fsave = 'real_model_' + sequence[0] + '.save'
+        fincon = 'real_model_' + 'incon' + '.incon'
+        fdato = 'real_model_' + 'original' + '.dat'
+        fdats = ['real_model_' + seq + '.dat' for seq in sequence]
+        fincs = ['real_model_' + seq + '.incon' for seq in sequence]
+        fsavs = ['real_model_' + seq + '.save' for seq in sequence]
+        if output_typ == 'h5':
+            flsts = ['real_model_' + seq + '.h5' for seq in sequence]
+        else:
+            flsts = ['real_model_' + seq + '.listing' for seq in sequence]
+    elif input_typ == 'waiwera':
+        fsave = 'real_model_' + sequence[0] + '.h5'
+        fincon = 'real_model_' + 'incon' + '.h5'
+        fdato = 'real_model_' + 'original' + '.json'
+        fdats = ['real_model_' + seq + '.json' for seq in sequence]
+        flsts = ['real_model_' + seq + '.h5' for seq in sequence]
+        fincs = ['real_model_' + seq + '_inc.h5' for seq in sequence]
+        fsavs = ['real_model_' + seq + '.h5' for seq in sequence]
+    filenames = {
+        # single string properties
+        'geom': 'g_real_model.dat',
+        'save': fsave,
+        'incon': fincon,
+        'dat_orig': fdato,
+        # following peoperties are lists
+        'all_geoms': all_geoms,
+        'dat_seq': fdats,
+        'lst_seq': flsts,
+        'inc_seq': fincs,
+        'sav_seq': fsavs,
+    }
+    if check:
+        for k,fnames in filenames.items():
+            if not isinstance(fnames, list):
+                fnames = [fnames]
+            for fn in fnames:
+                if not os.path.exists(fn):
+                    raise Exception('Cannot find file %s in the working dir.' % fn)
+    return filenames
+runtime['filename'] = runtime_filenames()
+
+########## utility classes and functions
+class TwoWayDict(dict):
+    def __init__(self,copyFrom=None):
+        if copyFrom is not None:
+            for k in copyFrom.keys():
+                self[k] = copyFrom[k]
+    def __len__(self):
+        return dict.__len__(self) / 2
+    def __setitem__(self, key, value):
+        if key in self: raise Exception('TwoWayDict repeated key: %s' % str(key))
+        dict.__setitem__(self, key, value)
+        if value in self: raise Exception('TwoWayDict repeated key: %s' % str(value))
+        dict.__setitem__(self, value, key)
+
+class Singleton(object):
+    __single = None # the one, true Singleton
+    def __new__(classtype, *args, **kwargs):
+        # Check to see if a __single exists already for this class
+        # Compare class types instead of just looking for None so
+        # that subclasses will create their own __single objects
+        if classtype != type(classtype.__single):
+            classtype.__single = object.__new__(classtype, *args, **kwargs)
+        return classtype.__single
+
+def readList(f):
+    """ reads a file and returns a list of entries, an entry is a list strings
+        all lines starts with '#' are ignored """
+    entryName, entries = [], []
+    for line in f.readlines():
+        if len(line.strip()) == 0: continue  # empty lines
+        if line.strip()[0] == '#': continue  # comment lines
+        if line.strip()[0] == '[':           # [something] item opening
+            tmp = line.strip()
+            ikeyend=tmp.find(']')
+            keyword=tmp[1:ikeyend]
+            if keyword=='END': break         # [END] finishes all
+            entryName.append(keyword)
+            entries.append([])
+            continue
+        # all other non-empty lines parts of last entry (until [END])
+        # error if some non-comment line apear above first keyword
+        entries[-1].append(line)
+    return (entryName, entries)
+
+def updateDict(dic,lines):
+    """ evaluate lines and update the content of the argument dictionary.  A KeyError
+        exception is raised if the key from lines is not in dic.  Each line in lines
+        is a simple string like 'key = val'. """
+    keys,vals = [],[]
+    for line in lines:
+        if line.count('=') != 1: raise Exception
+        k,v = line.split('=')
+        if k.strip() not in dic:
+            print(dic.keys())
+            raise KeyError
+        dic[k.strip()] = eval(v)
+    return dic
+
+def merge_dols(dol1, dol2):
+    """ merging dicts of lists into a new dict of lists. """
+    keys = set(dol1).union(dol2)
+    no = []
+    return dict((k, dol1.get(k, no) + dol2.get(k, no)) for k in keys)
+
+def updateObj(obj,lines):
+    """ evaluate lines and modify the members of the argument object, A KeyError
+        exception is raised if the key from lines is not a member of the object.
+        Each line in lines is a simple string like 'key = val'. """
+    keys,vals = [],[]
+    for line in lines:
+        if line.count('=') != 1: raise Exception
+        k,v = line.split('=')
+        # if k.strip() not in obj.__dict__.keys():
+        #     print(k, ' not supported, try: ', obj.__dict__.keys())
+        #     raise KeyError
+        setattr(obj,k.strip(),eval(v))
+    return obj
+
+def private_cleanup_name(s):
+    """ cleans up string so no space, no funny symbols.  it will remove all
+    punctuations chars apart from those specified in rep, which will be '_' """
+    rep = ' .*'
+    rmv = string.punctuation
+    for c in rep: rmv = rmv.replace(c,'_')
+    for c in rmv: s = s.replace(c,'')
+    for c in rep: s = s.replace(c,'_')
+    return s
+
+def getFromDict(dataDict, mapList):
+    return reduce(operator.getitem, mapList, dataDict)
+
+def setInDict(dataDict, mapList, value):
+    getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value
+

gopest/data/case.pst

@@ -0,0 +1,67 @@
+pcf
+* control data
+restart  estimation
+4 16 8 0 7
+     1     1 single  point  1   0   0
+10.0  -20.0  0.3  0.03 -100 0 run_slow_fac=20.0 lamforgive derforgive
+ 10.0  10.0  0.001
+ 0.1 5 noaui
+ 0  0.005  4  4  0.005  4
+ 1  1  1 jcosaveitn parsaveitn rrfsave
+* singular value decomposition
+1
+3249 5.0e-7
+0
+* parameter groups
+ k1       relative    0.01  0.0  switch  2.0 parabolic
+ k2       relative    0.01  0.0  switch  2.0 parabolic
+ k3       relative    0.01  0.0  switch  2.0 parabolic
+ upflow   relative    0.01  0.0  switch  2.0 parabolic
+ por      relative    0.03  0.0  switch  2.0 parabolic
+ rech     relative    0.05  0.0  switch  2.0 parabolic
+ heat     relative    0.01  0.0  switch  2.0 parabolic
+ rpcap    relative    0.05  0.0  switch  2.0 parabolic
+* parameter data
+R1upper log factor 5e-15 1e-17 4.5e-12 k1 1.0 0.0 1
+R2upper log factor 5e-15 1e-17 4.5e-12 k2 1.0 0.0 1
+R3upper log factor 5e-15 1e-17 3e-12 k3 1.0 0.0 1
+RPupper log factor 0.15 0.01 0.5 por 1.0 0.0 1
+* observation groups
+ temp
+ enth
+ press
+ time
+ enth_g
+ vpress
+ boilp
+* observation data
+ tt_GGL_1_0001         1.4320198019802e+02  3.50000e-02 temp
+ tt_GGL_1_0002         1.6236329113924e+02  3.50000e-02 temp
+ tt_GGL_1_0003         1.6844148148148e+02  3.50000e-02 temp
+ tt_GGL_1_0004         1.9702444444444e+02  3.50000e-02 temp
+ tt_GGL_1_0005         2.0340636942675e+02  3.50000e-02 temp
+ tt_GGL_1_0006         1.8860000000000e+02  3.50000e-02 temp
+ tt_GGL_1_0007         1.8903172147002e+02  3.50000e-02 temp
+ tt_GGL_1_0008         1.9153636363636e+02  3.50000e-02 temp
+ tt_GGL_1_0009         1.9463612565445e+02  3.50000e-02 temp
+ tt_GGL_1_0010         1.7681111111111e+02  3.50000e-02 temp
+ tt_GGL_1_0011         1.8117493036212e+02  3.50000e-02 temp
+ tt_GGL_1_0012         1.9303333333333e+02  3.50000e-02 temp
+ tt_GGL_1_0013         2.0483466042155e+02  3.50000e-02 temp
+ tt_GGL_1_0014         2.1203960396040e+02  3.50000e-02 temp
+ tt_GGL_1_0015         2.1337673267327e+02  3.50000e-02 temp
+ tt_GGL_1_0016         2.0401980198020e+02  3.50000e-02 temp
+* model command line
+gopest run-pest-model
+* model input/output
+pest_model.tpl pest_model.dat
+pest_model.ins  pest_model.obf
+* prior information
+* distribution files
+2 real_model_ns.save real_model_incon.incon real_model_incon.incon real_model_incon.incon
+1 real_model_ns.save real_model_incon.incon.999
+1 real_model_ns.dat real_model_ns.dat.999
+1 real_model_pr.h5 real_model_pr.h5.999
+1 pest_model.dat pest_model.dat.999
+command = "gopest save-iter-model"
+# end

gopest/data/goPESTconfig.aut2.toml

@@ -0,0 +1,95 @@
+# This is goPEST's main configuration file.
+
+[pest]
+mode = 'local'
+dir = ""
+executable = "pest_hp"
+executable_agent = "agent_hp"
+port = "24001"
+num_slaves = 2
+slave_dirs = ""
+
+case-name = 'case'
+switches = []
+# additional swiches for beopest, eg /s for restart, /p1 for parallise 1st model run
+# should not use /p1 with svda, no /p1 with obsreref_10 either
+# /hpstart, make sure PST_NAME.hp exists
+# /i jco reuse, make sure PST_NAME.jco exists
+# NOTE it's possible to use /hpstart and /i together, will start update tests right away
+# NOTE working directory is assumed to be where this script is launched
+
+[model]
+skip = true # skips actual simulation, existing model output is used
+skip-pr = true
+silent = true
+sequence = ['ns', 'pr']
+
+[model.original]
+# these original model files will be renamed to goPEST's internal convention,
+# which is usually something like real_model_xx, where xx is the sequence name.
+
+# specify at least Mulgrid geometry file here, needs .msh etc if running Waiwera
+geometry-files = ['raw/gair_smodel.dat', 'raw/gair_smodel.msh']
+incon-file = 'raw/air_smodel_ns.incon'
+ns-input-file = 'raw/air_smodel_ns.dat'
+pr-input-file = 'raw/air_smodel_pr.dat'
+
+# optional original output file, useful for debug
+ns-output-file = 'raw/air_smodel_ns.h5'
+pr-output-file = 'raw/air_smodel_pr.h5'
+
+[simulator]
+input-type = "aut2" # "waiwera" or "aut2"
+output-type = "h5" # "listing" or "h5", will always be "h5" if input-type = "waiwera"
+
+executable = 'AUTOUGH2_7_1-7.exe'
+
+# these will be attached to the end of simulation command (useful for waiwera)
+cmd-options = []
+
+[nesi]
+project = "uoa00123"
+cluster_master = "mahuika"
+cluster_forward = "maui"
+walltime_master = "72:00:00" # hour:min:sec
+walltime_forward = "12:00:00" # hour:min:sec
+ntasks = 40
+
+[nesi.maui]
+project = "uoa00123"
+walltime_forward = "12:00:00" # hour:min:sec
+ntasks = 40
+env_init = [
+    "module swap PrgEnv-cray PrgEnv-gnu",
+    "module swap gcc gcc/8.3.0",
+    "module load cray-python",
+    "export HDF5_USE_FILE_LOCKING=FALSE", # avoids h5py unable to lock file (Maui only)
+    "export PYTHONPATH=/nesi/project/uoa00124/pytough:/nesi/project/uoa00124/utils:/nesi/project/uoa00124/software/py27-maui/:$PYTHONPATH",
+    "printenv PYTHONPATH",
+]
+
+[nesi.mahuika]
+project = "uoa00123"
+walltime_forward = "24:00:00" # hour:min:sec
+ntasks = 40
+env_init = [
+    "module load gimkl/2018b",
+]
+
+[files]
+# all files required by a PEST slave and forward model run:
+slave = [
+    "g_real_model.dat",
+    "g_real_model.msh",
+    "real_model_original.json",
+    "real_model.incon.h5",
+    # observation data files
+    "data.json",
+    # these are for forward run
+    "gs_production.json",
+    "real_model_pr.output.json",
+    "real_model_pr.time.json",
+]
+# additional files required by PEST master
+master = []
+

gopest/data/goPESTconfig.toml

@@ -0,0 +1,94 @@
+# This is goPEST's main configuration file.
+
+[pest]
+dir = ""
+executable = "pest_hp"
+executable_agent = "agent_hp"
+port = "24001"
+num_slaves = 1
+slave_dirs = ""
+
+case-name = 'case'
+switches = []
+# additional swiches for beopest, eg /s for restart, /p1 for parallise 1st model run
+# should not use /p1 with svda, no /p1 with obsreref_10 either
+# /hpstart, make sure PST_NAME.hp exists
+# /i jco reuse, make sure PST_NAME.jco exists
+# NOTE it's possible to use /hpstart and /i together, will start update tests right away
+# NOTE working directory is assumed to be where this script is launched
+
+[model]
+skip = true # skips actual simulation, existing model output is used
+skip-pr = true
+silent = true
+sequence = ['ns', 'pr']
+
+[model.original]
+# these original model files will be renamed to goPEST's internal convention,
+# which is usually something like real_model_xx, where xx is the sequence name.
+
+# specify at least Mulgrid geometry file here, needs .msh etc if running Waiwera
+geometry-files = ['raw/gair_smodel.dat', 'raw/gair_smodel.msh']
+incon-file = 'raw/air_smodel_ns.incon'
+ns-input-file = 'raw/air_smodel_ns.dat'
+pr-input-file = 'raw/air_smodel_pr.dat'
+
+# optional original output file, useful for debug
+ns-output-file = 'raw/air_smodel_ns.h5'
+pr-output-file = 'raw/air_smodel_pr.h5'
+
+[simulator]
+input-type = "aut2" # "waiwera" or "aut2"
+output-type = "h5" # "listing" or "h5", will always be "h5" if input-type = "waiwera"
+
+executable = 'AUTOUGH2_7_1-7.exe'
+
+# these will be attached to the end of simulation command (useful for waiwera)
+cmd-options = []
+
+[nesi]
+project = "uoa00123"
+cluster_master = "mahuika"
+cluster_forward = "maui"
+walltime_master = "72:00:00" # hour:min:sec
+walltime_forward = "12:00:00" # hour:min:sec
+ntasks = 40
+
+[nesi.maui]
+project = "uoa00123"
+walltime_forward = "12:00:00" # hour:min:sec
+ntasks = 40
+env_init = [
+    "module swap PrgEnv-cray PrgEnv-gnu",
+    "module swap gcc gcc/8.3.0",
+    "module load cray-python",
+    "export HDF5_USE_FILE_LOCKING=FALSE", # avoids h5py unable to lock file (Maui only)
+    "export PYTHONPATH=/nesi/project/uoa00124/pytough:/nesi/project/uoa00124/utils:/nesi/project/uoa00124/software/py27-maui/:$PYTHONPATH",
+    "printenv PYTHONPATH",
+]
+
+[nesi.mahuika]
+project = "uoa00123"
+walltime_forward = "24:00:00" # hour:min:sec
+ntasks = 40
+env_init = [
+    "module load gimkl/2018b",
+]
+
+[files]
+# all files required by a PEST slave and forward model run:
+slave = [
+    "g_real_model.dat",
+    "g_real_model.msh",
+    "real_model_original.json",
+    "real_model.incon.h5",
+    # observation data files
+    "data.json",
+    # these are for forward run
+    "gs_production.json",
+    "real_model_pr.output.json",
+    "real_model_pr.time.json",
+]
+# additional files required by PEST master
+master = []
+