parsac 0.5.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsac-0.5.8/.github/dependabot.yml +10 -0
- parsac-0.5.8/.github/workflows/parsac.yml +22 -0
- parsac-0.5.8/.github/workflows/publish_to_pypi.yml +41 -0
- parsac-0.5.8/.gitignore +9 -0
- parsac-0.5.8/.zenodo.json +18 -0
- parsac-0.5.8/PKG-INFO +46 -0
- parsac-0.5.8/README.md +31 -0
- parsac-0.5.8/admin/pull_optimize +1 -0
- parsac-0.5.8/admin/pull_optimize.bat +3 -0
- parsac-0.5.8/admin/push_optimize +1 -0
- parsac-0.5.8/need_updates/clear_database.py +48 -0
- parsac-0.5.8/need_updates/create_database.py +42 -0
- parsac-0.5.8/need_updates/extractpop.py +38 -0
- parsac-0.5.8/need_updates/mysqlinfo.py +48 -0
- parsac-0.5.8/need_updates/runbadresult.py +47 -0
- parsac-0.5.8/need_updates/runyears.py +125 -0
- parsac-0.5.8/need_updates/showruninfo.py +46 -0
- parsac-0.5.8/parsac/.gitignore +6 -0
- parsac-0.5.8/parsac/__init__.py +48 -0
- parsac-0.5.8/parsac/autocalibration.py +42 -0
- parsac-0.5.8/parsac/config.xsd +106 -0
- parsac-0.5.8/parsac/ensemble.py +146 -0
- parsac-0.5.8/parsac/examples/idealized.xml +13 -0
- parsac-0.5.8/parsac/examples/kinneret_wind.xml +16 -0
- parsac-0.5.8/parsac/job/__init__.py +37 -0
- parsac-0.5.8/parsac/job/function.py +25 -0
- parsac-0.5.8/parsac/job/gotm.py +90 -0
- parsac-0.5.8/parsac/job/idealized.py +35 -0
- parsac-0.5.8/parsac/job/namelist.py +188 -0
- parsac-0.5.8/parsac/job/program.py +861 -0
- parsac-0.5.8/parsac/job/shared.py +484 -0
- parsac-0.5.8/parsac/optimize/.gitignore +1 -0
- parsac-0.5.8/parsac/optimize/__init__.py +1 -0
- parsac-0.5.8/parsac/optimize/bfgs.py +528 -0
- parsac-0.5.8/parsac/optimize/core.py +295 -0
- parsac-0.5.8/parsac/optimize/desolver.py +327 -0
- parsac-0.5.8/parsac/optimize/likelihood.py +11 -0
- parsac-0.5.8/parsac/optimize/test_de.py +38 -0
- parsac-0.5.8/parsac/parsac_run.py +17 -0
- parsac-0.5.8/parsac/report.py +230 -0
- parsac-0.5.8/parsac/result/__init__.py +143 -0
- parsac-0.5.8/parsac/result/animate_2d.py +134 -0
- parsac-0.5.8/parsac/result/plot.py +234 -0
- parsac-0.5.8/parsac/result/plotbest.py +238 -0
- parsac-0.5.8/parsac/result/summary.py +40 -0
- parsac-0.5.8/parsac/run.py +98 -0
- parsac-0.5.8/parsac/sensitivity.py +395 -0
- parsac-0.5.8/parsac/service.py +63 -0
- parsac-0.5.8/parsac/service.txt +13 -0
- parsac-0.5.8/parsac/setup.py +94 -0
- parsac-0.5.8/parsac/transport/__init__.py +237 -0
- parsac-0.5.8/parsac.egg-info/PKG-INFO +46 -0
- parsac-0.5.8/parsac.egg-info/SOURCES.txt +62 -0
- parsac-0.5.8/parsac.egg-info/dependency_links.txt +1 -0
- parsac-0.5.8/parsac.egg-info/entry_points.txt +2 -0
- parsac-0.5.8/parsac.egg-info/not-zip-safe +1 -0
- parsac-0.5.8/parsac.egg-info/requires.txt +1 -0
- parsac-0.5.8/parsac.egg-info/top_level.txt +1 -0
- parsac-0.5.8/setup.cfg +4 -0
- parsac-0.5.8/setup.py +32 -0
- parsac-0.5.8/webserver/index.wn +2 -0
- parsac-0.5.8/webserver/settings.php +8 -0
- parsac-0.5.8/webserver/startrun.php +65 -0
- parsac-0.5.8/webserver/submit.php +81 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
name: Install and test
|
|
2
|
+
on: push
|
|
3
|
+
jobs:
|
|
4
|
+
test:
|
|
5
|
+
strategy:
|
|
6
|
+
matrix:
|
|
7
|
+
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
|
|
8
|
+
fail-fast: false
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- name: Clone repository
|
|
12
|
+
uses: actions/checkout@v4
|
|
13
|
+
- name: Setup python
|
|
14
|
+
uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: ${{ matrix.python-version }}
|
|
17
|
+
- name: Install parsac
|
|
18
|
+
run: pip install .
|
|
19
|
+
- name: Test calibration
|
|
20
|
+
run: |
|
|
21
|
+
cd parsac/examples
|
|
22
|
+
parsac calibration run idealized.xml
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Build and publish
|
|
2
|
+
|
|
3
|
+
on: push
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
name: Build distribution
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
steps:
|
|
10
|
+
- name: Clone repository
|
|
11
|
+
uses: actions/checkout@v4
|
|
12
|
+
with:
|
|
13
|
+
fetch-depth: 0 # for version detection by hatch-vcs/setuptools_scm
|
|
14
|
+
- name: Set up Python
|
|
15
|
+
uses: actions/setup-python@v5
|
|
16
|
+
- name: Install build
|
|
17
|
+
run: pip install build
|
|
18
|
+
- name: Build a binary wheel and a source tarball
|
|
19
|
+
run: python -m build
|
|
20
|
+
- name: Store the distribution packages
|
|
21
|
+
uses: actions/upload-artifact@v4
|
|
22
|
+
with:
|
|
23
|
+
name: python-package-distributions
|
|
24
|
+
path: dist/
|
|
25
|
+
publish-to-pypi:
|
|
26
|
+
name: Publish to PyPI
|
|
27
|
+
if: startsWith(github.ref, 'refs/tags/')
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
environment:
|
|
30
|
+
name: pypi
|
|
31
|
+
url: https://pypi.org/p/${{ github.event.repository.name }}
|
|
32
|
+
permissions:
|
|
33
|
+
id-token: write
|
|
34
|
+
steps:
|
|
35
|
+
- name: Download all the dists
|
|
36
|
+
uses: actions/download-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: python-package-distributions
|
|
39
|
+
path: dist/
|
|
40
|
+
- name: Publish package to PyPI
|
|
41
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
parsac-0.5.8/.gitignore
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
{
|
|
2
|
+
"title": "parsac: parallel sensitivity analysis and calibration",
|
|
3
|
+
"creators": [
|
|
4
|
+
{
|
|
5
|
+
"name": "Jorn Bruggeman",
|
|
6
|
+
"affiliation": "Bolding & Bruggeman ApS",
|
|
7
|
+
"orcid": "0000-0003-2493-2323"
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
"name": "Karsten Bolding",
|
|
11
|
+
"affiliation": "Bolding & Bruggeman ApS",
|
|
12
|
+
"orcid": "0000-0001-8465-3196"
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"upload_type": "software",
|
|
16
|
+
"access_right": "open",
|
|
17
|
+
"license": "GPL-2.0"
|
|
18
|
+
}
|
parsac-0.5.8/PKG-INFO
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: parsac
|
|
3
|
+
Version: 0.5.8
|
|
4
|
+
Summary: Parallel Sensitivity Analysis and Calibration
|
|
5
|
+
Home-page: https://github.com/BoldingBruggeman/parsac
|
|
6
|
+
Author: Jorn Bruggeman
|
|
7
|
+
Author-email: jorn@bolding-bruggeman.com
|
|
8
|
+
License: GPL
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Topic :: Scientific/Engineering
|
|
12
|
+
Classifier: License :: OSI Approved :: GNU General Public License (GPL)
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Requires-Dist: netCDF4
|
|
15
|
+
|
|
16
|
+
# parsac
|
|
17
|
+
|
|
18
|
+
parsac (formerly acpy) is a Python-based tool for sensitivity analysis and auto-calibration in parallel.
|
|
19
|
+
It is designed for analysis of models that take significant time to run.
|
|
20
|
+
For that reason, it focuses on storing and exploiting every single model result,
|
|
21
|
+
and performing model runs in parallel on either a single machine or
|
|
22
|
+
on computer clusters. It works with models that are run by calling one binary,
|
|
23
|
+
that use text-based configuration files based on YAML or Fortran namelists,
|
|
24
|
+
and that write their output to NetCDF.
|
|
25
|
+
|
|
26
|
+
[](https://zenodo.org/badge/latestdoi/206791023) [](https://travis-ci.com/BoldingBruggeman/parsac)
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
`pip install parsac --user`
|
|
31
|
+
|
|
32
|
+
Remove `--user` to install in the system's shared Python directory (not recommended).
|
|
33
|
+
Some systems have multiple versions of pip, e.g., pip for Python 2, pip3 for Python 3.
|
|
34
|
+
Make sure you use the command that corresponds to the Python version you want to install into.
|
|
35
|
+
|
|
36
|
+
### Dependencies
|
|
37
|
+
|
|
38
|
+
parsac supports parallel simulations through [Parallel Python](https://www.parallelpython.com).
|
|
39
|
+
This package supports Python 2 out of the box (`pip install pp --user`), but its Python 3 version
|
|
40
|
+
is currently in beta. To install pp in Python 3, [download the zip file with the Python 3 port of Parallel Python](https://www.parallelpython.com/content/view/18/32), extract its contents, go to the contained directory and open a command prompt there, then run `python setup.py install`.
|
|
41
|
+
|
|
42
|
+
parsac uses [SALib](https://github.com/SALib/SALib) for sensitivity analysis. Typically, this can be installed with `pip install SALib --user`. If you are using [the Anaconda Python distribution](https://www.anaconda.com), you can instead do `conda install SALib` (you may need to add `-c conda-forge`).
|
|
43
|
+
|
|
44
|
+
## Known issues
|
|
45
|
+
|
|
46
|
+
* On Windows, parallel runs may finish with several "ERROR: The process "xxx" not found." messages. These are harmless and can be ignored - the analysis has completed successfully and all results have been correctly processed.
|
parsac-0.5.8/README.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# parsac
|
|
2
|
+
|
|
3
|
+
parsac (formerly acpy) is a Python-based tool for sensitivity analysis and auto-calibration in parallel.
|
|
4
|
+
It is designed for analysis of models that take significant time to run.
|
|
5
|
+
For that reason, it focuses on storing and exploiting every single model result,
|
|
6
|
+
and performing model runs in parallel on either a single machine or
|
|
7
|
+
on computer clusters. It works with models that are run by calling one binary,
|
|
8
|
+
that use text-based configuration files based on YAML or Fortran namelists,
|
|
9
|
+
and that write their output to NetCDF.
|
|
10
|
+
|
|
11
|
+
[](https://zenodo.org/badge/latestdoi/206791023) [](https://travis-ci.com/BoldingBruggeman/parsac)
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
`pip install parsac --user`
|
|
16
|
+
|
|
17
|
+
Remove `--user` to install in the system's shared Python directory (not recommended).
|
|
18
|
+
Some systems have multiple versions of pip, e.g., pip for Python 2, pip3 for Python 3.
|
|
19
|
+
Make sure you use the command that corresponds to the Python version you want to install into.
|
|
20
|
+
|
|
21
|
+
### Dependencies
|
|
22
|
+
|
|
23
|
+
parsac supports parallel simulations through [Parallel Python](https://www.parallelpython.com).
|
|
24
|
+
This package supports Python 2 out of the box (`pip install pp --user`), but its Python 3 version
|
|
25
|
+
is currently in beta. To install pp in Python 3, [download the zip file with the Python 3 port of Parallel Python](https://www.parallelpython.com/content/view/18/32), extract its contents, go to the contained directory and open a command prompt there, then run `python setup.py install`.
|
|
26
|
+
|
|
27
|
+
parsac uses [SALib](https://github.com/SALib/SALib) for sensitivity analysis. Typically, this can be installed with `pip install SALib --user`. If you are using [the Anaconda Python distribution](https://www.anaconda.com), you can instead do `conda install SALib` (you may need to add `-c conda-forge`).
|
|
28
|
+
|
|
29
|
+
## Known issues
|
|
30
|
+
|
|
31
|
+
* On Windows, parallel runs may finish with several "ERROR: The process "xxx" not found." messages. These are harmless and can be ignored - the analysis has completed successfully and all results have been correctly processed.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
git subtree pull --prefix parsac/optimize git@bitbucket.org:jbruggeman/optimize.git master --squash
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
git subtree push --prefix parsac/optimize git@bitbucket.org:jbruggeman/optimize.git master --squash
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
import optparse,sys
|
|
4
|
+
|
|
5
|
+
import mysqlinfo
|
|
6
|
+
|
|
7
|
+
parser = optparse.OptionParser()
|
|
8
|
+
parser.add_option('-r', '--run', type='int', help='run identifier')
|
|
9
|
+
parser.add_option('-j', '--job', type='int', help='job identifier')
|
|
10
|
+
parser.add_option('--emptyrows', action='store_true', help='delete empty rows only')
|
|
11
|
+
parser.set_defaults(emptyrows=False)
|
|
12
|
+
(options, args) = parser.parse_args()
|
|
13
|
+
|
|
14
|
+
db = mysqlinfo.connect(mysqlinfo.admin)
|
|
15
|
+
|
|
16
|
+
c = db.cursor()
|
|
17
|
+
|
|
18
|
+
addedwhere = ''
|
|
19
|
+
selection = 'all'
|
|
20
|
+
if options.emptyrows:
|
|
21
|
+
addedwhere = ' AND (`parameters`=\'\' OR `parameters`=NULL)'
|
|
22
|
+
selection = 'empty'
|
|
23
|
+
|
|
24
|
+
def delRun(run):
|
|
25
|
+
print 'Removing %s records for run %i from the database...' % (selection,run)
|
|
26
|
+
c.execute('DELETE FROM `results` WHERE (`run`=%i%s);' % (run,addedwhere))
|
|
27
|
+
print '%i records removed from "results" table.' % db.affected_rows()
|
|
28
|
+
if not addedwhere:
|
|
29
|
+
c.execute('DELETE FROM `runs` WHERE `id`=%i;' % (run,))
|
|
30
|
+
print '%i records removed from "runs" table.' % db.affected_rows()
|
|
31
|
+
|
|
32
|
+
if options.run is not None:
|
|
33
|
+
delRun(options.run)
|
|
34
|
+
elif options.job is not None:
|
|
35
|
+
query = "SELECT `id` FROM `runs` WHERE `job`='%i'" % options.job
|
|
36
|
+
c.execute(query)
|
|
37
|
+
for run, in c:
|
|
38
|
+
delRun(run)
|
|
39
|
+
else:
|
|
40
|
+
print 'Currently the clearing of the entire database is disabled for safety.'
|
|
41
|
+
sys.exit(1)
|
|
42
|
+
c.execute('DELETE FROM `results`;')
|
|
43
|
+
print '%i records removed from "results" table.' % db.affected_rows()
|
|
44
|
+
c.execute('DELETE FROM `runs`;')
|
|
45
|
+
print '%i records removed from "runs" table.' % db.affected_rows()
|
|
46
|
+
|
|
47
|
+
db.commit()
|
|
48
|
+
db.close()
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
import mysqlinfo
|
|
5
|
+
|
|
6
|
+
# Get user name and password to be used to connect to MySQL server.
|
|
7
|
+
db = mysqlinfo.connect(mysqlinfo.admin)
|
|
8
|
+
c = db.cursor()
|
|
9
|
+
|
|
10
|
+
print 'Connected to MySQL server %s.' % mysqlinfo.host
|
|
11
|
+
|
|
12
|
+
# Get list of existing databases.
|
|
13
|
+
c.execute('SHOW DATABASES;')
|
|
14
|
+
dbs = [d[0] for d in c.fetchall()]
|
|
15
|
+
|
|
16
|
+
if mysqlinfo.database in dbs:
|
|
17
|
+
resp = ''
|
|
18
|
+
while resp not in ('y','n'):
|
|
19
|
+
resp = raw_input('Database "%s" already exists. Do you want to drop it and create a new one? (y/n): ' % mysqlinfo.database)
|
|
20
|
+
if resp=='n':
|
|
21
|
+
print 'Database creation cancelled.'
|
|
22
|
+
sys.exit(0)
|
|
23
|
+
c.execute('DROP DATABASE `%s`;' % mysqlinfo.database)
|
|
24
|
+
|
|
25
|
+
#c.execute('ALTER TABLE `runs` ADD COLUMN `job` INT AFTER `time`;')
|
|
26
|
+
#c.execute('ALTER TABLE `results` MODIFY `parameters` VARCHAR(500);')
|
|
27
|
+
|
|
28
|
+
# Delete and recreate database
|
|
29
|
+
c.execute('CREATE DATABASE `%s`;' % mysqlinfo.database)
|
|
30
|
+
|
|
31
|
+
# Create users and grant minimum permissions
|
|
32
|
+
c.execute('CREATE USER \'%s\'@\'%%\' IDENTIFIED BY \'%s\';' % (mysqlinfo.runuser,mysqlinfo.runpassword))
|
|
33
|
+
c.execute('CREATE USER \'%s\'@\'localhost\' IDENTIFIED BY \'%s\';' % (mysqlinfo.viewuser,mysqlinfo.viewpassword))
|
|
34
|
+
c.execute('GRANT SELECT ON `%s`.* TO \'%s\'@\'localhost\';' % (mysqlinfo.database,mysqlinfo.viewuser))
|
|
35
|
+
c.execute('GRANT INSERT ON `%s`.* TO \'%s\'@\'%%\';' % (mysqlinfo.database,mysqlinfo.runuser))
|
|
36
|
+
|
|
37
|
+
# Create tables
|
|
38
|
+
c.execute('USE `%s`;' % mysqlinfo.database)
|
|
39
|
+
c.execute('CREATE TABLE `runs` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE PRIMARY KEY,`source` VARCHAR(50) NOT NULL,`time` DATETIME NOT NULL,`job` INT,`description` TEXT);')
|
|
40
|
+
c.execute('CREATE TABLE `results` (`id` INT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE PRIMARY KEY,`run` INT UNSIGNED NOT NULL,`time` DATETIME NOT NULL,`parameters` VARCHAR(500) NOT NULL,`lnlikelihood` DOUBLE);')
|
|
41
|
+
|
|
42
|
+
db.close()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
# Import from standard Python library
|
|
4
|
+
import sys,optparse
|
|
5
|
+
|
|
6
|
+
# Import third-party modules
|
|
7
|
+
import numpy
|
|
8
|
+
|
|
9
|
+
# Import custom modules
|
|
10
|
+
import mysqlinfo
|
|
11
|
+
|
|
12
|
+
parser = optparse.OptionParser()
|
|
13
|
+
(options, args) = parser.parse_args()
|
|
14
|
+
|
|
15
|
+
assert len(args)>1, 'First argument must be the run identifier, second argument the file to save to.'
|
|
16
|
+
|
|
17
|
+
db = mysqlinfo.connect(mysqlinfo.select)
|
|
18
|
+
|
|
19
|
+
# Retrieve all results
|
|
20
|
+
c = db.cursor()
|
|
21
|
+
query = "SELECT `id`,`parameters` FROM `results` WHERE `run`=%i" % int(args[0])
|
|
22
|
+
c.execute(query)
|
|
23
|
+
history = []
|
|
24
|
+
for resid,strpars in c:
|
|
25
|
+
parameters = map(float,strpars.split(';'))
|
|
26
|
+
history.append(parameters)
|
|
27
|
+
db.close()
|
|
28
|
+
print 'Found %i results.' % (len(history),)
|
|
29
|
+
|
|
30
|
+
# Stop if no results were found
|
|
31
|
+
if len(history)==0: sys.exit(0)
|
|
32
|
+
|
|
33
|
+
# Convert results into numpy array
|
|
34
|
+
res = numpy.zeros((len(history),len(history[0])))
|
|
35
|
+
for i,v in enumerate(history):
|
|
36
|
+
res[i,:] = v
|
|
37
|
+
|
|
38
|
+
res.dump(args[1])
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
host = 'localhost'
|
|
2
|
+
database = 'optimize'
|
|
3
|
+
runuser = 'run'
|
|
4
|
+
runpassword = 'g0tm'
|
|
5
|
+
viewuser = 'jorn'
|
|
6
|
+
viewpassword = '1r3z2g6$'
|
|
7
|
+
adminuser = None
|
|
8
|
+
adminpassword = None
|
|
9
|
+
defaultfile = None
|
|
10
|
+
|
|
11
|
+
admin = 0
|
|
12
|
+
select = 1
|
|
13
|
+
insert = 2
|
|
14
|
+
|
|
15
|
+
import getpass
|
|
16
|
+
import MySQLdb
|
|
17
|
+
|
|
18
|
+
def connect(task=None):
|
|
19
|
+
# Set default user naem/password based on chosen task.
|
|
20
|
+
username,password = None,None
|
|
21
|
+
if task==admin:
|
|
22
|
+
username,password = adminuser,adminpassword
|
|
23
|
+
elif task==select:
|
|
24
|
+
username,password = viewuser,viewpassword
|
|
25
|
+
elif task==insert:
|
|
26
|
+
username,password = runuser,viewpassword
|
|
27
|
+
|
|
28
|
+
# If we do not have a user name yet, and we do not have a default file either, ask the user interactively.
|
|
29
|
+
if username is None and defaultfile is None:
|
|
30
|
+
username = 'root'
|
|
31
|
+
print 'Connecting to MySQL server %s.' % host
|
|
32
|
+
username = raw_input('User name [root]: ')
|
|
33
|
+
if not username: username = 'root'
|
|
34
|
+
|
|
35
|
+
# If we do not have a passsword yet, and we do not have a default file either, ask the user interactively.
|
|
36
|
+
if password is None and defaultfile is None:
|
|
37
|
+
password = getpass.getpass('Password for user %s: ' % username)
|
|
38
|
+
|
|
39
|
+
# Connect to database
|
|
40
|
+
kwargs = {}
|
|
41
|
+
if host is not None: kwargs['host'] = host
|
|
42
|
+
if database is not None: kwargs['db'] = database
|
|
43
|
+
if username is not None: kwargs['user'] = username
|
|
44
|
+
if password is not None: kwargs['passwd'] = password
|
|
45
|
+
if defaultfile is not None: kwargs['read_default_file'] = defaultfile
|
|
46
|
+
db = MySQLdb.connect(**kwargs)
|
|
47
|
+
|
|
48
|
+
return db
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
import sys,math,optparse
|
|
4
|
+
|
|
5
|
+
import acpy.run
|
|
6
|
+
import acpy.gotmcontroller
|
|
7
|
+
import mysqlinfo
|
|
8
|
+
|
|
9
|
+
parser = optparse.OptionParser()
|
|
10
|
+
parser.add_option('-i', '--index', type='int', help='index of bad result (>=0)')
|
|
11
|
+
parser.set_defaults(index=0)
|
|
12
|
+
(options, args) = parser.parse_args()
|
|
13
|
+
|
|
14
|
+
if len(args)<1:
|
|
15
|
+
print 'No job identifier provided.'
|
|
16
|
+
sys.exit(2)
|
|
17
|
+
jobid = int(args[0])
|
|
18
|
+
|
|
19
|
+
print 'Looking for bad result %i of job %i.' % (options.index,jobid)
|
|
20
|
+
|
|
21
|
+
# Connect to database and retrieve best parameter set.
|
|
22
|
+
db = mysqlinfo.connect(mysqlinfo.select)
|
|
23
|
+
c = db.cursor()
|
|
24
|
+
c.execute("SELECT `parameters`,`lnlikelihood` FROM `runs`,`results` WHERE (`runs`.`id`=`results`.`run` AND `runs`.`job`=%i AND `lnlikelihood` IS NULL) LIMIT %i,1" % (jobid,options.index))
|
|
25
|
+
parameters = None
|
|
26
|
+
for strpars,lnlikelihood in c:
|
|
27
|
+
parameters = map(float,strpars.split(';'))
|
|
28
|
+
db.close()
|
|
29
|
+
if parameters==None:
|
|
30
|
+
print 'No bad results found. Exiting...'
|
|
31
|
+
sys.exit(0)
|
|
32
|
+
|
|
33
|
+
job = acpy.run.getJob(jobid)
|
|
34
|
+
|
|
35
|
+
# Show best parameter set
|
|
36
|
+
print 'Testing bad parameter set number %i.' % options.index
|
|
37
|
+
print 'Problem parameter set:'
|
|
38
|
+
for i,val in enumerate(parameters):
|
|
39
|
+
pi = job.controller.parameters[i]
|
|
40
|
+
if pi['logscale']: val = math.pow(10.,val)
|
|
41
|
+
print ' %s = %.6g' % (pi['name'],val)
|
|
42
|
+
|
|
43
|
+
# Initialize the GOTM controller.
|
|
44
|
+
job.controller.initialize()
|
|
45
|
+
|
|
46
|
+
# Run and retrieve results.
|
|
47
|
+
nc = job.controller.run(parameters,showoutput=True)
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
import re,os.path
|
|
4
|
+
from Scientific.IO.NetCDF import NetCDFFile
|
|
5
|
+
import matplotlib.pylab,numpy
|
|
6
|
+
import acpy.gotmcontroller,acpy.run,acpy.optimizer
|
|
7
|
+
|
|
8
|
+
jobid = 25
|
|
9
|
+
scenpath = os.path.join('acpy','./scenarios/%i' % jobid)
|
|
10
|
+
obsdir = os.path.join('acpy',acpy.run.obsdir)
|
|
11
|
+
|
|
12
|
+
nmlfile = os.path.join(scenpath,'gotmrun.nml')
|
|
13
|
+
|
|
14
|
+
for year in range(2000,2006):
|
|
15
|
+
print 'Running year %04i' % year
|
|
16
|
+
nmls,nmlorder = acpy.gotmcontroller.parseNamelistFile(nmlfile)
|
|
17
|
+
nmls['time']['start'] = '"%04i-01-01 00:00:00"' % year
|
|
18
|
+
nmls['time']['stop'] = '"%04i-01-01 00:00:00"' % (year+1)
|
|
19
|
+
acpy.gotmcontroller.writeNamelistFile(nmlfile,nmls,nmlorder)
|
|
20
|
+
|
|
21
|
+
job = acpy.optimizer.Job(jobid,
|
|
22
|
+
scenpath,
|
|
23
|
+
gotmexe=acpy.run.exe,
|
|
24
|
+
transports=(acpy.transport.Dummy(),),
|
|
25
|
+
copyexe=True)
|
|
26
|
+
|
|
27
|
+
job.verbose = False
|
|
28
|
+
job.addObservation(os.path.join(obsdir,'pottemp.obs'),'temp',maxdepth=310,cache=False)
|
|
29
|
+
job.addObservation(os.path.join(obsdir,'salt.obs'),'salt',maxdepth=310,cache=False)
|
|
30
|
+
job.verbose = True
|
|
31
|
+
|
|
32
|
+
job.initialize()
|
|
33
|
+
|
|
34
|
+
obsinfo = job.observations
|
|
35
|
+
outputvars = [oi['outputvariable'] for oi in obsinfo]
|
|
36
|
+
|
|
37
|
+
# Run and retrieve results.
|
|
38
|
+
ncpath = job.controller.run((),showoutput=False,returnncpath=True)
|
|
39
|
+
if ncpath==None:
|
|
40
|
+
print 'GOTM run failed - exiting.'
|
|
41
|
+
sys.exit(1)
|
|
42
|
+
nc = NetCDFFile(ncpath,'r')
|
|
43
|
+
res = acpy.run.job.controller.getNetCDFVariables(nc,outputvars,addcoordinates=True)
|
|
44
|
+
nc.close()
|
|
45
|
+
|
|
46
|
+
# Shortcuts to coordinates
|
|
47
|
+
tim_cent,z_cent,z1_cent = res['time_center'],res['z_center'],res['z1_center']
|
|
48
|
+
tim_stag,z_stag,z1_stag = res['time_staggered'],res['z_staggered'],res['z1_staggered']
|
|
49
|
+
|
|
50
|
+
# Find the depth index from where we start
|
|
51
|
+
ifirstz = z_cent.searchsorted(-300)
|
|
52
|
+
viewdepth = 300
|
|
53
|
+
|
|
54
|
+
hres = matplotlib.pylab.figure()
|
|
55
|
+
herr = matplotlib.pylab.figure()
|
|
56
|
+
for i,oi in enumerate(obsinfo):
|
|
57
|
+
modeldata = res[oi['outputvariable']]
|
|
58
|
+
obsdata = oi['observeddata']
|
|
59
|
+
|
|
60
|
+
modelmin = oi.get('modelminimum',None)
|
|
61
|
+
if modelmin!=None: modeldata[modeldata<modelmin] = modelmin
|
|
62
|
+
|
|
63
|
+
# Calculate model predictions on observation coordinates.
|
|
64
|
+
pred = acpy.gotmcontroller.interp2(tim_cent,z_cent,modeldata,obsdata[:,0],obsdata[:,1])
|
|
65
|
+
|
|
66
|
+
# If we do a relative fit, scale the model result to best match observations.
|
|
67
|
+
if oi['relativefit']:
|
|
68
|
+
if (pred==0.).all():
|
|
69
|
+
print 'ERROR: cannot calculate optimal scaling factor for %s because all model values equal zero.' % oi['outputvariable']
|
|
70
|
+
sys.exit(1)
|
|
71
|
+
scale = sum(obsdata[:,2]*pred)/sum(pred*pred)
|
|
72
|
+
print 'Optimal model-to-observation scaling factor for %s = %.6g.' % (oi['outputvariable'],scale)
|
|
73
|
+
pred *= scale
|
|
74
|
+
modeldata *= scale
|
|
75
|
+
|
|
76
|
+
varrange = (min(modeldata[:,ifirstz:].min(),obsdata[:,2].min()),max(modeldata[:,ifirstz:].max(),obsdata[:,2].max()))
|
|
77
|
+
|
|
78
|
+
diff = pred-obsdata[:,2]
|
|
79
|
+
print '%s: mean absolute error = %.4g, s.d. = %.4g.' % (oi['outputvariable'],numpy.mean(numpy.abs(diff)),numpy.sqrt(numpy.mean(diff**2)))
|
|
80
|
+
|
|
81
|
+
if False:
|
|
82
|
+
# Create figure for model-data comparison
|
|
83
|
+
matplotlib.pylab.figure(hres.number)
|
|
84
|
+
|
|
85
|
+
# Plot model result
|
|
86
|
+
matplotlib.pylab.subplot(len(obsinfo),2,i*2+1)
|
|
87
|
+
matplotlib.pylab.pcolor(tim_stag,z_stag,modeldata.transpose())
|
|
88
|
+
matplotlib.pylab.clim(varrange)
|
|
89
|
+
matplotlib.pylab.ylim(-viewdepth,0)
|
|
90
|
+
matplotlib.pylab.colorbar()
|
|
91
|
+
matplotlib.pylab.xlim(tim_stag[0],tim_stag[-1])
|
|
92
|
+
xax = matplotlib.pylab.gca().xaxis
|
|
93
|
+
loc = matplotlib.dates.AutoDateLocator()
|
|
94
|
+
xax.set_major_formatter(matplotlib.dates.AutoDateFormatter(loc))
|
|
95
|
+
xax.set_major_locator(loc)
|
|
96
|
+
matplotlib.pylab.grid(True)
|
|
97
|
+
|
|
98
|
+
# Plot observations
|
|
99
|
+
matplotlib.pylab.subplot(len(obsinfo),2,i*2+2)
|
|
100
|
+
matplotlib.pylab.scatter(obsdata[:,0],obsdata[:,1],s=10,c=obsdata[:,2],cmap=matplotlib.cm.jet,vmin=varrange[0],vmax=varrange[1],faceted=False)
|
|
101
|
+
matplotlib.pylab.ylim(-viewdepth,0)
|
|
102
|
+
matplotlib.pylab.xlim(tim_stag[0],tim_stag[-1])
|
|
103
|
+
matplotlib.pylab.clim(varrange)
|
|
104
|
+
xax = matplotlib.pylab.gca().xaxis
|
|
105
|
+
loc = matplotlib.dates.AutoDateLocator()
|
|
106
|
+
xax.set_major_formatter(matplotlib.dates.AutoDateFormatter(loc))
|
|
107
|
+
xax.set_major_locator(loc)
|
|
108
|
+
matplotlib.pylab.grid(True)
|
|
109
|
+
|
|
110
|
+
# Plot histogram with errors.
|
|
111
|
+
matplotlib.pylab.figure(herr.number)
|
|
112
|
+
matplotlib.pylab.subplot(len(obsinfo),1,i+1)
|
|
113
|
+
#matplotlib.pylab.plot(diff,obs[:,1],'o')
|
|
114
|
+
#matplotlib.pylab.figure()
|
|
115
|
+
n, bins, patches = matplotlib.pylab.hist(diff, 100, normed=1)
|
|
116
|
+
#y = matplotlib.pylab.normpdf(bins, 0., numpy.sqrt(ssq/len(diff)))
|
|
117
|
+
#l = matplotlib.pylab.plot(bins, y, 'r--', linewidth=2)
|
|
118
|
+
|
|
119
|
+
if False: matplotlib.pylab.show()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
# Import from standard Python library
|
|
4
|
+
import sys,optparse
|
|
5
|
+
|
|
6
|
+
# Import custom Python modules
|
|
7
|
+
import mysqlinfo
|
|
8
|
+
|
|
9
|
+
parser = optparse.OptionParser()
|
|
10
|
+
(options, args) = parser.parse_args()
|
|
11
|
+
|
|
12
|
+
if len(args)==0:
|
|
13
|
+
print 'No run identifier specified.'
|
|
14
|
+
sys.exit(1)
|
|
15
|
+
runid = int(args[0])
|
|
16
|
+
|
|
17
|
+
print 'Showing information for run with identifier %i' % runid
|
|
18
|
+
|
|
19
|
+
# Retrieve run information
|
|
20
|
+
db = mysqlinfo.connect(mysqlinfo.select)
|
|
21
|
+
c = db.cursor()
|
|
22
|
+
c.execute("SELECT `source`,`time`,`description` FROM `runs` WHERE `id`=%i" % runid)
|
|
23
|
+
for (source,time,description) in c:
|
|
24
|
+
print 'Source: %s' % source
|
|
25
|
+
print 'Start time: %s' % time
|
|
26
|
+
|
|
27
|
+
if description==None:
|
|
28
|
+
print 'No description was provided.'
|
|
29
|
+
continue
|
|
30
|
+
import pickle
|
|
31
|
+
info = pickle.loads(description)
|
|
32
|
+
|
|
33
|
+
print 'Parameters:'
|
|
34
|
+
for p in info['parameters']:
|
|
35
|
+
print ' %s/%s/%s, range: %.6g - %.6g, log scale: %s' % (p['namelistfile'],
|
|
36
|
+
p['namelistname'],
|
|
37
|
+
p['name'],
|
|
38
|
+
p['minimum'],
|
|
39
|
+
p['maximum'],
|
|
40
|
+
p['logscale'])
|
|
41
|
+
print 'Observations:'
|
|
42
|
+
for p in info['observations']:
|
|
43
|
+
print ' file: %s, variable: %s, relative fit: %s' % (p['sourcepath'],p['outputvariable'],p['relativefit'])
|
|
44
|
+
|
|
45
|
+
db.close()
|
|
46
|
+
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from . import service
|
|
6
|
+
service.read()
|
|
7
|
+
from . import autocalibration
|
|
8
|
+
from . import sensitivity
|
|
9
|
+
from . import ensemble
|
|
10
|
+
|
|
11
|
+
__version__ = 'version not available'
|
|
12
|
+
|
|
13
|
+
import importlib.metadata
|
|
14
|
+
try:
|
|
15
|
+
__version__ = importlib.metadata.version("parsac")
|
|
16
|
+
except importlib.metadata.PackageNotFoundError:
|
|
17
|
+
try:
|
|
18
|
+
from setuptools_scm import get_version
|
|
19
|
+
__version__ = get_version(root='..', relative_to=__file__)
|
|
20
|
+
except ImportError:
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
def main():
|
|
24
|
+
parser = argparse.ArgumentParser(description='parsac - Parallel Sensitivity Analysis and Calibration')
|
|
25
|
+
# parser.add_argument('--help', action='help')
|
|
26
|
+
parser.add_argument('-v', '--version', action='version', version=__version__)
|
|
27
|
+
subparsers = parser.add_subparsers()
|
|
28
|
+
|
|
29
|
+
parser_sa = subparsers.add_parser('sensitivity', help='Sensitivity analysis')
|
|
30
|
+
sensitivity.configure_argument_parser(parser_sa)
|
|
31
|
+
parser_sa.set_defaults(func=sensitivity.main)
|
|
32
|
+
|
|
33
|
+
parser_ac = subparsers.add_parser('calibration', help='Auto calibration')
|
|
34
|
+
autocalibration.configure_argument_parser(parser_ac)
|
|
35
|
+
parser_ac.set_defaults(func=autocalibration.main)
|
|
36
|
+
|
|
37
|
+
parser_ensemble = subparsers.add_parser('ensemble', help='Ensemble simulation')
|
|
38
|
+
ensemble.configure_argument_parser(parser_ensemble)
|
|
39
|
+
parser_ensemble.set_defaults(func=ensemble.main)
|
|
40
|
+
|
|
41
|
+
parser_service = subparsers.add_parser('service', help='Service information')
|
|
42
|
+
parser_service.set_defaults(func=service.main)
|
|
43
|
+
|
|
44
|
+
args = parser.parse_args()
|
|
45
|
+
if getattr(args, 'func', None) is None:
|
|
46
|
+
print('parsac must be called with a subcommand. Use -h to see options')
|
|
47
|
+
sys.exit(2)
|
|
48
|
+
args.func(args)
|