integrate-module 0.98.0__tar.gz → 0.99.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {integrate_module-0.98.0/integrate_module.egg-info → integrate_module-0.99.1}/PKG-INFO +82 -26
- {integrate_module-0.98.0 → integrate_module-0.99.1}/README.md +65 -21
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/__init__.py +6 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate.py +113 -28
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_borehole.py +99 -36
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_io.py +11 -1
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_plot.py +340 -171
- integrate_module-0.99.1/integrate/integrate_query.py +1609 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_rejection.py +34 -8
- integrate_module-0.99.1/integrate/integrate_www_cli.py +8 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1/integrate_module.egg-info}/PKG-INFO +82 -26
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate_module.egg-info/SOURCES.txt +1 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate_module.egg-info/entry_points.txt +1 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate_module.egg-info/requires.txt +15 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/pyproject.toml +19 -7
- integrate_module-0.98.0/integrate/integrate_query.py +0 -979
- {integrate_module-0.98.0 → integrate_module-0.99.1}/LICENSE +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/gex.py +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_hdf5_info_cli.py +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_rejection_cli.py +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_rejection_jax.py +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate/integrate_timing_cli.py +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate_module.egg-info/dependency_links.txt +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/integrate_module.egg-info/top_level.txt +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/setup.cfg +0 -0
- {integrate_module-0.98.0 → integrate_module-0.99.1}/tests/test_likelihood_multinomial.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: integrate_module
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.99.1
|
|
4
4
|
Summary: Localized probabilistic data integration
|
|
5
5
|
Author-email: Thomas Mejer Hansen <tmeha@geo.au.dk>
|
|
6
6
|
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/
|
|
8
|
-
Project-URL: Documentation, https://
|
|
9
|
-
Project-URL: Repository, https://github.com/
|
|
10
|
-
Project-URL: Issues, https://github.com/
|
|
7
|
+
Project-URL: Homepage, https://github.com/AUProbGeo/integrate_module
|
|
8
|
+
Project-URL: Documentation, https://auprobgeo.github.io/integrate_module/
|
|
9
|
+
Project-URL: Repository, https://github.com/AUProbGeo/integrate_module
|
|
10
|
+
Project-URL: Issues, https://github.com/AUProbGeo/integrate_module/issues
|
|
11
11
|
Keywords: inversion,electromagnetic,geophysics,geology,prior,tarantola
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Requires-Python: >=3.10
|
|
@@ -24,19 +24,31 @@ Requires-Dist: geoprior1d>=0.9
|
|
|
24
24
|
Requires-Dist: ga-aem-forward-win; platform_system == "Windows"
|
|
25
25
|
Requires-Dist: jupyter>=1.0.0
|
|
26
26
|
Requires-Dist: jupytext
|
|
27
|
+
Requires-Dist: ipykernel
|
|
27
28
|
Requires-Dist: pandas
|
|
28
29
|
Requires-Dist: pyvista
|
|
29
30
|
Requires-Dist: litellm
|
|
31
|
+
Requires-Dist: streamlit
|
|
32
|
+
Requires-Dist: libaarhusxyz
|
|
30
33
|
Provides-Extra: dev
|
|
31
34
|
Requires-Dist: pytest; extra == "dev"
|
|
32
35
|
Requires-Dist: black; extra == "dev"
|
|
36
|
+
Provides-Extra: docs
|
|
37
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
38
|
+
Requires-Dist: nbsphinx; extra == "docs"
|
|
39
|
+
Requires-Dist: sphinx-gallery; extra == "docs"
|
|
40
|
+
Requires-Dist: pandoc; extra == "docs"
|
|
41
|
+
Requires-Dist: myst-parser; extra == "docs"
|
|
42
|
+
Requires-Dist: sphinx-rtd-theme; extra == "docs"
|
|
43
|
+
Requires-Dist: furo; extra == "docs"
|
|
44
|
+
Requires-Dist: tomli; python_version < "3.11" and extra == "docs"
|
|
33
45
|
Dynamic: license-file
|
|
34
46
|
|
|
35
47
|
# INTEGRATE Python Module
|
|
36
48
|
|
|
37
|
-
[](https://github.com/AUProbGeo/integrate_module/actions/workflows/docs.yml)
|
|
38
50
|
[](https://badge.fury.io/py/integrate-module)
|
|
39
|
-
[](https://
|
|
51
|
+
[](https://auprobgeo.github.io/integrate_module/)
|
|
40
52
|
[](https://opensource.org/licenses/MIT)
|
|
41
53
|
[](https://www.python.org/downloads/)
|
|
42
54
|
|
|
@@ -47,42 +59,86 @@ This repository contains the INTEGRATE Python module for localized probabilistic
|
|
|
47
59
|
|
|
48
60
|
Assuming you already have Python 3.10+ installed:
|
|
49
61
|
|
|
50
|
-
`libaarhusxyz` must be installed manually before installing `integrate_module`, because the version that supports Python >= 3.12 is only available on GitHub (not PyPI):
|
|
51
|
-
|
|
52
|
-
pip install git+https://github.com/cultpenguin/libaarhusxyz
|
|
53
62
|
pip install integrate_module
|
|
54
63
|
|
|
55
64
|
On Windows, this will also install the Python wrapper for GA-AEM (1D EM forward modeling - GPL v2 code): [ga-aem-forward-win](https://pypi.org/project/ga-aem-forward-win/)
|
|
56
65
|
|
|
57
66
|
On Linux/macOS, you will need to install GA-AEM manually.
|
|
58
67
|
|
|
59
|
-
### Using
|
|
68
|
+
### Using uv (recommended, from PyPI)
|
|
69
|
+
|
|
70
|
+
[uv](https://github.com/astral-sh/uv) is a fast Python package manager. Install it first if needed:
|
|
71
|
+
|
|
72
|
+
# Install uv (Linux/macOS)
|
|
73
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
74
|
+
# or: pip install uv
|
|
75
|
+
|
|
76
|
+
# Create virtual environment in .venv/ inside the module root
|
|
77
|
+
cd path/to/integrate_module
|
|
78
|
+
uv venv .venv --python 3.11
|
|
79
|
+
|
|
80
|
+
# Activate
|
|
81
|
+
source .venv/bin/activate # Linux/macOS
|
|
82
|
+
.venv\Scripts\activate # Windows
|
|
83
|
+
|
|
84
|
+
# Install integrate module
|
|
85
|
+
uv pip install integrate_module
|
|
60
86
|
|
|
61
|
-
|
|
87
|
+
### Using uv (from source)
|
|
88
|
+
|
|
89
|
+
# Install uv (Linux/macOS)
|
|
90
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
91
|
+
# or: pip install uv
|
|
92
|
+
|
|
93
|
+
# Create .venv and install all dependencies in one step (recommended for development)
|
|
94
|
+
cd path/to/integrate_module
|
|
95
|
+
uv sync
|
|
96
|
+
|
|
97
|
+
# Activate
|
|
98
|
+
source .venv/bin/activate # Linux/macOS
|
|
99
|
+
.venv\Scripts\activate # Windows
|
|
100
|
+
|
|
101
|
+
### Using pip + venv (from PyPI, on Ubuntu)
|
|
102
|
+
|
|
103
|
+
# Install python3-venv
|
|
62
104
|
sudo apt install python3-venv
|
|
63
|
-
|
|
64
|
-
# Create virtual environment
|
|
65
|
-
|
|
66
|
-
|
|
105
|
+
|
|
106
|
+
# Create virtual environment in .venv/ inside the module root
|
|
107
|
+
cd path/to/integrate_module
|
|
108
|
+
python3 -m venv .venv
|
|
109
|
+
source .venv/bin/activate
|
|
67
110
|
pip install --upgrade pip
|
|
68
|
-
|
|
111
|
+
|
|
69
112
|
# Install integrate module
|
|
70
113
|
pip install integrate_module
|
|
71
|
-
|
|
72
|
-
### Using pip (from source, on Ubuntu)
|
|
73
114
|
|
|
74
|
-
|
|
115
|
+
### Using pip + venv (from source, on Ubuntu)
|
|
116
|
+
|
|
117
|
+
# Install python3-venv
|
|
75
118
|
sudo apt install python3-venv
|
|
76
119
|
|
|
77
|
-
# Create virtual environment
|
|
78
|
-
|
|
79
|
-
|
|
120
|
+
# Create virtual environment in .venv/ inside the module root
|
|
121
|
+
cd path/to/integrate_module
|
|
122
|
+
python3 -m venv .venv
|
|
123
|
+
source .venv/bin/activate
|
|
80
124
|
pip install --upgrade pip
|
|
81
125
|
|
|
82
|
-
# Install integrate module
|
|
83
|
-
cd path/to/integrate_module
|
|
126
|
+
# Install integrate module from source
|
|
84
127
|
pip install -e .
|
|
85
128
|
|
|
129
|
+
### Installing documentation dependencies
|
|
130
|
+
|
|
131
|
+
To also install the packages needed to build the Sphinx documentation, use the `docs` extra:
|
|
132
|
+
|
|
133
|
+
# With uv (from source)
|
|
134
|
+
uv sync --extra docs
|
|
135
|
+
|
|
136
|
+
# With uv pip (from source)
|
|
137
|
+
uv pip install -e ".[docs]"
|
|
138
|
+
|
|
139
|
+
# With pip (from source)
|
|
140
|
+
pip install -e ".[docs]"
|
|
141
|
+
|
|
86
142
|
### Using Conda + pip (from PyPI)
|
|
87
143
|
|
|
88
144
|
Create a Conda environment (called integrate) and install the required modules:
|
|
@@ -164,7 +220,7 @@ The `develop` branch contains the current development code and may be updated fr
|
|
|
164
220
|
An extra set of tests and examples are located in the ``experimental`` sub-branch `https://github.com/cultpenguin/integrate_module_experimental/ <https://github.com/cultpenguin/integrate_module_experimental/>`_.
|
|
165
221
|
Please ask the developers for access to this branch if needed. To clone the main repository with the experimental branch, use:
|
|
166
222
|
|
|
167
|
-
git clone --recurse-submodules git@github.com:
|
|
223
|
+
git clone --recurse-submodules git@github.com:AUProbGeo/integrate_module.git
|
|
168
224
|
|
|
169
225
|
You may need to run the following command to update the submodules:
|
|
170
226
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# INTEGRATE Python Module
|
|
2
2
|
|
|
3
|
-
[](https://github.com/AUProbGeo/integrate_module/actions/workflows/docs.yml)
|
|
4
4
|
[](https://badge.fury.io/py/integrate-module)
|
|
5
|
-
[](https://
|
|
5
|
+
[](https://auprobgeo.github.io/integrate_module/)
|
|
6
6
|
[](https://opensource.org/licenses/MIT)
|
|
7
7
|
[](https://www.python.org/downloads/)
|
|
8
8
|
|
|
@@ -13,42 +13,86 @@ This repository contains the INTEGRATE Python module for localized probabilistic
|
|
|
13
13
|
|
|
14
14
|
Assuming you already have Python 3.10+ installed:
|
|
15
15
|
|
|
16
|
-
`libaarhusxyz` must be installed manually before installing `integrate_module`, because the version that supports Python >= 3.12 is only available on GitHub (not PyPI):
|
|
17
|
-
|
|
18
|
-
pip install git+https://github.com/cultpenguin/libaarhusxyz
|
|
19
16
|
pip install integrate_module
|
|
20
17
|
|
|
21
18
|
On Windows, this will also install the Python wrapper for GA-AEM (1D EM forward modeling - GPL v2 code): [ga-aem-forward-win](https://pypi.org/project/ga-aem-forward-win/)
|
|
22
19
|
|
|
23
20
|
On Linux/macOS, you will need to install GA-AEM manually.
|
|
24
21
|
|
|
25
|
-
### Using
|
|
22
|
+
### Using uv (recommended, from PyPI)
|
|
23
|
+
|
|
24
|
+
[uv](https://github.com/astral-sh/uv) is a fast Python package manager. Install it first if needed:
|
|
25
|
+
|
|
26
|
+
# Install uv (Linux/macOS)
|
|
27
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
28
|
+
# or: pip install uv
|
|
29
|
+
|
|
30
|
+
# Create virtual environment in .venv/ inside the module root
|
|
31
|
+
cd path/to/integrate_module
|
|
32
|
+
uv venv .venv --python 3.11
|
|
33
|
+
|
|
34
|
+
# Activate
|
|
35
|
+
source .venv/bin/activate # Linux/macOS
|
|
36
|
+
.venv\Scripts\activate # Windows
|
|
37
|
+
|
|
38
|
+
# Install integrate module
|
|
39
|
+
uv pip install integrate_module
|
|
26
40
|
|
|
27
|
-
|
|
41
|
+
### Using uv (from source)
|
|
42
|
+
|
|
43
|
+
# Install uv (Linux/macOS)
|
|
44
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
45
|
+
# or: pip install uv
|
|
46
|
+
|
|
47
|
+
# Create .venv and install all dependencies in one step (recommended for development)
|
|
48
|
+
cd path/to/integrate_module
|
|
49
|
+
uv sync
|
|
50
|
+
|
|
51
|
+
# Activate
|
|
52
|
+
source .venv/bin/activate # Linux/macOS
|
|
53
|
+
.venv\Scripts\activate # Windows
|
|
54
|
+
|
|
55
|
+
### Using pip + venv (from PyPI, on Ubuntu)
|
|
56
|
+
|
|
57
|
+
# Install python3-venv
|
|
28
58
|
sudo apt install python3-venv
|
|
29
|
-
|
|
30
|
-
# Create virtual environment
|
|
31
|
-
|
|
32
|
-
|
|
59
|
+
|
|
60
|
+
# Create virtual environment in .venv/ inside the module root
|
|
61
|
+
cd path/to/integrate_module
|
|
62
|
+
python3 -m venv .venv
|
|
63
|
+
source .venv/bin/activate
|
|
33
64
|
pip install --upgrade pip
|
|
34
|
-
|
|
65
|
+
|
|
35
66
|
# Install integrate module
|
|
36
67
|
pip install integrate_module
|
|
37
|
-
|
|
38
|
-
### Using pip (from source, on Ubuntu)
|
|
39
68
|
|
|
40
|
-
|
|
69
|
+
### Using pip + venv (from source, on Ubuntu)
|
|
70
|
+
|
|
71
|
+
# Install python3-venv
|
|
41
72
|
sudo apt install python3-venv
|
|
42
73
|
|
|
43
|
-
# Create virtual environment
|
|
44
|
-
|
|
45
|
-
|
|
74
|
+
# Create virtual environment in .venv/ inside the module root
|
|
75
|
+
cd path/to/integrate_module
|
|
76
|
+
python3 -m venv .venv
|
|
77
|
+
source .venv/bin/activate
|
|
46
78
|
pip install --upgrade pip
|
|
47
79
|
|
|
48
|
-
# Install integrate module
|
|
49
|
-
cd path/to/integrate_module
|
|
80
|
+
# Install integrate module from source
|
|
50
81
|
pip install -e .
|
|
51
82
|
|
|
83
|
+
### Installing documentation dependencies
|
|
84
|
+
|
|
85
|
+
To also install the packages needed to build the Sphinx documentation, use the `docs` extra:
|
|
86
|
+
|
|
87
|
+
# With uv (from source)
|
|
88
|
+
uv sync --extra docs
|
|
89
|
+
|
|
90
|
+
# With uv pip (from source)
|
|
91
|
+
uv pip install -e ".[docs]"
|
|
92
|
+
|
|
93
|
+
# With pip (from source)
|
|
94
|
+
pip install -e ".[docs]"
|
|
95
|
+
|
|
52
96
|
### Using Conda + pip (from PyPI)
|
|
53
97
|
|
|
54
98
|
Create a Conda environment (called integrate) and install the required modules:
|
|
@@ -130,7 +174,7 @@ The `develop` branch contains the current development code and may be updated fr
|
|
|
130
174
|
An extra set of tests and examples are located in the ``experimental`` sub-branch `https://github.com/cultpenguin/integrate_module_experimental/ <https://github.com/cultpenguin/integrate_module_experimental/>`_.
|
|
131
175
|
Please ask the developers for access to this branch if needed. To clone the main repository with the experimental branch, use:
|
|
132
176
|
|
|
133
|
-
git clone --recurse-submodules git@github.com:
|
|
177
|
+
git clone --recurse-submodules git@github.com:AUProbGeo/integrate_module.git
|
|
134
178
|
|
|
135
179
|
You may need to run the following command to update the submodules:
|
|
136
180
|
|
|
@@ -102,6 +102,7 @@ from integrate.integrate_plot import plot_feature_2d
|
|
|
102
102
|
from integrate.integrate_plot import plot_posterior_cumulative_thickness
|
|
103
103
|
from integrate.integrate_plot import h5_get_clim_cmap
|
|
104
104
|
from integrate.integrate_plot import get_colormap_and_limits
|
|
105
|
+
from integrate.integrate_plot import plot_xy
|
|
105
106
|
from integrate.integrate_plot import find_points_along_line_segments
|
|
106
107
|
from integrate.integrate_plot import plot_boreholes
|
|
107
108
|
|
|
@@ -120,11 +121,16 @@ from integrate.integrate_borehole import save_borehole_data
|
|
|
120
121
|
|
|
121
122
|
# Import query functions
|
|
122
123
|
from integrate.integrate_query import query
|
|
124
|
+
from integrate.integrate_query import query_probability
|
|
125
|
+
from integrate.integrate_query import query_percentile
|
|
123
126
|
from integrate.integrate_query import query_plot
|
|
127
|
+
from integrate.integrate_query import query_percentile_plot
|
|
124
128
|
from integrate.integrate_query import save_query
|
|
125
129
|
from integrate.integrate_query import load_query
|
|
126
130
|
from integrate.integrate_query import get_prior_model_info
|
|
131
|
+
from integrate.integrate_query import prior_describe
|
|
127
132
|
from integrate.integrate_query import query_from_text
|
|
133
|
+
from integrate.integrate_query import title_from_json
|
|
128
134
|
from integrate.integrate_query import query_test_llm
|
|
129
135
|
|
|
130
136
|
# Import gex module functions
|
|
@@ -895,6 +895,7 @@ def forward_gaaem(C=np.array(()),
|
|
|
895
895
|
from tqdm import tqdm
|
|
896
896
|
|
|
897
897
|
showInfo = kwargs.get('showInfo', 0)
|
|
898
|
+
progress_callback = kwargs.get('progress_callback', None)
|
|
898
899
|
if (showInfo<0):
|
|
899
900
|
disableTqdm=True
|
|
900
901
|
else:
|
|
@@ -1067,7 +1068,13 @@ def forward_gaaem(C=np.array(()),
|
|
|
1067
1068
|
|
|
1068
1069
|
# Compute forward data
|
|
1069
1070
|
t1=time.time()
|
|
1071
|
+
# Throttle callback to ~100 updates so it does not dominate runtime
|
|
1072
|
+
progress_step = max(1, nd // 100)
|
|
1073
|
+
|
|
1070
1074
|
for i in tqdm(range(nd), mininterval=1, disable=disableTqdm, desc='gatdaem1d', leave=False):
|
|
1075
|
+
if progress_callback and ((i + 1) % progress_step == 0 or i + 1 == nd):
|
|
1076
|
+
_report_progress(progress_callback, i + 1, nd,
|
|
1077
|
+
'computing', 'Forward modeling (%d/%d soundings)' % (i + 1, nd))
|
|
1071
1078
|
if C.ndim==1:
|
|
1072
1079
|
# Only one model
|
|
1073
1080
|
conductivity = C
|
|
@@ -1255,6 +1262,8 @@ def prior_data_gaaem(f_prior_h5, file_gex=None, stmfiles=None, N=0, doMakePriorC
|
|
|
1255
1262
|
Ncpu = kwargs.get('Ncpu', 0)
|
|
1256
1263
|
# of 'Nproc' is set in kwargs use it
|
|
1257
1264
|
Ncpu = kwargs.get('Nproc', Ncpu)
|
|
1265
|
+
# Pop (not get): the callback must never be pickled to worker processes
|
|
1266
|
+
progress_callback = kwargs.pop('progress_callback', None)
|
|
1258
1267
|
|
|
1259
1268
|
if showInfo>0:
|
|
1260
1269
|
print('prior_data_gaaem: %s/%s -- starting' % (type, method))
|
|
@@ -1349,22 +1358,24 @@ def prior_data_gaaem(f_prior_h5, file_gex=None, stmfiles=None, N=0, doMakePriorC
|
|
|
1349
1358
|
if im_height>0:
|
|
1350
1359
|
if (showInfo>0):
|
|
1351
1360
|
print('Using tx_height')
|
|
1352
|
-
D = ig.forward_gaaem(C=C,
|
|
1353
|
-
thickness=thickness,
|
|
1354
|
-
tx_height=tx_height,
|
|
1355
|
-
file_gex=file_gex,
|
|
1361
|
+
D = ig.forward_gaaem(C=C,
|
|
1362
|
+
thickness=thickness,
|
|
1363
|
+
tx_height=tx_height,
|
|
1364
|
+
file_gex=file_gex,
|
|
1356
1365
|
stmfiles=stmfiles,
|
|
1357
|
-
Nhank=Nhank,
|
|
1358
|
-
Nfreq=Nfreq,
|
|
1359
|
-
parallel=parallel,
|
|
1366
|
+
Nhank=Nhank,
|
|
1367
|
+
Nfreq=Nfreq,
|
|
1368
|
+
parallel=parallel,
|
|
1369
|
+
progress_callback=progress_callback, **kwargs)
|
|
1360
1370
|
else:
|
|
1361
|
-
D = ig.forward_gaaem(C=C,
|
|
1362
|
-
thickness=thickness,
|
|
1363
|
-
file_gex=file_gex,
|
|
1371
|
+
D = ig.forward_gaaem(C=C,
|
|
1372
|
+
thickness=thickness,
|
|
1373
|
+
file_gex=file_gex,
|
|
1364
1374
|
stmfiles=stmfiles,
|
|
1365
|
-
Nhank=Nhank,
|
|
1366
|
-
Nfreq=Nfreq,
|
|
1367
|
-
parallel=parallel,
|
|
1375
|
+
Nhank=Nhank,
|
|
1376
|
+
Nfreq=Nfreq,
|
|
1377
|
+
parallel=parallel,
|
|
1378
|
+
progress_callback=progress_callback, **kwargs)
|
|
1368
1379
|
if is_log:
|
|
1369
1380
|
D = np.log10(D)
|
|
1370
1381
|
else:
|
|
@@ -1384,14 +1395,19 @@ def prior_data_gaaem(f_prior_h5, file_gex=None, stmfiles=None, N=0, doMakePriorC
|
|
|
1384
1395
|
# 1: Define a function to compute a chunk
|
|
1385
1396
|
## OUTSIDE
|
|
1386
1397
|
# 2: Create chunks
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1398
|
+
if progress_callback is None:
|
|
1399
|
+
n_chunks = Ncpu
|
|
1400
|
+
else:
|
|
1401
|
+
# Finer chunking gives smoother live progress updates
|
|
1402
|
+
n_chunks = min(C.shape[0], Ncpu * 4)
|
|
1403
|
+
C_chunks = np.array_split(C, n_chunks)
|
|
1404
|
+
|
|
1405
|
+
if im_height>0:
|
|
1406
|
+
tx_height_chunks = np.array_split(tx_height, n_chunks)
|
|
1407
|
+
|
|
1392
1408
|
else:
|
|
1393
|
-
# create tx_height_chunks as a list of length
|
|
1394
|
-
tx_height_chunks = [np.array(())]*
|
|
1409
|
+
# create tx_height_chunks as a list of length n_chunks, where each entry is tx_height=np.array(())
|
|
1410
|
+
tx_height_chunks = [np.array(())]*n_chunks
|
|
1395
1411
|
|
|
1396
1412
|
|
|
1397
1413
|
import os
|
|
@@ -1421,7 +1437,22 @@ def prior_data_gaaem(f_prior_h5, file_gex=None, stmfiles=None, N=0, doMakePriorC
|
|
|
1421
1437
|
else:
|
|
1422
1438
|
ctx = multiprocessing.get_context('fork')
|
|
1423
1439
|
with ctx.Pool(processes=Ncpu) as p:
|
|
1424
|
-
|
|
1440
|
+
if progress_callback is None:
|
|
1441
|
+
D_chunks = p.starmap(forward_gaaem_chunk_partial, zip(C_chunks, tx_height_chunks))
|
|
1442
|
+
else:
|
|
1443
|
+
# apply_async + ordered get() keeps chunk order for the
|
|
1444
|
+
# concatenate below while reporting per finished chunk
|
|
1445
|
+
async_results = [p.apply_async(forward_gaaem_chunk_partial, args=(Cc, th))
|
|
1446
|
+
for Cc, th in zip(C_chunks, tx_height_chunks)]
|
|
1447
|
+
D_chunks = []
|
|
1448
|
+
n_total = C.shape[0]
|
|
1449
|
+
n_done = 0
|
|
1450
|
+
for r in async_results:
|
|
1451
|
+
D_chunk = r.get()
|
|
1452
|
+
D_chunks.append(D_chunk)
|
|
1453
|
+
n_done += D_chunk.shape[0]
|
|
1454
|
+
_report_progress(progress_callback, n_done, n_total,
|
|
1455
|
+
'computing', 'Forward modeling (%d/%d soundings)' % (n_done, n_total))
|
|
1425
1456
|
finally:
|
|
1426
1457
|
if _spec_patched:
|
|
1427
1458
|
_main_module.__spec__ = None
|
|
@@ -1444,7 +1475,10 @@ def prior_data_gaaem(f_prior_h5, file_gex=None, stmfiles=None, N=0, doMakePriorC
|
|
|
1444
1475
|
t_elapsed = t2 - t1
|
|
1445
1476
|
if (showInfo>-1):
|
|
1446
1477
|
print('prior_data_gaaem: Time=%5.1fs/%d soundings. %4.1fms/sounding, %3.1fit/s' % (t_elapsed, N, 1000*t_elapsed/N,N/t_elapsed))
|
|
1447
|
-
|
|
1478
|
+
|
|
1479
|
+
_report_progress(progress_callback, N, N,
|
|
1480
|
+
'saving', 'Saving forward data to %s' % f_prior_data_h5)
|
|
1481
|
+
|
|
1448
1482
|
# Write D to f_prior['/D1']
|
|
1449
1483
|
with h5py.File(f_prior_data_h5, 'a') as f_prior:
|
|
1450
1484
|
if Dname in f_prior:
|
|
@@ -1463,7 +1497,10 @@ def prior_data_gaaem(f_prior_h5, file_gex=None, stmfiles=None, N=0, doMakePriorC
|
|
|
1463
1497
|
f_prior[Dname].attrs['Nfreq'] = Nfreq
|
|
1464
1498
|
|
|
1465
1499
|
integrate_update_prior_attributes(f_prior_data_h5)
|
|
1466
|
-
|
|
1500
|
+
|
|
1501
|
+
_report_progress(progress_callback, N, N,
|
|
1502
|
+
'completed', 'Forward data saved to %s' % f_prior_data_h5)
|
|
1503
|
+
|
|
1467
1504
|
return f_prior_data_h5
|
|
1468
1505
|
|
|
1469
1506
|
|
|
@@ -1567,6 +1604,21 @@ def prior_data_identity(f_prior_h5, id=0, im=1, N=0, doMakePriorCopy=False, **kw
|
|
|
1567
1604
|
return f_prior_data_h5, id
|
|
1568
1605
|
|
|
1569
1606
|
# %% PRIOR MODEL GENERATORS
|
|
1607
|
+
def _report_progress(progress_callback, current, total, phase, status):
|
|
1608
|
+
"""Invoke an optional progress callback as progress_callback(current, total, info_dict).
|
|
1609
|
+
|
|
1610
|
+
Follows the same convention as integrate_rejection(): info_dict carries
|
|
1611
|
+
'phase' and 'status' keys, and errors raised by the callback are ignored
|
|
1612
|
+
so they can never break the computation.
|
|
1613
|
+
"""
|
|
1614
|
+
if progress_callback is None:
|
|
1615
|
+
return
|
|
1616
|
+
try:
|
|
1617
|
+
progress_callback(current, total, {'phase': phase, 'status': status})
|
|
1618
|
+
except Exception:
|
|
1619
|
+
pass
|
|
1620
|
+
|
|
1621
|
+
|
|
1570
1622
|
def prior_model_layered(lay_dist='uniform', dz = 1, z_max = 90,
|
|
1571
1623
|
NLAY_min=3, NLAY_max=6, NLAY_deg=6,
|
|
1572
1624
|
RHO_dist='log-uniform', RHO_min=0.1, RHO_max=5000, RHO_mean=100, RHO_std=80,
|
|
@@ -1642,6 +1694,7 @@ def prior_model_layered(lay_dist='uniform', dz = 1, z_max = 90,
|
|
|
1642
1694
|
|
|
1643
1695
|
showInfo = kwargs.get('showInfo', 0)
|
|
1644
1696
|
f_prior_h5 = kwargs.get('f_prior_h5', '')
|
|
1697
|
+
progress_callback = kwargs.get('progress_callback', None)
|
|
1645
1698
|
|
|
1646
1699
|
if NLAY_max < NLAY_min:
|
|
1647
1700
|
NLAY_max = NLAY_min
|
|
@@ -1715,7 +1768,13 @@ def prior_model_layered(lay_dist='uniform', dz = 1, z_max = 90,
|
|
|
1715
1768
|
else:
|
|
1716
1769
|
iterator = range(N)
|
|
1717
1770
|
|
|
1771
|
+
# Throttle callback to ~100 updates so it does not dominate runtime
|
|
1772
|
+
progress_step = max(1, N // 100)
|
|
1773
|
+
|
|
1718
1774
|
for i in iterator:
|
|
1775
|
+
if progress_callback and ((i + 1) % progress_step == 0 or i + 1 == N):
|
|
1776
|
+
_report_progress(progress_callback, i + 1, N,
|
|
1777
|
+
'generating', 'Generating prior realizations')
|
|
1719
1778
|
n_lay = NLAY[i]
|
|
1720
1779
|
n_boundaries = n_lay - 1
|
|
1721
1780
|
|
|
@@ -1739,6 +1798,8 @@ def prior_model_layered(lay_dist='uniform', dz = 1, z_max = 90,
|
|
|
1739
1798
|
|
|
1740
1799
|
if showInfo > 0:
|
|
1741
1800
|
print("prior_model_layered: Saving prior model to %s" % f_prior_h5)
|
|
1801
|
+
_report_progress(progress_callback, N, N,
|
|
1802
|
+
'saving', 'Saving prior model to %s' % f_prior_h5)
|
|
1742
1803
|
|
|
1743
1804
|
# Save to HDF5 file
|
|
1744
1805
|
im = 0
|
|
@@ -1796,6 +1857,9 @@ def prior_model_layered(lay_dist='uniform', dz = 1, z_max = 90,
|
|
|
1796
1857
|
**save_kwargs,
|
|
1797
1858
|
)
|
|
1798
1859
|
|
|
1860
|
+
_report_progress(progress_callback, N, N,
|
|
1861
|
+
'completed', 'Prior model saved to %s' % f_prior_h5)
|
|
1862
|
+
|
|
1799
1863
|
return f_prior_h5
|
|
1800
1864
|
|
|
1801
1865
|
def prior_model_workbench_direct(N=100000, RHO_dist='log-uniform', z1=0, z_max= 100,
|
|
@@ -1862,6 +1926,11 @@ def prior_model_workbench_direct(N=100000, RHO_dist='log-uniform', z1=0, z_max=
|
|
|
1862
1926
|
|
|
1863
1927
|
showInfo = kwargs.get('showInfo', 0)
|
|
1864
1928
|
f_prior_h5 = kwargs.get('f_prior_h5', '')
|
|
1929
|
+
progress_callback = kwargs.get('progress_callback', None)
|
|
1930
|
+
|
|
1931
|
+
# Fully vectorized: only coarse phase updates are possible
|
|
1932
|
+
_report_progress(progress_callback, 0, 100,
|
|
1933
|
+
'generating', 'Generating prior realizations')
|
|
1865
1934
|
|
|
1866
1935
|
if nlayers<1:
|
|
1867
1936
|
nlayers = 30
|
|
@@ -1905,13 +1974,15 @@ def prior_model_workbench_direct(N=100000, RHO_dist='log-uniform', z1=0, z_max=
|
|
|
1905
1974
|
|
|
1906
1975
|
if (showInfo>0):
|
|
1907
1976
|
print("prior_model_workbench_direct: Saving prior model to %s" % f_prior_h5)
|
|
1977
|
+
_report_progress(progress_callback, 80, 100,
|
|
1978
|
+
'saving', 'Saving prior model to %s' % f_prior_h5)
|
|
1908
1979
|
|
|
1909
1980
|
if (showInfo>1):
|
|
1910
1981
|
print("Saving '/M1' prior model %s" % f_prior_h5)
|
|
1911
1982
|
ig.save_prior_model(f_prior_h5,M_rho.astype(np.float32),
|
|
1912
1983
|
im=1,
|
|
1913
1984
|
name='Resistivity',
|
|
1914
|
-
is_discrete = 0,
|
|
1985
|
+
is_discrete = 0,
|
|
1915
1986
|
x = z,
|
|
1916
1987
|
z = z,
|
|
1917
1988
|
delete_if_exist = True,
|
|
@@ -1919,6 +1990,8 @@ def prior_model_workbench_direct(N=100000, RHO_dist='log-uniform', z1=0, z_max=
|
|
|
1919
1990
|
showInfo=showInfo,
|
|
1920
1991
|
)
|
|
1921
1992
|
|
|
1993
|
+
_report_progress(progress_callback, 100, 100,
|
|
1994
|
+
'completed', 'Prior model saved to %s' % f_prior_h5)
|
|
1922
1995
|
|
|
1923
1996
|
return f_prior_h5
|
|
1924
1997
|
|
|
@@ -1996,6 +2069,7 @@ def prior_model_workbench(N=100000, p=2, z1=0, z_max= 100, dz=1,
|
|
|
1996
2069
|
|
|
1997
2070
|
f_prior_h5 = kwargs.get('f_prior_h5', '')
|
|
1998
2071
|
showInfo = kwargs.get('showInfo', 0)
|
|
2072
|
+
progress_callback = kwargs.get('progress_callback', None)
|
|
1999
2073
|
if nlayers>0:
|
|
2000
2074
|
NLAY_min = nlayers
|
|
2001
2075
|
NLAY_max = nlayers
|
|
@@ -2046,7 +2120,13 @@ def prior_model_workbench(N=100000, p=2, z1=0, z_max= 100, dz=1,
|
|
|
2046
2120
|
M_rho_sparse = np.ones((N, nm_sparse))*np.nan
|
|
2047
2121
|
|
|
2048
2122
|
|
|
2123
|
+
# Throttle callback to ~100 updates so it does not dominate runtime
|
|
2124
|
+
progress_step = max(1, N // 100)
|
|
2125
|
+
|
|
2049
2126
|
for i in tqdm(range(N), mininterval=1, disable=(showInfo<0), desc='prior_workbench', leave=False):
|
|
2127
|
+
if progress_callback and ((i + 1) % progress_step == 0 or i + 1 == N):
|
|
2128
|
+
_report_progress(progress_callback, i + 1, N,
|
|
2129
|
+
'generating', 'Generating prior realizations')
|
|
2050
2130
|
nlayers = NLAY[i][0]
|
|
2051
2131
|
#print(nlayers)
|
|
2052
2132
|
z2=z_max
|
|
@@ -2086,6 +2166,8 @@ def prior_model_workbench(N=100000, p=2, z1=0, z_max= 100, dz=1,
|
|
|
2086
2166
|
|
|
2087
2167
|
if (showInfo>0):
|
|
2088
2168
|
print("prior_model_workbench: Saving prior model to %s" % f_prior_h5)
|
|
2169
|
+
_report_progress(progress_callback, N, N,
|
|
2170
|
+
'saving', 'Saving prior model to %s' % f_prior_h5)
|
|
2089
2171
|
|
|
2090
2172
|
if (showInfo>1):
|
|
2091
2173
|
print("Saving '/M1' prior model %s" % f_prior_h5)
|
|
@@ -2117,13 +2199,16 @@ def prior_model_workbench(N=100000, p=2, z1=0, z_max= 100, dz=1,
|
|
|
2117
2199
|
ig.save_prior_model(f_prior_h5,NLAY.astype(np.float32),
|
|
2118
2200
|
im=3,
|
|
2119
2201
|
name = 'Number of layers',
|
|
2120
|
-
is_discrete=0,
|
|
2121
|
-
x=np.array([0]),
|
|
2202
|
+
is_discrete=0,
|
|
2203
|
+
x=np.array([0]),
|
|
2122
2204
|
z=np.array([0]),
|
|
2123
|
-
force_replace=True,
|
|
2205
|
+
force_replace=True,
|
|
2124
2206
|
showInfo=showInfo,
|
|
2125
2207
|
)
|
|
2126
2208
|
|
|
2209
|
+
_report_progress(progress_callback, N, N,
|
|
2210
|
+
'completed', 'Prior model saved to %s' % f_prior_h5)
|
|
2211
|
+
|
|
2127
2212
|
# return the full filepath to f_prior_h5
|
|
2128
2213
|
return f_prior_h5
|
|
2129
2214
|
|
|
@@ -3688,7 +3773,7 @@ def timing_plot(f_timing=''):
|
|
|
3688
3773
|
plt.stackplot(N_arr, T, labels=['Prior', 'Forward', 'Rejection', 'PostStat'])
|
|
3689
3774
|
plt.plot(N_arr, T_total[:, i_proc], 'k--')
|
|
3690
3775
|
plt.xscale('log')
|
|
3691
|
-
|
|
3776
|
+
plt.yscale('log')
|
|
3692
3777
|
plt.xlabel('$N_{lookup}$')
|
|
3693
3778
|
plt.ylabel('Time [$s$]')
|
|
3694
3779
|
plt.title('Cumulative time, using %d processors' % Nproc_arr[i_proc])
|