requake 0.8.1__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {requake-0.8.1 → requake-0.8.2}/CHANGELOG.md +10 -0
- {requake-0.8.1 → requake-0.8.2}/PKG-INFO +10 -7
- {requake-0.8.1 → requake-0.8.2}/README.md +8 -5
- {requake-0.8.1 → requake-0.8.2}/requake/__init__.py +9 -0
- {requake-0.8.1 → requake-0.8.2}/requake/_version.py +3 -3
- {requake-0.8.1 → requake-0.8.2}/requake/config/__init__.py +1 -1
- {requake-0.8.1 → requake-0.8.2}/requake/config/configspec.conf +6 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/parse_arguments.py +22 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/rq_setup.py +46 -3
- {requake-0.8.1 → requake-0.8.2}/requake/database/trace_metadata.py +9 -1
- requake-0.8.2/requake/scan/scan_catalog.py +220 -0
- requake-0.8.2/requake/scan/scan_catalog_helpers.py +317 -0
- requake-0.8.2/requake/scan/scan_catalog_pairs.py +182 -0
- requake-0.8.2/requake/scan/scan_catalog_workers.py +561 -0
- {requake-0.8.1 → requake-0.8.2}/requake/waveforms/waveforms.py +21 -7
- {requake-0.8.1 → requake-0.8.2}/requake.egg-info/PKG-INFO +10 -7
- {requake-0.8.1 → requake-0.8.2}/requake.egg-info/SOURCES.txt +3 -0
- requake-0.8.1/requake/scan/scan_catalog.py +0 -645
- {requake-0.8.1 → requake-0.8.2}/CONTRIBUTORS.txt +0 -0
- {requake-0.8.1 → requake-0.8.2}/LICENSE.txt +0 -0
- {requake-0.8.1 → requake-0.8.2}/MANIFEST.in +0 -0
- {requake-0.8.1 → requake-0.8.2}/pyproject.toml +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/catalog.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/print_catalog.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/read_catalog.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/read_catalog_from_csv.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/read_catalog_from_fdsnws.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/catalog/read_catalog_from_quakeml.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/config.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/configobj/LICENSE +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/configobj/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/configobj/_version.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/configobj/validate.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/generic_printer.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/config/utils.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/database/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/database/catalog.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/database/db.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/database/families.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/database/pairs.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/database/templates.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/build_families.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/build_templates.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/families.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/flag_family.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/pairs.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/print_families.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/families/print_pairs.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/formulas/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/formulas/conversion.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/formulas/moment.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/formulas/slip.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/main.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/cached_tiler.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/colormaps.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/map_families.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/map_tiles.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/plot_cumulative.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/plot_families.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/plot_pair.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/plot_timespans.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/plot/plot_utils.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/scan/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/scan/scan_templates.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/waveforms/__init__.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/waveforms/arrivals.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/waveforms/station_metadata.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake/waveforms/waveform_pair.py +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake.egg-info/dependency_links.txt +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake.egg-info/entry_points.txt +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake.egg-info/requires.txt +0 -0
- {requake-0.8.1 → requake-0.8.2}/requake.egg-info/top_level.txt +0 -0
- {requake-0.8.1 → requake-0.8.2}/setup.cfg +0 -0
- {requake-0.8.1 → requake-0.8.2}/setup.py +0 -0
|
@@ -2,6 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
Copyright (c) 2021-2026 Claudio Satriano <satriano@ipgp.fr>
|
|
4
4
|
|
|
5
|
+
## [0.8.2] - 2026-06-03
|
|
6
|
+
|
|
7
|
+
Note: you might want to run `requake update_config` to update your config file
|
|
8
|
+
to the latest version.
|
|
9
|
+
|
|
10
|
+
- Parallel processing of earthquake pairs in `scan_catalog`. On by default
|
|
11
|
+
with automatic detection of the number of worker processes. This can be
|
|
12
|
+
configured with the `catalog_scan_nprocs` config option.
|
|
13
|
+
|
|
5
14
|
## [0.8.1] - 2026-05-31
|
|
6
15
|
|
|
7
16
|
- Slimmed-down DB size by optimizing `event_pairs` storage. This allows storing
|
|
@@ -178,6 +187,7 @@ Mostly a bugfix release with a slightly improved trace plotting.
|
|
|
178
187
|
|
|
179
188
|
- Initial release, not yet feature complete
|
|
180
189
|
|
|
190
|
+
[0.8.2]: https://github.com/SeismicSource/requake/compare/v0.8.1...v0.8.2
|
|
181
191
|
[0.8.1]: https://github.com/SeismicSource/requake/compare/v0.8...v0.8.1
|
|
182
192
|
[0.8]: https://github.com/SeismicSource/requake/compare/v0.7.3...v0.8
|
|
183
193
|
[0.7.3]: https://github.com/SeismicSource/requake/compare/v0.7.2...v0.7.3
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: requake
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: Repeating earthquakes search and analysis
|
|
5
5
|
Author-email: Claudio Satriano <satriano@ipgp.fr>
|
|
6
6
|
License-Expression: GPL-3.0-or-later
|
|
@@ -37,7 +37,7 @@ Dynamic: description
|
|
|
37
37
|
Dynamic: description-content-type
|
|
38
38
|
Dynamic: license-file
|
|
39
39
|
|
|
40
|
-
<img src="https://cdn.jsdelivr.net/gh/SeismicSource/requake@
|
|
40
|
+
<img src="https://cdn.jsdelivr.net/gh/SeismicSource/requake@9071fd7343802f3ef642a16f6bb02ff5ce1d88f5/imgs/Requake_logo.svg" width="400">
|
|
41
41
|
|
|
42
42
|
# Requake
|
|
43
43
|
|
|
@@ -223,11 +223,14 @@ repeating earthquake families:
|
|
|
223
223
|
|
|
224
224
|
## Performances
|
|
225
225
|
|
|
226
|
-
- `requake scan_catalog`
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
226
|
+
- `requake scan_catalog` is designed to fully utilize available CPU cores by
|
|
227
|
+
processing earthquake pairs in parallel. On an M2 MacBook Air, scanning
|
|
228
|
+
95,034 earthquake pairs downloaded via FDSNWS took approximately 5 minutes
|
|
229
|
+
using 7 worker processes, yielding ~19,000 pairs per minute. When repeating
|
|
230
|
+
the same scan with a fully cached set of waveforms, runtime dropped to 84
|
|
231
|
+
seconds (~68,000 pairs per minute). These results indicate that overall
|
|
232
|
+
performance is typically dominated by waveform download latency rather than
|
|
233
|
+
computation.
|
|
231
234
|
|
|
232
235
|
- `requake build_families` is fast™.
|
|
233
236
|
|
|
@@ -184,11 +184,14 @@ repeating earthquake families:
|
|
|
184
184
|
|
|
185
185
|
## Performances
|
|
186
186
|
|
|
187
|
-
- `requake scan_catalog`
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
187
|
+
- `requake scan_catalog` is designed to fully utilize available CPU cores by
|
|
188
|
+
processing earthquake pairs in parallel. On an M2 MacBook Air, scanning
|
|
189
|
+
95,034 earthquake pairs downloaded via FDSNWS took approximately 5 minutes
|
|
190
|
+
using 7 worker processes, yielding ~19,000 pairs per minute. When repeating
|
|
191
|
+
the same scan with a fully cached set of waveforms, runtime dropped to 84
|
|
192
|
+
seconds (~68,000 pairs per minute). These results indicate that overall
|
|
193
|
+
performance is typically dominated by waveform download latency rather than
|
|
194
|
+
computation.
|
|
192
195
|
|
|
193
196
|
- `requake build_families` is fast™.
|
|
194
197
|
|
|
@@ -9,5 +9,14 @@ Initialize requake package.
|
|
|
9
9
|
GNU General Public License v3.0 or later
|
|
10
10
|
(https://www.gnu.org/licenses/gpl-3.0-standalone.html)
|
|
11
11
|
"""
|
|
12
|
+
import multiprocessing
|
|
13
|
+
import signal
|
|
14
|
+
|
|
15
|
+
# Child processes spawned for parallel scans may import requake before worker
|
|
16
|
+
# initializers run. Ignore SIGINT early in children so Ctrl+C is managed by
|
|
17
|
+
# the parent process only.
|
|
18
|
+
if multiprocessing.current_process().name != 'MainProcess':
|
|
19
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
20
|
+
|
|
12
21
|
from . import _version
|
|
13
22
|
__version__ = _version.get_versions()['version']
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2026-
|
|
11
|
+
"date": "2026-06-03T14:39:27+0200",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "0.8.
|
|
14
|
+
"full-revisionid": "9071fd7343802f3ef642a16f6bb02ff5ce1d88f5",
|
|
15
|
+
"version": "0.8.2"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -9,7 +9,7 @@ Configuration and initialization of the Requake package.
|
|
|
9
9
|
GNU General Public License v3.0 or later
|
|
10
10
|
(https://www.gnu.org/licenses/gpl-3.0-standalone.html)
|
|
11
11
|
"""
|
|
12
|
-
from .rq_setup import rq_exit # noqa
|
|
12
|
+
from .rq_setup import rq_exit, wait_for_sigint_pause # noqa
|
|
13
13
|
from .generic_printer import generic_printer # noqa
|
|
14
14
|
# The config object is created in config.py and needs to be populated
|
|
15
15
|
# when using requake from command line, this is done by the configure()
|
|
@@ -72,10 +72,16 @@ catalog_trace_id = force_list(default=NET.STA.LOC.CHAN)
|
|
|
72
72
|
## catalog-based scan. Higher values increase memory usage and can reduce
|
|
73
73
|
## repeated waveform downloads.
|
|
74
74
|
catalog_waveform_cache_size = integer(min=2, default=5000)
|
|
75
|
+
## Per-worker waveform cache size for parallel catalog scan.
|
|
76
|
+
## Set to 0 to derive it automatically from catalog_waveform_cache_size.
|
|
77
|
+
catalog_waveform_cache_size_parallel = integer(min=0, default=0)
|
|
75
78
|
## Keep a persistent copy of fetched event waveforms on disk to speed up
|
|
76
79
|
## reruns. This on-disk cache lives in OUTDIR/waveform_cache and is not
|
|
77
80
|
## limited by catalog_waveform_cache_size.
|
|
78
81
|
catalog_waveform_disk_cache_enabled = boolean(default=true)
|
|
82
|
+
## Number of worker processes for catalog scan.
|
|
83
|
+
## 0 means automatic selection; 1 disables parallelism.
|
|
84
|
+
catalog_scan_nprocs = integer(min=0, default=0)
|
|
79
85
|
|
|
80
86
|
#### Template-based scan
|
|
81
87
|
### The following parameters are for template-based scan:
|
|
@@ -49,6 +49,21 @@ class SubcommandHelpFormatter(argparse.RawDescriptionHelpFormatter):
|
|
|
49
49
|
return parts
|
|
50
50
|
|
|
51
51
|
|
|
52
|
+
def _nonnegative_int(value):
|
|
53
|
+
"""Argparse type for integers >= 0."""
|
|
54
|
+
try:
|
|
55
|
+
int_value = int(value)
|
|
56
|
+
except ValueError as err:
|
|
57
|
+
raise argparse.ArgumentTypeError(
|
|
58
|
+
f'invalid non-negative integer value: {value}'
|
|
59
|
+
) from err
|
|
60
|
+
if int_value < 0:
|
|
61
|
+
raise argparse.ArgumentTypeError(
|
|
62
|
+
f'invalid non-negative integer value: {value}'
|
|
63
|
+
)
|
|
64
|
+
return int_value
|
|
65
|
+
|
|
66
|
+
|
|
52
67
|
def parse_arguments(progname='requake'):
|
|
53
68
|
"""
|
|
54
69
|
Parse command line arguments.
|
|
@@ -177,6 +192,13 @@ def parse_arguments(progname='requake'):
|
|
|
177
192
|
action='store_true',
|
|
178
193
|
help='continue an interrupted scan and keep existing event pairs'
|
|
179
194
|
)
|
|
195
|
+
scan_catalog.add_argument(
|
|
196
|
+
'--nprocs',
|
|
197
|
+
type=_nonnegative_int,
|
|
198
|
+
default=None,
|
|
199
|
+
help='number of worker processes for scan_catalog '
|
|
200
|
+
'(0: auto, 1: disable parallelism)'
|
|
201
|
+
)
|
|
180
202
|
# ---
|
|
181
203
|
# --- print_pairs
|
|
182
204
|
print_pairs = subparser.add_parser(
|
|
@@ -10,11 +10,13 @@ Setup functions for Requake.
|
|
|
10
10
|
(https://www.gnu.org/licenses/gpl-3.0-standalone.html)
|
|
11
11
|
"""
|
|
12
12
|
import contextlib
|
|
13
|
+
import multiprocessing
|
|
13
14
|
import sys
|
|
14
15
|
import os
|
|
15
16
|
import shutil
|
|
16
17
|
import logging
|
|
17
18
|
import signal
|
|
19
|
+
import time
|
|
18
20
|
import tqdm
|
|
19
21
|
from .._version import get_versions
|
|
20
22
|
from ..database.db import get_db_path
|
|
@@ -31,6 +33,10 @@ PYTHON_VERSION_STR = None
|
|
|
31
33
|
NUMPY_VERSION_STR = None
|
|
32
34
|
SCIPY_VERSION_STR = None
|
|
33
35
|
OBSPY_VERSION_STR = None
|
|
36
|
+
SIGINT_CONFIRM_WINDOW_S = 2.0
|
|
37
|
+
SIGINT_PAUSE_POLL_S = 0.05
|
|
38
|
+
_LAST_SIGINT_TS = None
|
|
39
|
+
_SIGINT_PAUSE_UNTIL_TS = None
|
|
34
40
|
|
|
35
41
|
|
|
36
42
|
def _check_library_versions():
|
|
@@ -330,8 +336,45 @@ def rq_exit(retval=0, abort=False, progname='requake'):
|
|
|
330
336
|
|
|
331
337
|
|
|
332
338
|
def sigint_handler(_sig, _frame):
|
|
333
|
-
"""
|
|
334
|
-
|
|
339
|
+
"""Ask for confirmation on first Ctrl+C and abort on second."""
|
|
340
|
+
global _LAST_SIGINT_TS # pylint: disable=global-statement
|
|
341
|
+
global _SIGINT_PAUSE_UNTIL_TS # pylint: disable=global-statement
|
|
335
342
|
|
|
343
|
+
if multiprocessing.current_process().name != 'MainProcess':
|
|
344
|
+
signal.default_int_handler(_sig, _frame)
|
|
345
|
+
return
|
|
346
|
+
|
|
347
|
+
now = time.monotonic()
|
|
348
|
+
if (
|
|
349
|
+
_LAST_SIGINT_TS is not None
|
|
350
|
+
and now - _LAST_SIGINT_TS <= SIGINT_CONFIRM_WINDOW_S
|
|
351
|
+
):
|
|
352
|
+
rq_exit(1, abort=True)
|
|
353
|
+
_LAST_SIGINT_TS = now
|
|
354
|
+
_SIGINT_PAUSE_UNTIL_TS = now + SIGINT_CONFIRM_WINDOW_S
|
|
355
|
+
print(
|
|
356
|
+
'\nCtrl+C pressed. '
|
|
357
|
+
f'Press Ctrl+C again within {SIGINT_CONFIRM_WINDOW_S:g}s '
|
|
358
|
+
'to abort.'
|
|
359
|
+
)
|
|
336
360
|
|
|
337
|
-
|
|
361
|
+
|
|
362
|
+
def wait_for_sigint_pause():
|
|
363
|
+
"""Block while the first Ctrl+C confirmation window is active."""
|
|
364
|
+
global _SIGINT_PAUSE_UNTIL_TS # pylint: disable=global-statement
|
|
365
|
+
|
|
366
|
+
if multiprocessing.current_process().name != 'MainProcess':
|
|
367
|
+
return
|
|
368
|
+
while _SIGINT_PAUSE_UNTIL_TS is not None:
|
|
369
|
+
if time.monotonic() >= _SIGINT_PAUSE_UNTIL_TS:
|
|
370
|
+
_SIGINT_PAUSE_UNTIL_TS = None
|
|
371
|
+
break
|
|
372
|
+
time.sleep(SIGINT_PAUSE_POLL_S)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
if multiprocessing.current_process().name == 'MainProcess':
|
|
376
|
+
signal.signal(signal.SIGINT, sigint_handler)
|
|
377
|
+
else:
|
|
378
|
+
# Workers must ignore SIGINT from startup to avoid pool breakage during
|
|
379
|
+
# spawn, before the worker initializer runs.
|
|
380
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
@@ -165,12 +165,20 @@ def _trace_metadata_fallback_rows(pairs):
|
|
|
165
165
|
if pair.trace_id in seen_trace_ids:
|
|
166
166
|
continue
|
|
167
167
|
seen_trace_ids.add(pair.trace_id)
|
|
168
|
+
try:
|
|
169
|
+
sampling_rate_hz = _infer_sampling_rate(pair)
|
|
170
|
+
except PairsMetadataError as err:
|
|
171
|
+
logger.warning(
|
|
172
|
+
'Skipping trace_metadata fallback row for trace_id '
|
|
173
|
+
f'{pair.trace_id}: {err}'
|
|
174
|
+
)
|
|
175
|
+
continue
|
|
168
176
|
rows.append(
|
|
169
177
|
{
|
|
170
178
|
'trace_id': pair.trace_id,
|
|
171
179
|
'valid_from_utc': TRACE_METADATA_VALID_FROM,
|
|
172
180
|
'valid_to_utc': None,
|
|
173
|
-
'sampling_rate_hz':
|
|
181
|
+
'sampling_rate_hz': sampling_rate_hz,
|
|
174
182
|
'trace_lon': None,
|
|
175
183
|
'trace_lat': None,
|
|
176
184
|
'elevation': None,
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# -*- coding: utf8 -*-
|
|
2
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
3
|
+
"""
|
|
4
|
+
Catalog-based repeater scan for Requake.
|
|
5
|
+
|
|
6
|
+
:copyright:
|
|
7
|
+
2021-2026 Claudio Satriano <satriano@ipgp.fr>
|
|
8
|
+
:license:
|
|
9
|
+
GNU General Public License v3.0 or later
|
|
10
|
+
(https://www.gnu.org/licenses/gpl-3.0-standalone.html)
|
|
11
|
+
"""
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
import logging
|
|
15
|
+
from concurrent.futures.process import BrokenProcessPool
|
|
16
|
+
from ..config import (
|
|
17
|
+
config,
|
|
18
|
+
rq_exit,
|
|
19
|
+
)
|
|
20
|
+
from ..database.db import get_db_path
|
|
21
|
+
from ..catalog import fix_non_locatable_events, read_stored_catalog
|
|
22
|
+
from ..database.pairs import (
|
|
23
|
+
count_pairs,
|
|
24
|
+
write_pair_records,
|
|
25
|
+
)
|
|
26
|
+
from ..database.trace_metadata import store_trace_metadata_from_inventory
|
|
27
|
+
from ..waveforms import (
|
|
28
|
+
load_inventory,
|
|
29
|
+
NoMetadataError, MetadataMismatchError,
|
|
30
|
+
)
|
|
31
|
+
from .scan_catalog_pairs import (
|
|
32
|
+
build_valid_pair_indices,
|
|
33
|
+
load_existing_pair_ids,
|
|
34
|
+
log_pair_grouping_stats,
|
|
35
|
+
mask_existing_pair_indices,
|
|
36
|
+
)
|
|
37
|
+
from .scan_catalog_helpers import (
|
|
38
|
+
get_slurm_context,
|
|
39
|
+
log_slurm_runtime_context,
|
|
40
|
+
resolve_scan_catalog_nprocs,
|
|
41
|
+
)
|
|
42
|
+
from .scan_catalog_workers import (
|
|
43
|
+
process_valid_pair_indices,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
logger = logging.getLogger('scan_catalog')
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _ask_existing_pairs_action(npairs_in_db):
|
|
50
|
+
"""Ask the user whether to overwrite or continue an existing scan."""
|
|
51
|
+
args = config.args
|
|
52
|
+
if args.force:
|
|
53
|
+
return 'overwrite'
|
|
54
|
+
if args.force_continue:
|
|
55
|
+
return 'continue'
|
|
56
|
+
if not sys.stdin.isatty():
|
|
57
|
+
logger.error(
|
|
58
|
+
f'Found {npairs_in_db:n} event pairs in db file '
|
|
59
|
+
f'{get_db_path()}.'
|
|
60
|
+
)
|
|
61
|
+
logger.error(
|
|
62
|
+
'Cannot prompt in non-interactive mode. '
|
|
63
|
+
'Use --force to overwrite or --force-continue to resume.'
|
|
64
|
+
)
|
|
65
|
+
rq_exit(1)
|
|
66
|
+
logger.warning(
|
|
67
|
+
f'Found {npairs_in_db:n} existing event pairs in db file '
|
|
68
|
+
f'{get_db_path()}.'
|
|
69
|
+
)
|
|
70
|
+
logger.warning(
|
|
71
|
+
'You can overwrite them and restart, or continue from where '
|
|
72
|
+
'the previous scan stopped.'
|
|
73
|
+
)
|
|
74
|
+
prompt = (
|
|
75
|
+
'Choose action: [o]verwrite, [c]ontinue, [a]bort '
|
|
76
|
+
'(default: abort): '
|
|
77
|
+
)
|
|
78
|
+
choices = {
|
|
79
|
+
'o': 'overwrite',
|
|
80
|
+
'overwrite': 'overwrite',
|
|
81
|
+
'c': 'continue',
|
|
82
|
+
'continue': 'continue',
|
|
83
|
+
'a': 'abort',
|
|
84
|
+
'abort': 'abort',
|
|
85
|
+
}
|
|
86
|
+
while True:
|
|
87
|
+
answer = input(prompt).strip().lower()
|
|
88
|
+
if not answer:
|
|
89
|
+
return 'abort'
|
|
90
|
+
action = choices.get(answer)
|
|
91
|
+
if action is not None:
|
|
92
|
+
return action
|
|
93
|
+
print('Invalid choice. Please type o, c, or a.')
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _process_pairs(catalog, continue_scan=False, slurm_context=None):
|
|
97
|
+
"""Process event pairs."""
|
|
98
|
+
if not continue_scan:
|
|
99
|
+
write_pair_records([], append=False)
|
|
100
|
+
# Ensure inventory is loaded in the parent process before parent-side
|
|
101
|
+
# writes, so trace_metadata rows keep full interval metadata in
|
|
102
|
+
# parallel mode as in serial mode.
|
|
103
|
+
try:
|
|
104
|
+
load_inventory()
|
|
105
|
+
except (NoMetadataError, MetadataMismatchError) as msg:
|
|
106
|
+
logger.error(msg)
|
|
107
|
+
rq_exit(1)
|
|
108
|
+
# Write trace metadata immediately after tables are created so that
|
|
109
|
+
# the DB is populated even when no pairs are found (e.g. all
|
|
110
|
+
# waveform fetches fail). Uses the inventory already in config.
|
|
111
|
+
trace_ids = (
|
|
112
|
+
[config.args.traceid]
|
|
113
|
+
if getattr(config.args, 'traceid', None) is not None
|
|
114
|
+
else list(config.catalog_trace_id)
|
|
115
|
+
)
|
|
116
|
+
store_trace_metadata_from_inventory(trace_ids)
|
|
117
|
+
nevents = len(catalog)
|
|
118
|
+
initial_npairs = nevents * (nevents - 1) // 2
|
|
119
|
+
logger.info('Building valid event pairs...')
|
|
120
|
+
t_grouping_start = time.monotonic()
|
|
121
|
+
valid_pair_idx = build_valid_pair_indices(catalog)
|
|
122
|
+
grouping_dt = time.monotonic() - t_grouping_start
|
|
123
|
+
logger.info(
|
|
124
|
+
f'Valid-pair spatial grouping completed in {grouping_dt:.1f}s'
|
|
125
|
+
)
|
|
126
|
+
skipped_npairs = 0
|
|
127
|
+
candidate_npairs = len(valid_pair_idx)
|
|
128
|
+
if continue_scan:
|
|
129
|
+
logger.info(
|
|
130
|
+
'Continue-scan mode: loading existing pairs for '
|
|
131
|
+
'pre-processing mask'
|
|
132
|
+
)
|
|
133
|
+
t_resume_filter_start = time.monotonic()
|
|
134
|
+
existing_pair_ids = load_existing_pair_ids(catalog)
|
|
135
|
+
valid_pair_idx, skipped_npairs = mask_existing_pair_indices(
|
|
136
|
+
valid_pair_idx,
|
|
137
|
+
existing_pair_ids,
|
|
138
|
+
nevents,
|
|
139
|
+
)
|
|
140
|
+
resume_filter_dt = time.monotonic() - t_resume_filter_start
|
|
141
|
+
logger.info(
|
|
142
|
+
'Loading existing pair IDs and applying mask completed in '
|
|
143
|
+
f'{resume_filter_dt:.1f}s'
|
|
144
|
+
)
|
|
145
|
+
npairs = len(valid_pair_idx)
|
|
146
|
+
total_valid_pairs = skipped_npairs + npairs
|
|
147
|
+
nprocs = resolve_scan_catalog_nprocs(npairs, slurm_context or {})
|
|
148
|
+
ratio = npairs / initial_npairs if initial_npairs > 0 else 0.0
|
|
149
|
+
logger.info(f'Initial pairs: {initial_npairs:n}')
|
|
150
|
+
logger.info(f'Candidate pairs: {candidate_npairs:n}')
|
|
151
|
+
logger.info(f'Final pairs: {npairs:n}')
|
|
152
|
+
logger.info(f'Pair ratio: {ratio:.6f} ({ratio:.2%})')
|
|
153
|
+
log_pair_grouping_stats(valid_pair_idx)
|
|
154
|
+
logger.info(
|
|
155
|
+
f'Processing {npairs:n} event pairs '
|
|
156
|
+
f'({skipped_npairs:n}/{total_valid_pairs:n} already processed)'
|
|
157
|
+
)
|
|
158
|
+
analyzed_npairs = process_valid_pair_indices(
|
|
159
|
+
catalog,
|
|
160
|
+
valid_pair_idx,
|
|
161
|
+
npairs,
|
|
162
|
+
initial_processed=skipped_npairs,
|
|
163
|
+
total_pairs=total_valid_pairs,
|
|
164
|
+
nprocs=nprocs,
|
|
165
|
+
slurm_context=slurm_context,
|
|
166
|
+
)
|
|
167
|
+
if continue_scan:
|
|
168
|
+
logger.info(
|
|
169
|
+
f'Skipped {skipped_npairs:n} event pairs already present '
|
|
170
|
+
'in the database'
|
|
171
|
+
)
|
|
172
|
+
return analyzed_npairs
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def scan_catalog():
|
|
176
|
+
"""Perform cross-correlation on catalog events."""
|
|
177
|
+
slurm_context = get_slurm_context()
|
|
178
|
+
log_slurm_runtime_context(slurm_context)
|
|
179
|
+
try:
|
|
180
|
+
catalog = read_stored_catalog()
|
|
181
|
+
except (ValueError, FileNotFoundError) as msg:
|
|
182
|
+
logger.error(msg)
|
|
183
|
+
rq_exit(1)
|
|
184
|
+
try:
|
|
185
|
+
fix_non_locatable_events(catalog)
|
|
186
|
+
except MetadataMismatchError as msg:
|
|
187
|
+
logger.error(msg)
|
|
188
|
+
rq_exit(1)
|
|
189
|
+
nevents = len(catalog)
|
|
190
|
+
if nevents < 2:
|
|
191
|
+
logger.error(
|
|
192
|
+
'Not enough events in catalog. '
|
|
193
|
+
'You need at least 2 events to run the scan 😉')
|
|
194
|
+
rq_exit(1)
|
|
195
|
+
logger.info(
|
|
196
|
+
f'{nevents:n} events read from db file {get_db_path()}'
|
|
197
|
+
)
|
|
198
|
+
continue_scan = False
|
|
199
|
+
existing_pairs = count_pairs()
|
|
200
|
+
if existing_pairs > 0:
|
|
201
|
+
action = _ask_existing_pairs_action(existing_pairs)
|
|
202
|
+
if action == 'abort':
|
|
203
|
+
logger.info('Scan aborted by user')
|
|
204
|
+
rq_exit(0)
|
|
205
|
+
continue_scan = action == 'continue'
|
|
206
|
+
try:
|
|
207
|
+
npairs = _process_pairs(
|
|
208
|
+
catalog,
|
|
209
|
+
continue_scan=continue_scan,
|
|
210
|
+
slurm_context=slurm_context,
|
|
211
|
+
)
|
|
212
|
+
except BrokenProcessPool as err:
|
|
213
|
+
logger.error(
|
|
214
|
+
'Parallel scan stopped because a worker process terminated '
|
|
215
|
+
'unexpectedly.'
|
|
216
|
+
)
|
|
217
|
+
logger.debug(f'Broken process pool details: {err}')
|
|
218
|
+
rq_exit(1, abort=True)
|
|
219
|
+
logger.info(f'Processed {npairs:n} event pairs')
|
|
220
|
+
logger.info(f'Done! Output written to {get_db_path()}')
|