gmprocess 2.0.0__tar.gz → 2.1.0__tar.gz

{gmprocess-2.0.0 → gmprocess-2.1.0}/MANIFEST.in

@@ -1,5 +1,7 @@
 recursive-include src/gmprocess/data *
 recursive-include src/gmprocess/bin *
+include src/gmprocess/gui/gui_ucla/Documentation.md
+include src/gmprocess/gui/gui_ucla/GUI_human_review.ipynb
 prune tests
 prune docs
 prune notebooks

{gmprocess-2.0.0/src/gmprocess.egg-info → gmprocess-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gmprocess
-Version: 2.0.0
+Version: 2.1.0
 Summary: USGS Automated Ground Motion Processing Software
 Author-email: Eric Thompson <emthompsone@usgs.gov>, Mike Hearne <mhearne@usgs.gov>, Brad Aagaard <baagaard@usgs.gov>, Bruce Worden <cbworden@contractor.usgs.gov>, John Rekoske <jrekoske@ucsd.edu>, Heather Hunsinger <hhunsinger@usgs.gov>, Gabe Ferragut <gferragut@usgs.gov>
 License: License
@@ -64,7 +64,7 @@ Requires-Dist: lxml>=4.6.1
 Requires-Dist: matplotlib>=3.1.0
 Requires-Dist: numba>=0.58
 Requires-Dist: numpy<2.0,>=1.21
-Requires-Dist: obspy>=1.4.0
+Requires-Dist: obspy>=1.4.1
 Requires-Dist: openpyxl>=3.0.8
 Requires-Dist: openquake.engine>=3.18
 Requires-Dist: pandas>=1.0
@@ -89,7 +89,7 @@ Requires-Dist: pylint>=2.17; extra == "dev"
 Provides-Extra: test
 Requires-Dist: pytest>=6.2; extra == "test"
 Requires-Dist: pytest-cov>=2.12; extra == "test"
-Requires-Dist: pytest-console-scripts>=1.2; extra == "test"
+Requires-Dist: pytest-console-scripts>=1.4; extra == "test"
 Requires-Dist: vcrpy>=4.1; extra == "test"
 Provides-Extra: doc
 Requires-Dist: altair>=4.2.2; extra == "doc"

{gmprocess-2.0.0 → gmprocess-2.1.0}/pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
     "matplotlib>=3.1.0",
     "numba>=0.58",
     "numpy>=1.21,<2.0",
-    "obspy>=1.4.0",
+    "obspy>=1.4.1",
     "openpyxl>=3.0.8",
     "openquake.engine>=3.18",
     "pandas>=1.0",
@@ -59,7 +59,7 @@ dev = [
 test = [
     "pytest>=6.2",
     "pytest-cov>=2.12",
-    "pytest-console-scripts>=1.2",
+    "pytest-console-scripts>=1.4",
     "vcrpy>=4.1",
 ]
 doc = [

{gmprocess-2.0.0 → gmprocess-2.1.0}/src/gmprocess/apps/gmrecords.py

@@ -84,17 +84,36 @@ class GMrecordsApp(object):
             self.parser.print_help()
             sys.exit()
         else:
+            if self.args.datadir is None and self.args.confdir is None:
+                self.use_projects = True
+            elif self.args.datadir is not None and self.args.confdir is not None:
+                self.use_projects = False
+            else:
+                raise ValueError(
+                    "Arguments --datadir and --confdir both must be set or both must "
+                    "not be set (and thus these paths will be taken from the "
+                    "configured project)."
+                )
+
             subcmds_noinit = ["init", "processing_steps"]
             if self.args.subcommand not in subcmds_noinit:
                 self._initialize()
 
             subcmds_quiet = ["init", "projects", "proj", "processing_steps"]
             if self.args.subcommand not in subcmds_quiet and not self.args.quiet:
-                # Print the current project information to avoid confusion
-                proj = Project.from_config(self.projects_conf, self.project_name)
+                if self.use_projects:
+                    # Print the current project information to avoid confusion
+                    proj = Project.from_config(self.projects_conf, self.project_name)
+                else:
+                    proj_paths = {
+                        "conf_path": self.args.confdir,
+                        "data_path": self.args.datadir,
+                    }
+                    proj = Project("None", proj_paths, "None", True)
                 print("-" * 80)
                 print(proj)
                 print("-" * 80)
+
             # -----------------------------------------------------------------
             # This calls the init method of the subcommand that was specified
             # at the command line and hands off the GmpApp object ("self") as
@@ -157,11 +176,17 @@ class GMrecordsApp(object):
 
     @property
     def conf_path(self):
-        return (self.projects_path / self.current_project["conf_path"]).resolve()
+        if self.projects_path:
+            return (self.projects_path / self.current_project["conf_path"]).resolve()
+        else:
+            return Path(self.current_project["conf_path"]).resolve()
 
     @property
     def data_path(self):
-        return (self.projects_path / self.current_project["data_path"]).resolve()
+        if self.projects_path:
+            return (self.projects_path / self.current_project["data_path"]).resolve()
+        else:
+            return Path(self.current_project["data_path"]).resolve()
 
     def _initialize(self):
         require_config = self.args.subcommand not in ["projects", "proj"]
@@ -183,7 +208,7 @@ class GMrecordsApp(object):
 
     def _load_config(self, require_config=True):
         self.projects_conf = None
-        if not self.projects_file.is_file():
+        if (not self.projects_file.is_file()) and self.use_projects:
             if not require_config:
                 return
 
@@ -211,9 +236,23 @@ class GMrecordsApp(object):
             else:
                 sys.exit(0)
 
-        self.projects_conf = configobj.ConfigObj(
-            str(self.projects_file), encoding="utf-8"
-        )
+        if not self.use_projects:
+            self.projects_path = None
+            self.projects_file = None
+            self.projects_conf = {
+                "project": "None",
+                "projects": {
+                    "None": {
+                        "conf_path": self.args.confdir,
+                        "data_path": self.args.datadir,
+                    }
+                },
+            }
+        else:
+            self.projects_conf = configobj.ConfigObj(
+                str(self.projects_file), encoding="utf-8"
+            )
+
         projmod.validate_projects_config(self.projects_conf, self.projects_path)
 
         if "CALLED_FROM_PYTEST" in os.environ:
@@ -338,6 +377,27 @@ class GMrecordsApp(object):
             action="store_true",
         )
 
+        parser.add_argument(
+            "--datadir",
+            type=str,
+            default=None,
+            help=(
+                "Path to data directory. "
+                "Setting this disables the use of projects. "
+                "Setting this also requires --confdir to be set. "
+            ),
+        )
+        parser.add_argument(
+            "--confdir",
+            type=str,
+            default=None,
+            help=(
+                "Path to directly containing config files. "
+                "Setting this disables the use of projects. "
+                "Setting this also requires --datadir to be set."
+            ),
+        )
+
         # Parsers for subcommands
         subparsers = parser.add_subparsers(
             title="Subcommands", dest="subcommand", metavar="<command> (<aliases>)"

{gmprocess-2.0.0 → gmprocess-2.1.0}/src/gmprocess/core/scalar_event.py

@@ -323,7 +323,10 @@ def get_event_ids(ids=None, file_ids=None, data_dir=None):
     event_ids = ids or file_ids
     if event_ids:
         if isinstance(event_ids, str):
-            event_ids = [event_ids]
+            if "," in event_ids:
+                event_ids = [eid.strip() for eid in event_ids.split(",")]
+            else:
+                event_ids = [event_ids]
     elif data_dir:
         event_ids = [ev_id.name for ev_id in data_dir.iterdir() if ev_id.is_dir()]
         event_ids = sorted(event_ids)

{gmprocess-2.0.0 → gmprocess-2.1.0}/src/gmprocess/data/config_production.yml

@@ -14,7 +14,7 @@ fetchers:
             # StableShallow GMM.
             # The distance will be the the distance at which the threshold PGA (in g) is
             # exceeded based on the GMM.
-            pga: 0.001
+            pga: 0.0001
             # The maximum distance (in km) that can be returned.
             max_distance: 800.0
             # NOTE: the GMM is evaluated with a number of simplifications, including 
@@ -526,7 +526,7 @@ metrics:
   # Do *not* delete a key to remove a component, it will be replaced with the default.
   # To remove a component, set it's types as an empty list.
   components_and_types: 
-    channels: [pga, pgv, sa, duration, cav]
+    channels: [pga, pgv, sa, duration, cav, arias]
     rotd: [pga, pgv, sa]
     geometric_mean: [duration]
     quadratic_mean: [fas]
@@ -547,7 +547,7 @@ metrics:
     # Additional parameters required for FAS
     fas:
         smoothing_method: konno_ohmachi
-        smoothing_parameter: 20.0
+        smoothing_parameter: 188.5
         allow_nans: True
         frequencies:
             # Parameters defining an array of frequencies.

gmprocess-2.1.0/src/gmprocess/gui/gui_ucla/Documentation.md

@@ -0,0 +1,88 @@
+# Graphical User Interface (GUI) to aid human review of processed-ground-motions
+
+```{admonition} Authors
+:class: note
+
+Maria E. Ramos-Sepulveda, Scott J. Brandenberg, Tristan Buckreis, Meibai Li, Shako Ahmed, and Jonathan P. Stewart
+```
+
+The UCLA geotechnical group prepared a graphical user interface (GUI) to facilitate a human review of records processed by gmprocess. The algorithms utilized by gmprocess to accept/reject, window, and filter records work well most of the time, but cannot account for all possible issues that might arise. For this reason, reviewing the processed records is prudent. The purpose of this GUI is to facilitate rapid review. Acceleration and displacement time series, as well as response spectra can be visually inspected to find any potential errors. The GUI utilizes Jupyter and allows the user to load an HDF5 file output from gmprocess, interact with the data by accepting or rejecting records, and adjusting high-pass and low-pass filter parameters. In the example below we describe the procedure to run and interact with the GUI.
+
+## Requirements
+- A workspace.h5 file produced by gmprocess through the "process_waveforms" step. For example `gmrecords --eventid nc73291880 download`, `gmrecords assemble` and `gmrecords process_waveforms` must be run before going through this review step. 
+- The Python script named `HumanReviewGUI.py` that contains the class variable and function defintions that will be called by a Jupyter Notebook. 
+- A Jupyter Notebook that instantiates the HumanReviewGUI class.
+- gmprocess
+- obspy
+- ipywidgets
+- pandas
+- numpy
+- matplotlib
+- ucla_geotech_tools.response_spectrum
+- json
+- prov
+- ipympl
+
+## Jupyter notebook
+
+Creating the GUI requires only three lines of code, shown below. Older versions of Jupyter might require %matplotlib notebook instead of %matplotlib widget. Running these three lines should produce plots like the ones shown in the sections that follow.
+
+```python
+%matplotlib widget
+import gmprocess.gui.gui_ucla.HumanReviewGUI as HRG
+GUI = HRG.HumanReviewGUI()
+```
+
+## Plots description
+The interface is shown in Figure 1, and consists of a control panel and four plots. The control panel contains a field where users input the filename and load the HDF5 file. The progress bar on top displays how many records have been loaded. Once the loading is done, the progress bar reflects how many records have been accepted. The toggle buttons next to the load button filter data by showing only data waveforms that have not been reviewed, those that have been accepted, and/or those that have been rejected. The third row of the control panel has buttons to browse to the previous or next record, along with a dropdown menu for selecting a specific record. Next to the right are the buttons for accepting or rejecting the record. A record should be accepted when the user considers it to have been processed correctly, and rejected when the user believes that the record should not be processed because it is too noisy or has other flaws. The bottom row of the control panel contains the name of the user who processed the record, the earthquake magnitude, and the epicentral distance. Users can also set the high-pass and low-pass corner frequencies (fchp and fclp respectively), and uncheck a box to control whether the fclp is not applied at all. The filters utilize the same order and filter type as specified by the user in gmprocess and are not configurable in the GUI. We found that these options do not require adjustment after processing in gmprocess.
+<br/><br/>
+The orange traces depict the unfiltered record. In this context, "unfiltered" refers to the trace that has not gone through the Butterworth filter, but is trimmed, demeaned and tapered. The blue trace ("filtered GUI") is filtered using the `apply_filter` function inside the python script and it will change if any of the corner frequencies are modified. The blue time series includes the minimum between 60 seconds of noise and the maximum noise length of a record while the traces called from gmprocess have 2 seconds (default in config file). The translucent red ("filtered gmprocess") is the record filtered inside gmprocess. This trace will not change if the corner frequencies are modified. The purpose of including both filtered traces (blue and red) is so the user can compare any improvements done in the GUI. The red trace was set to translucent to aid the visual comparison. As a result of superimposing the red over the blue, this trace will seem purple whenever "filtered GUI" overlaps "filtered gmprocess". The smoothed noise spectrum is shown in green and only for the Fourier transform spectra. The black dotted line is the theoretical acceleration decay at low frequency according to the $f^2$ model (Brune 1970; Boore and Bommer 2005), the user can use it as reference. The dashed-black lines depict corner frequencies in the Fourier transform spectra and the highest-usable period (factor of 0.8) in the response spectra. The grey solid and dashed lines are the signal-to-noise ratio (SNR) and SNR threshold, respectively. The ordinates on the the Fourier transform spectra are not meant to be used to read the SNR, the idea is to evaluate the SNR with respect to the SNR threshold, signal, and noise traces.
+
+```{figure} ../../_static/ucla_gui/interface3.png
+*Figure 1: Interface of Graphic User Interface for human review*
+```
+
+## Procedure
+Changes implemented in the GUI are recorded in the HDF5 file through an `auxiliary_data` folder named `review`. In the `review` section you can find which records were accepted or rejected and the corresponding corner frequency values. To integrate these changes into the `provenance` and update the `waveforms`, the user needs to run `gmrecords process -r`. The changes in the `auxiliary_data` folder will then be reflected in the `provenance` directory of the .h5 file.
+
+The processing steps are listed below:
+- Load workspace.h5
+- Evaluate acceleration, displacement, amplitude and response plots.
+- Increase values of fchp if necessary.
+- Change and/or remove value of fclp if necessary.
+- Accept or reject the record accordingly. The GUI will automatically proceed to the next trace.
+- Run `gmrecords process -r` to overwrite the traces objects in the workspace.h5.
+
+### Changes in high-pass corner frequencies
+The most common modification is an adjustment to the fchp. The automated selection of fchp is done using the `auto_fchp` package inside gmprocess, which fits a 6th order polynomial to the displacement trace and adjusts the fchp until the amplitude of the polynomial fit is a specified fraction of the amplitude of the displacement trace. Sometimes, the automated fchp value is not high enough to discard lingering long period noise (which can be observed as a wobbly displacement trace, shown in Figure 2). In this case, the fchp was increased to 0.19 Hz to render a displacement trace that is stable prior to the p-wave arrival, as shown in Figure 3.
+
+```{figure} ../../_static/ucla_gui/fchp_before3.png
+*Figure 2: Example trace with an automated selected fchp resulting in an irregular displacement time series*
+```
+
+```{figure} ../../_static/ucla_gui/fchp_after3.png
+*Figure 3: Example trace shown in Figure 2 with a modified fchp resulting in a regular displacement time series*
+```
+
+### Changes in low-pass corner frequencies
+Depending on how the user configured their gmprocess, the fclp could be applied every time or never. In the GUI, by default, fclp is applied to the blue trace ("filtered GUI"). The value shown in the header of the plots is the fclp selected by gmprocess which is chosen to be the minimum between $0.7*Nyquist$ frequency and the highest frequency where SNR undergoes certain threshold. High amplitude short period noise could be reflected as an irregular peak in the response spectrum at short periods (Figure 4). The noise can be excluded from the trace by checking the box to the right and applying an fclp of the same value (Figure 5). In this case a low-pass filter at 14.6 Hz stabilized the short-period portion of the response spectrum.
+
+```{figure} ../../_static/ucla_gui/fclp_before3.png
+*Figure 4: Example trace without fclp resulting in an irregular response spectra shape*
+```
+
+```{figure} ../../_static/ucla_gui/fclp_after3.png
+*Figure 5: Example trace shown in Figure 4 with fclp resulting in a flat response spectra at short periods*
+```
+
+### Example of rejected record
+Some records contaminated with significant noise might pass the "failing criteria" of gmprocess. We believe this may be the result of randomness in the pre-event noise window. Usually, these records are easy to spot in the acceleration time series. An example is shown in Figure 6. Although the earthquake trace is visible above the noise level, we consider this record to be too noisy and opted to reject it.
+
+```{figure} ../../_static/ucla_gui/rejected3.png
+*Figure 6: Example trace with significant noise that should be rejected*
+```
+
+## Limitations
+1. Gmprocess might miscompute the location of the p-wave arrival or the coda. None of those issues could be further improved using the GUI. 
+2. The GUI will not update the `provenance` folder in the workspace.h5. If the user wants to pass this information from the `auxiliary_data` folder to the `provenance` folder and recompute the `waveforms`, the user needs to run `gmrecords process -r`.
+3. The GUI will convert the units of unprocessed traces into $m/s^2$ if the raw data is in g units or in a mseed format. Otherwise, the traces in the plots will be offset and the user will have to go to the python code and apply the corresponding conversion.

gmprocess-2.1.0/src/gmprocess/gui/gui_ucla/GUI_human_review.ipynb

@@ -0,0 +1,39 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "893fe763",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%matplotlib widget\n",
+    "import gmprocess.gui.gui_ucla.HumanReviewGUI as HRG\n",
+    "GUI = HRG.HumanReviewGUI()"
+   ]
+  }
+ ],
+ "metadata": {
+  "IMAGE_NAME": "taccsciapps/ds-nb-img:base-0.1.3",
+  "UUID": "b10b61b0-984e-11ee-8eb8-86839c98a1b4",
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}