gmprocess 2.2.0__tar.gz → 2.3.0__tar.gz

{gmprocess-2.2.0 → gmprocess-2.3.0}/CHANGELOG.md

@@ -1,5 +1,37 @@
 ## main
 
+
+## 2.3.0 / 2025-01-23
+
+- Enhancements
+  - Substantial changes to the default processing steps.
+  - Add support for psudo-response spectra (PSA and PSV) and change default metrics to use PSA rather than SA.
+  - Allow for user to set a different query window duration for different tectonic envirnments.
+  - Increase default query window duraiton.
+  - Add a new "source_path" method to compute the signal end that uses 1/fc as the source duration, which is then added to a path duration that is parameterized with coefficients for an intercept and distance term.
+  - Exclude network 7D, which is for ocean bottom stations.
+  - Apply clipping check to strong motion instruments. In the past, we thought it was safe to assume strong motion instruments did not clip but we have found exceptions to this. 
+  - Update summary plots so that the time axis is the same across raw, accel, vel, and disp plots.
+  - Update report to plot stations in order of epicentral distance.
+  - Update setup_env.sh to use miniforge rather than miniconda and renamed to install.sh to be consistent with other repos.
+  - Add taper before differentiation for velocity instruments.
+  - Expand instrument response QA to include consistency of units with instrument type. 
+  - Set unit registry to treat "counts" as having a unique dimension, rather than being dimensionless. 
+  - Simplify SNR failure message.
+  - Allow water_level to be specified as "None" for the instrument response in config file.
+  - Add QA check for inconsistency in sample rate reported in data and metadata.
+  - Increase `sec_before_split` default value for trimming the record.
+  - Add config options to allow for using a Butterworth filter for filtering prior to instrument response removal, rather than the default options that ObsPy uses with their `remove_response` method. 
+  - Add `bandpass` processing step.
+- Documentation
+  - Fix the description of the --textfile option for gmrecords.
+- Bugfix
+  - Do not exit when event id not found in comcat while looking for a rupture file.
+  - Ensure fill value matches the data type of the trace.
+  - Add provenance entry for when the detrend method is "pre".
+  - Fix errors in some filtering provenance entries.
+  - Bugfix in normalizing pre-event noise spectra.
+
 ## 2.2.0 / 2024-12-23
 - Enhancements
   - Add the PRISM/Jones et al. (doi: 10.1785/0220160200) adaptive baseline correction step.

{gmprocess-2.2.0 → gmprocess-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: gmprocess
-Version: 2.2.0
+Version: 2.3.0
 Summary: USGS Automated Ground Motion Processing Software
 Author-email: Eric Thompson <emthompsone@usgs.gov>, Mike Hearne <mhearne@usgs.gov>, Brad Aagaard <baagaard@usgs.gov>, Bruce Worden <cbworden@contractor.usgs.gov>, John Rekoske <jrekoske@ucsd.edu>, Heather Hunsinger <hhunsinger@usgs.gov>, Gabe Ferragut <gferragut@usgs.gov>
 License: License

{gmprocess-2.2.0 → gmprocess-2.3.0}/code.json

@@ -1,4 +1,45 @@
 [
+    {
+        "name": "groundmotion-processing",
+        "description": "Read and process ground motion waveform data",
+        "version": "v2.3.0",
+        "status": "Production",
+        "organization": "U.S. Geological Survey",
+        "vcs": "git",
+        "repositoryURL": "https://code.usgs.gov/ghsc/esi/groundmotion-processing.git",
+        "homepageURL": "https://code.usgs.gov/ghsc/esi/groundmotion-processing",
+        "downloadURL": "https://code.usgs.gov/ghsc/esi/groundmotion-processing/-/archive/v2.3.0/groundmotion-processing-v2.3.0.zip",
+        "disclaimerURL": "https://code.usgs.gov/ghsc/esi/groundmotion-processing/-/raw/v2.3.0/DISCLAIMER.md",
+        "permissions": {
+            "licenses": [
+                {
+                    "name": "Public Domain, CC0-1.0",
+                    "URL": "https://code.usgs.gov/ghsc/esi/groundmotion-processing/-/raw/v2.3.0/LICENSE.md"
+                }
+            ],
+            "usageType": "openSource",
+            "exemptionText": null
+        },
+        "laborHours": 1,
+        "languages": [
+            "python"
+        ],
+        "tags": [
+            "earthquake",
+            "impact",
+            "strong motion",
+            "amplitudes",
+            "ground motion",
+            "usg-artificial-intelligence"
+        ],
+        "contact": {
+            "name": "Eric Thompson",
+            "email": "emthompson@usgs.gov"
+        },
+        "date": {
+            "metadataLastUpdated": "2025-01-23"
+        }
+    },
     {
         "name": "groundmotion-processing",
         "description": "Read and process ground motion waveform data",

{gmprocess-2.2.0 → gmprocess-2.3.0}/setup_env.sh

@@ -8,11 +8,9 @@ echo "Installing ${VENV}...${cwd}"
 unamestr=`uname`
 if [ "$unamestr" == 'Linux' ]; then
     prof=~/.bashrc
-    mini_conda_url=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
     matplotlibdir=~/.config/matplotlib
 elif [ "$unamestr" == 'FreeBSD' ] || [ "$unamestr" == 'Darwin' ]; then
     prof=~/.bash_profile
-    mini_conda_url=https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
     matplotlibdir=~/.matplotlib
 else
     echo "Unsupported environment. Exiting."
@@ -39,9 +37,7 @@ elif [ ! grep -Fxq "backend" $matplotlibrc ]; then
     echo "NOTE: A non-interactive matplotlib backend (Agg) has been set for this user."
 else
     sed -i '' 's/backend.*/backend : Agg/' $matplotlibrc
-    echo "###############"
     echo "NOTE: $matplotlibrc has been changed to set 'backend : Agg'"
-    echo "###############"
 fi
 
 
@@ -50,21 +46,45 @@ conda --version
 if [ $? -ne 0 ]; then
     echo "No conda detected, installing miniconda..."
 
-    curl -L $mini_conda_url -o miniconda.sh;
+    command -v curl >/dev/null 2>&1 || { echo >&2 "Script requires curl but it's not installed. Aborting."; exit 1; }
+
+    miniforge_url="https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+    curl -L $miniforge_url -o miniforge.sh &>/dev/null
+
+    # if miniforge.sh fails, bow out gracefully
+    if [ $? -ne 0 ];then
+        echo "Failed to run miniconda installer shell script. Exiting."
+        exit 1
+    fi
+
     echo "Install directory: $HOME/miniconda"
 
-    bash miniconda.sh -f -b -p $HOME/miniconda
+    bash miniforge.sh -f -b -p $HOME/miniconda
 
     # Need this to get conda into path
     . $HOME/miniconda/etc/profile.d/conda.sh
+
+    rm miniforge.sh
 else
     echo "conda detected, installing $VENV environment..."
 fi
 
-echo "PATH:"
-echo $PATH
-echo ""
-
+# Update the conda tool
+CVNUM=`conda -V | cut -f2 -d' '`
+LATEST=`conda search conda | tail -1 | tr -s ' ' | cut -f2 -d" "`
+echo "${CVNUM}"
+echo "${LATEST}"
+if [ ${LATEST} != ${CVNUM} ]; then
+    echo "##################Updating conda tool..."
+    CVERSION=`conda -V`
+    echo "Current conda version: ${CVERSION}"
+    conda update -n base -c defaults conda -y
+    CVERSION=`conda -V`
+    echo "New conda version: ${CVERSION}"
+    echo "##################Done updating conda tool..."
+else
+    echo "conda ${CVNUM} already matches latest version ${LATEST}. No update required."
+fi
 
 # only add this line if it does not already exist
 grep "/etc/profile.d/conda.sh" $prof
@@ -72,6 +92,9 @@ if [ $? -ne 0 ]; then
     echo ". $_CONDA_ROOT/etc/profile.d/conda.sh" >> $prof
 fi
 
+# Set libmamba as solver
+conda config --set solver libmamba &>/dev/null
+
 # Start in conda base environment
 echo "Activate base virtual environment"
 conda activate base
@@ -87,7 +110,6 @@ if [ $? -ne 0 ]; then
     exit
 fi
 
-
 # Activate the new environment
 echo "Activating the $VENV virtual environment"
 conda activate $VENV

{gmprocess-2.2.0 → gmprocess-2.3.0}/src/gmprocess/apps/gmrecords.py

@@ -358,7 +358,8 @@ class GMrecordsApp(object):
                 "(2) six columns in which those columns are: "
                 "event_id (string, no spaces), time (any ISO standard for date/time), "
                 "latitutde (float, decimal degrees), longitude (float, decimal "
-                "degrees), depth (float, km), magnitude (float)."
+                "degrees), depth (float, km), magnitude (float), and magnitude type "
+                "(string)."
             ),
         )
 

{gmprocess-2.2.0 → gmprocess-2.3.0}/src/gmprocess/core/scalar_event.py

@@ -276,7 +276,7 @@ def get_events_from_file(filename=None):
                 fields = line.split(",")
                 if len(fields) != 7:
                     raise IOError(
-                        f"Expected 6 columns in events file '{filename}'. Error parsing line '{line}'."
+                        f"Expected 7 columns in events file '{filename}'. Error parsing line '{line}'."
                     )
                 event_values = {
                     "id": fields[0].strip(),

{gmprocess-2.2.0 → gmprocess-2.3.0}/src/gmprocess/core/stationstream.py

@@ -184,8 +184,6 @@ class StationStream(Stream):
 
     def validate(self):
         """Validation checks for Traces within a StationStream."""
-        logging.debug(self)
-
         # Check that channel codes are unique_npts
         self.__check_channels()
 
@@ -520,7 +518,6 @@ def _channel_from_stats(stats):
         response = Response(instrument_sensitivity=sensitivity)
 
     comments = Comment(stats.standard.comments)
-    logging.debug("channel: %s", stats.channel)
     channel = Channel(
         stats.channel,
         stats.location,

{gmprocess-2.2.0 → gmprocess-2.3.0}/src/gmprocess/core/stationtrace.py

@@ -13,11 +13,11 @@ from obspy.core.trace import Trace
 from scipy.integrate import cumulative_trapezoid
 
 # local imports
-from gmprocess.utils.config import get_config
-from gmprocess.utils import constants
+from gmprocess.core.provenance import TraceProvenance
 from gmprocess.io.cosmos.data_structures import BUILDING_TYPES
 from gmprocess.io.seedname import get_units_type
-from gmprocess.core.provenance import TraceProvenance
+from gmprocess.utils.config import get_config
+from gmprocess.utils import constants
 
 
 PROCESS_LEVELS = {
@@ -386,6 +386,7 @@ class StationTrace(Trace):
                 "output_units": output_units,
             },
         )
+
         return self
 
     def zero_pad(self, length):
@@ -398,9 +399,18 @@ class StationTrace(Trace):
             length (float):
                 The length (in sec) to padd with zeros before and after the trace.
         """
+        old_start_time = self.stats.starttime
+        old_end_time = self.stats.endtime
         start_time = self.stats.starttime - length
         end_time = self.stats.endtime + length
-        self.trim(starttime=start_time, endtime=end_time, pad=True, fill_value=0.0)
+        fill_value = np.array(0, dtype=self.data.dtype)
+        self.trim(
+            starttime=start_time,
+            endtime=end_time,
+            pad=True,
+            fill_value=fill_value,
+            suppress_provenance=True,
+        )
 
         self.set_provenance(
             "pad",
@@ -408,10 +418,24 @@ class StationTrace(Trace):
                 "fill_value": 0.0,
                 "new_start_time": start_time,
                 "new_end_time": end_time,
+                "old_start_time": old_start_time,
+                "old_end_time": old_end_time,
             },
         )
         return self
 
+    def strip_zero_pad(self):
+        """Remove zero pads from trace."""
+
+        pad_prov = self.get_provenance("pad")
+        if len(pad_prov) > 1:
+            raise ValueError("More than one 'pad' entry in provenance.")
+        start_time = pad_prov[0]["prov_attributes"].get("old_start_time")
+        end_time = pad_prov[0]["prov_attributes"].get("old_end_time")
+        self.trim(starttime=start_time, endtime=end_time)
+
+        return self
+
     def taper(self, max_percentage, type="hann", max_length=None, side="both"):
         """Taper trace and add entry to provenance.
 
@@ -472,6 +496,7 @@ class StationTrace(Trace):
         pad=False,
         nearest_sample=True,
         fill_value=None,
+        suppress_provenance=False,
     ):
         """Trim trace and add entry to provenance.
 
@@ -489,14 +514,15 @@ class StationTrace(Trace):
                 See obspy docs.
             fill_value (int, float):
                 Fill value for gaps.
+            suppress_provenance (bool):
+                Do not put a provenance entry. Useful becasue we need a custom
+                provenance entry for zero padding, which calls this method.
         """
-        self.set_provenance(
-            "cut",
-            {
-                "new_start_time": starttime,
-                "new_end_time": endtime,
-            },
-        )
+        if not suppress_provenance:
+            prov_dict = {"new_start_time": starttime, "new_end_time": endtime}
+            if pad:
+                prov_dict["fill_value"] = fill_value
+            self.set_provenance("cut", prov_dict)
         return super().trim(starttime, endtime, pad, nearest_sample, fill_value)
 
     def integrate(
@@ -709,7 +735,6 @@ class StationTrace(Trace):
                 self.set_provenance(
                     "bandpass_filter",
                     {
-                        "bandpass_filter": {"code": "bp", "label": "Bandpass Filter"},
                         "filter_type": "Butterworth ObsPy",
                         "filter_order": corners,
                         "number_of_passes": number_of_passes,
@@ -740,7 +765,6 @@ class StationTrace(Trace):
                 self.set_provenance(
                     "bandpass_filter",
                     {
-                        "bandpass_filter": {"code": "bp", "label": "Bandpass Filter"},
                         "filter_type": "Butterworth gmrocess",
                         "filter_order": corners,
                         "number_of_passes": number_of_passes,
@@ -753,7 +777,6 @@ class StationTrace(Trace):
                 self.set_provenance(
                     "bandstop_filter",
                     {
-                        "bandstop_filter": {"code": "bs", "label": "Bandstop Filter"},
                         "filter_type": "Butterworth ObsPy",
                         "filter_order": corners,
                         "number_of_passes": number_of_passes,
@@ -786,7 +809,6 @@ class StationTrace(Trace):
                 self.set_provenance(
                     "bandstop_filter",
                     {
-                        "bandstop_filter": {"code": "bs", "label": "Bandstop Filter"},
                         "filter_type": "Butterworth gmprocess",
                         "filter_order": corners,
                         "number_of_passes": number_of_passes,

{gmprocess-2.2.0 → gmprocess-2.3.0}/src/gmprocess/core/streamcollection.py

@@ -90,7 +90,6 @@ class StreamCollection(StreamArray):
             if not isinstance(st, StationStream):
                 raise TypeError("streams must be a list of StationStream objects.")
 
-            logging.debug(st.get_id())
             st.id = st.get_id()
             st.use_array = False
 
@@ -98,7 +97,7 @@ class StreamCollection(StreamArray):
                 if st[0].free_field:
                     newstreams.append(st)
                 else:
-                    logging.debug(
+                    logging.info(
                         "Omitting station trace %s from stream collection "
                         "because it is not free field.",
                         st[0].id,

{gmprocess-2.2.0 → gmprocess-2.3.0}/src/gmprocess/data/config_production.yml

@@ -5,8 +5,12 @@ fetchers:
         enabled: True
         duration:
             # Duration is computed as c0 + c1 * magnitude, in minutes.
-            c0: 0.0
+            c0: 4.5
             c1: 0.5
+            # Optionally, adjust values based on tectonic environment
+            Stable:
+                c0: 5.5
+                c1: 0.5
         distance:
             # The search radius uses a ground motion model, as specified in the 
             # 'gmm_selection' section. If STREC is enabled, it will select the model
@@ -94,7 +98,7 @@ fetchers:
             # SY is a network for synthetic data
             # 'XO' is an assortment of temporary deployment networks for Alaska region
             # Other networks listed here are network codes known to be problematic.
-            exclude_networks: ['SY', 'ZY', 'YH', 'YG', 'AM', 'PB', 'XO']
+            exclude_networks: ['SY', 'ZY', 'YH', 'YG', 'AM', 'PB', 'XO', '7D']
 
             # The minimum length of the data as a fraction of the requested time frame.
             # After a channel has been downloaded it will be checked that its total
@@ -348,26 +352,49 @@ windows:
     signal_end:
         # Options for setting the end of the signal window. Options are set with the key
         # "method":
-        #    - "velocity": Set the end of the signal can be set using a phase velocity;
-        #      this option makes use of the keys "vmin" and "floor".
         #    - "model": Set the end of the signal window using a shaking duration model;
-        #      this options makes use of the keys "model" and "epsilon"
+        #      this options makes use of the keys "model" and "epsilon". Duration is 
+        #      relative to split time.
+        #    - "source_path": Compute the end of the signal using
+        #      d0 + source_duration + epi_dist
+        #      Duration is relative to split time.
         #    - "magnitude": Use the magnitude-based signal end as defined by CISN, which
-        #      is magnitude/2 (units of minutes).
+        #      is magnitude/2 (units of minutes). Duration is relative to origin time.
+        #    - "velocity": Set the end of the signal can be set using a phase velocity;
+        #      this option makes use of the keys "vmin" and "floor". Duration is 
+        #      relative to origin time.
         #    - "none": Use the full available record; this is useful if the records
         #      have already been trimmed to a resonable level and you do not wish to
         #      further reduce trace duration.
 
-        method: model
-        vmin: 1.0
-        # Minimum duration in sec for use with 'vmin' option.
-        floor: 120.0
+        method: source_path
+
         # Duration model
         model: AS16
         # Number of standard deviations; if epsilon is 1.0, then the signal
         # window duration is the mean Ds + 1 standard deviation.
         epsilon: 3.0
 
+        # "velocity" method
+        vmin: 1.0
+        floor: 120.0
+
+        # Source-path model; stress_drop (bars) for source duration.
+        stress_drop: 10.0
+        dur0: 150.0
+        dur1: 0.6
+
+        Regions:
+            Stable:
+                method: source_path
+                model: AS16
+                epsilon: 3.0
+                vmin: 1.0
+                floor: 120.0
+                stress_drop: 100.0
+                dur0: 150.0
+                dur1: 0.6
+
     window_checks:
         # Minimum noise duration; can be zero but this will allow for errors
         # to occur if requesting signal-to-noise ratios.
@@ -408,41 +435,37 @@ processing:
         min_crossings: 0.1
 
     - remove_response:
+        pre_filt: False
+        output_as_acceleration: False
+        custom_pre_filt:
+            lp_factor: 0.8
+            hp_freq: 0.005
+            corners: 40
 
-    - detrend:
-        detrending_method: linear
+    - cut:
 
-    - detrend:
-        detrending_method: demean
+    - taper:
+
+    - convert_to_acceleration:
+        taper: False
 
     - compute_snr:
 
     - snr_check:
 
     - get_corner_frequencies:
-        method: snr
-        snr:
-            same_horiz: False
 
     - lowpass_max_frequency:
 
-    - cut:
-
-    - taper:
-
     - ridder_fchp:
 
-    - highpass_filter:
+    - bandpass_filter:
 
-    - lowpass_filter:
-        
     - detrend:    
         detrending_method: pre
 
     - fit_spectra:
 
-    - check_tail:
-
 colocated:
     # Enable the colocation algorithm?
     enabled: True
@@ -530,8 +553,8 @@ metrics:
   # Do *not* delete a key to remove a component, it will be replaced with the default.
   # To remove a component, set it's types as an empty list.
   components_and_types: 
-    channels: [pga, pgv, sa, duration, cav, arias]
-    rotd: [pga, pgv, sa]
+    channels: [pga, pgv, psa, duration, cav, arias]
+    rotd: [pga, pgv, psa]
     geometric_mean: [duration]
     quadratic_mean: [fas]