gmprocess 1.3.2__tar.gz → 2.0.0__tar.gz

{gmprocess-1.3.2 → gmprocess-2.0.0}/MANIFEST.in

@@ -1,5 +1,5 @@
 recursive-include src/gmprocess/data *
+recursive-include src/gmprocess/bin *
 prune tests
-prune doc_source
+prune docs
 prune notebooks
-prune src/gmprocess/data/asdf

{gmprocess-1.3.2/src/gmprocess.egg-info → gmprocess-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gmprocess
-Version: 1.3.2
+Version: 2.0.0
 Summary: USGS Automated Ground Motion Processing Software
 Author-email: Eric Thompson <emthompsone@usgs.gov>, Mike Hearne <mhearne@usgs.gov>, Brad Aagaard <baagaard@usgs.gov>, Bruce Worden <cbworden@contractor.usgs.gov>, John Rekoske <jrekoske@ucsd.edu>, Heather Hunsinger <hhunsinger@usgs.gov>, Gabe Ferragut <gferragut@usgs.gov>
 License: License
@@ -48,7 +48,7 @@ License: License
         
         [1]: https://creativecommons.org/publicdomain/zero/1.0/legalcode
         
-Requires-Python: <3.11,>=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: beautifulsoup4>=4.11.0
@@ -62,10 +62,11 @@ Requires-Dist: gmpacket>=0.1.6
 Requires-Dist: h5py>=2.8
 Requires-Dist: lxml>=4.6.1
 Requires-Dist: matplotlib>=3.1.0
+Requires-Dist: numba>=0.58
 Requires-Dist: numpy<2.0,>=1.21
 Requires-Dist: obspy>=1.4.0
 Requires-Dist: openpyxl>=3.0.8
-Requires-Dist: openquake.engine>=3.14
+Requires-Dist: openquake.engine>=3.18
 Requires-Dist: pandas>=1.0
 Requires-Dist: ps2ff>=1.5.2
 Requires-Dist: pyasdf>=0.7
@@ -73,7 +74,7 @@ Requires-Dist: pydantic>=2.0
 Requires-Dist: requests>=2.29
 Requires-Dist: ruamel.yaml>=0.17.16
 Requires-Dist: schema>=0.7
-Requires-Dist: scipy>=1.7
+Requires-Dist: scipy<1.13,>=1.7
 Requires-Dist: setuptools-scm>=6.3.2
 Requires-Dist: statsmodels>=0.12.2
 Requires-Dist: usgs-strec>=2.3.2
@@ -114,8 +115,9 @@ Most of the extensions that we provide are to handle strong motion data and rela
 
 # Documentation
 - Please note, we are in the process of improving the documentation and that there are some incomplete sections.
-- The full documentation can be viewed locally by running the `doc_source/makedocs.sh` script.
+- The full documentation can be viewed locally by running the `docs/makedocs.sh` script.
 - The documentation is also available [here](https://ghsc.code-pages.usgs.gov/esi/groundmotion-processing/).
+- The MATLAB functions included in the `tools/matlab` directory include a short [tutorial](tools/matlab/readme.md)
 
 # Getting Help
 The developers can be reached at [gmprocess@usgs.gov](mailto:gmprocess@usgs.gov).
@@ -125,8 +127,12 @@ If you would like to post an issue to ask a question, request a new feature, or
 If you would like to contribute merge requests to this repository, please let us know and we can add you as a collaborator. 
 This will require that you create an account at code.usgs.gov, which is a more involved process than creating a personal account at gitub.com and posting to the issues on the github mirror.
 
-# Reference
-If you wish to cite this software, please use this reference:
+# Suggested Citation
+If you wish to cite this software please cite:
 
-- Hearne, M., E. M. Thompson, H. Schovanec, J. Rekoske, B. T. Aagaard, and C. B. Worden (2019). USGS automated ground motion processing software, USGS Software Release, doi: [10.5066/P9ANQXN3](https://dx.doi.org/10.5066/P9ANQXN3).
+- Thompson, E.M., M. Hearne, B.T. Aagaard, J.M. Rekoske, C.B. Worden, M.P. Moschetti, H.E. Hunsinger, G.C. Ferragut, G.A. Parker, J.A. Smith, K.K. Smith, and A.R. Kottke (2024). USGS Automated Ground Motion Processing Software Version 2, U.S. Geological Survey software release. [https://doi.org/10.5066/P13HMKFJ](https://doi.org/10.5066/P13HMKFJ)
   
+
+# Disclaimer
+
+Any use of trade, firm, or product names is for descriptive purposes only and does not imply endorsement by the U.S. Government.

{gmprocess-1.3.2 → gmprocess-2.0.0}/README.md

@@ -5,8 +5,9 @@ Most of the extensions that we provide are to handle strong motion data and rela
 
 # Documentation
 - Please note, we are in the process of improving the documentation and that there are some incomplete sections.
-- The full documentation can be viewed locally by running the `doc_source/makedocs.sh` script.
+- The full documentation can be viewed locally by running the `docs/makedocs.sh` script.
 - The documentation is also available [here](https://ghsc.code-pages.usgs.gov/esi/groundmotion-processing/).
+- The MATLAB functions included in the `tools/matlab` directory include a short [tutorial](tools/matlab/readme.md)
 
 # Getting Help
 The developers can be reached at [gmprocess@usgs.gov](mailto:gmprocess@usgs.gov).
@@ -16,8 +17,12 @@ If you would like to post an issue to ask a question, request a new feature, or
 If you would like to contribute merge requests to this repository, please let us know and we can add you as a collaborator. 
 This will require that you create an account at code.usgs.gov, which is a more involved process than creating a personal account at gitub.com and posting to the issues on the github mirror.
 
-# Reference
-If you wish to cite this software, please use this reference:
+# Suggested Citation
+If you wish to cite this software please cite:
 
-- Hearne, M., E. M. Thompson, H. Schovanec, J. Rekoske, B. T. Aagaard, and C. B. Worden (2019). USGS automated ground motion processing software, USGS Software Release, doi: [10.5066/P9ANQXN3](https://dx.doi.org/10.5066/P9ANQXN3).
-  
+- Thompson, E.M., M. Hearne, B.T. Aagaard, J.M. Rekoske, C.B. Worden, M.P. Moschetti, H.E. Hunsinger, G.C. Ferragut, G.A. Parker, J.A. Smith, K.K. Smith, and A.R. Kottke (2024). USGS Automated Ground Motion Processing Software Version 2, U.S. Geological Survey software release. [https://doi.org/10.5066/P13HMKFJ](https://doi.org/10.5066/P13HMKFJ)
+  
+
+# Disclaimer
+
+Any use of trade, firm, or product names is for descriptive purposes only and does not imply endorsement by the U.S. Government.

{gmprocess-1.3.2 → gmprocess-2.0.0}/pyproject.toml

@@ -13,7 +13,7 @@ authors = [
 ]
 license = {file = "LICENSE.md"}
 readme = "README.md"
-requires-python = ">=3.8,<3.11"
+requires-python = ">=3.9"
 
 # Requirements:
 dependencies = [
@@ -28,10 +28,11 @@ dependencies = [
     "h5py>=2.8",
     "lxml>=4.6.1",
     "matplotlib>=3.1.0",
+    "numba>=0.58",
     "numpy>=1.21,<2.0",
     "obspy>=1.4.0",
     "openpyxl>=3.0.8",
-    "openquake.engine>=3.14",
+    "openquake.engine>=3.18",
     "pandas>=1.0",
     "ps2ff>=1.5.2",
     "pyasdf>=0.7",
@@ -39,7 +40,7 @@ dependencies = [
     "requests>=2.29",
     "ruamel.yaml>=0.17.16",
     "schema>=0.7",
-    "scipy>=1.7",
+    "scipy>=1.7,<1.13",
     "setuptools-scm>=6.3.2",
     "statsmodels>=0.12.2",
     "usgs-strec>=2.3.2",
@@ -87,8 +88,9 @@ gmconvert = "gmprocess.bin.gmconvert:main"
 gminfo = "gmprocess.bin.gminfo:main"
 gmrecords = "gmprocess.bin.gmrecords:main"
 gmworkspace = "gmprocess.bin.gmworkspace:main"
+gmprocess_config = "gmprocess.bin.gmprocess_config:main"
 list_metrics = "gmprocess.bin.list_metrics:main"
-cwb_gather = "gmprocess.bin.cwb_gather:main"
+cwa_gather = "gmprocess.bin.cwa_gather:main"
 fix_inventory = "gmprocess.bin.fix_inventory:main"
 
 [build-system]

{gmprocess-1.3.2 → gmprocess-2.0.0}/src/gmprocess/apps/gmrecords.py

@@ -279,16 +279,17 @@ class GMrecordsApp(object):
             "--log",
             type=str,
             default=None,
-            help="Path to log file; if provided, loging is directed to this file.",
+            help="Path to log file; if provided, logging is directed to this file.",
         )
 
         parser.add_argument(
             "-e",
             "--eventid",
             type=str,
+            dest="event_id",
             default=None,
             help=(
-                "Comcat event ID. If None (default) all events in project data "
+                "ComCat event ID. If None (default) all events in project data "
                 "directory will be used. To specify multiple eventids, use a comma "
                 "separated list like "
                 '`gmrecords -e "nc73799091, nc73774300" autoprocess`'
@@ -304,7 +305,7 @@ class GMrecordsApp(object):
                 "A CSV file without column headers. The columns can be either: "
                 "(1) a single column with ComCat event IDs, or "
                 "(2) six columns in which those columns are: "
-                "id (string, no spaces), time (any ISO standard for date/time), "
+                "event_id (string, no spaces), time (any ISO standard for date/time), "
                 "latitutde (float, decimal degrees), longitude (float, decimal "
                 "degrees), depth (float, km), magnitude (float)."
             ),

gmprocess-2.0.0/src/gmprocess/bin/cwa_gather.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+
+# stdlib imports
+import os
+import argparse
+import sys
+import pathlib
+
+# third party imports
+import numpy as np
+from obspy import read, read_inventory
+import configobj
+
+# local imports
+from gmprocess.core import scalar_event
+from gmprocess.utils import constants
+from gmprocess.utils import download_utils
+from gmprocess.core.scalar_event import ScalarEvent
+
+
+def main():
+    desc = """Convert CWA data from web site into form ingestible by gmprocess.
+
+    To obtain CWA strong motion data, create an account on the CWA:
+
+    https://gdms.cwa.gov.tw/
+
+
+    Step 1: CWBSN metadata
+    ----------------------
+
+    Click on the "Data" icon, then click on "Instrument Response". 
+
+     - For "Output Format", choose "Station XML".
+     - For "Network", choose "CWBSN".
+     - For "Station", check "All Stations".
+     - For "Location", choose "*" for all locations.
+     - For "Channel", choose "HN?" for all strong motion stations.
+     - For "Start Time (UTC)", enter a date before the event of interest.
+     - For "End Time (UTC)", enter a date after the event of interest.
+     - "Label", doesn't matter becasue they don't use what you put here.
+     - Click the "Submit" button, and you should see a "Success!" pop up.
+     - Next will be a screen showing a list of all of your downloads.
+     - Right click on the most recent entry, it will be called "output.stationXML"
+     - Select "save link as" and name it <eventid>_cwbsn.xml
+     
+    Step 2: TSMIP metadata
+    ----------------------
+
+    Click on the "Data" icon, then click on "Instrument Response". 
+
+     - For "Output Format", choose "Station XML".
+     - For "Network", choose "TSMIP".
+     - For "Station", check "All Stations".
+     - For "Location", choose "*" for all locations.
+     - For "Channel", choose "HL?" for all strong motion stations.
+     - For "Start Time (UTC)", enter a date before the event of interest.
+     - For "End Time (UTC)", enter a date after the event of interest.
+     - "Label", doesn't matter becasue they don't use what you put here.
+     - Click the "Submit" button, and you should see a "Success!" pop up.
+     - Next will be a screen showing a list of all of your downloads.
+     - Right click on the most recent entry, it will be called "output.stationXML"
+     - Select "save link as" and name it <eventid>_tsmip.xml
+
+    Step 3: CWBSN ground motions
+    ----------------------------
+
+    Click on the "Data" icon, then click on "Multi-Station Waveform Data". 
+
+     - For "Output Format", choose "MiniSEED".
+     - For "Network", choose "CWBSN".
+     - For "Station", check "All Stations".
+     - For "Location", choose "*" for all locations.
+     - For "Channel", choose "HN?" for all strong motion stations.
+     - For "Start Time (UTC)", enter a time 30 seconds before the origin time of 
+       interest.
+     - For "End Time (UTC)", enter a time 4 minutes after the origin time of interest.
+       Note that this can be adjusted up for larger earthquakes or down for smaller.
+     - For "Label" use <eventid>_cwbsn_data
+     - Click the "Submit" button, and you should see a "Success!" pop up.
+     - Next will be a screen showing a list of all of your downloads. Data links will
+       take a few minutes to process.
+     - Download the file, which will be named <eventid>_cwbsn_data
+
+    Step 4: TSMIP ground motions
+    ----------------------------
+
+    Click on the "Data" icon, then click on "Multi-Station Waveform Data". 
+
+     - For "Output Format", choose "MiniSEED".
+     - For "Network", choose "TSMIP".
+     - For "Station", check "All Stations".
+     - For "Location", choose "*" for all locations.
+     - For "Channel", choose "HN?" for all strong motion stations.
+     - For "Start Time (UTC)", enter a time 30 seconds before the origin time of 
+       interest.
+     - For "End Time (UTC)", enter a time 4 minutes after the origin time of interest.
+       Note that this can be adjusted up for larger earthquakes or down for smaller.
+     - For "Label" use <eventid>_tsmip_data
+     - Click the "Submit" button, and you should see a "Success!" pop up.
+     - Next will be a screen showing a list of all of your downloads. Data links will
+       take a few minutes to process.
+     - Download the file, which will be named <eventid>_tsmip_data
+     
+
+    You will need to run the cwa_gather command twice, once for each pair of XML and
+    tgz files for the CWBSN and TSMIP networks.
+    """
+    parser = argparse.ArgumentParser(
+        description=desc,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--event",
+        dest="event_id",
+        help="ComCat Event ID",
+    )
+    parser.add_argument(
+        "datafile",
+        help="The <eventid>_*_data.mseed file that you downloaded.",
+    )
+    parser.add_argument(
+        "metadata",
+        help="StationXML file that you downloaded.",
+    )
+    if len(sys.argv) == 1:
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+    args = parser.parse_args()
+
+    if "CALLED_FROM_PYTEST" in os.environ:
+        CONF_PATH = pathlib.Path(".") / ".gmprocess"
+    else:
+        CONF_PATH = pathlib.Path.home() / ".gmprocess"
+    PROJECTS_FILE = CONF_PATH / "projects.conf"
+    projects_conf = configobj.ConfigObj(str(PROJECTS_FILE), encoding="utf-8")
+    project = projects_conf["project"]
+    current_project = projects_conf["projects"][project]
+    data_parts = pathlib.PurePath(current_project["data_path"]).parts
+    data_path = CONF_PATH.joinpath(*data_parts).resolve()
+    event_path = data_path / args.event_id
+    raw_path = event_path / "raw"
+    if not event_path.exists():
+        event_path.mkdir()
+        raw_path.mkdir()
+    raw_stream = read(args.datafile)
+    # some of these files are returned from CWB web site in time chunks
+    # using this merge method joins up all of the traces with the same
+    # NSCL. Thanks Obspy!
+    stream = raw_stream.merge()
+    trace_count = 0
+    for trace in stream:
+        if isinstance(trace.data, np.ma.core.MaskedArray):
+            continue
+        network = trace.stats.network
+        station = trace.stats.station
+        channel = trace.stats.channel
+        location = trace.stats.location
+        starttime_str = trace.stats.starttime.strftime("%Y%m%dT%H%M%SZ")
+        endtime_str = trace.stats.endtime.strftime("%Y%m%dT%H%M%SZ")
+        fname = (
+            f"{network}.{station}.{location}.{channel}__"
+            f"{starttime_str}__{endtime_str}.mseed"
+        )
+        filename = raw_path / fname
+        trace.write(str(filename), format="MSEED")
+        trace_count += 1
+    print(f"{len(stream)} channels written to {raw_path}.")
+
+    inventory = read_inventory(args.metadata)
+    network = inventory.networks[0]
+    netcode = network.code
+    for station in network.stations:
+        stacode = station.code
+        xml_name = f"{netcode}.{stacode}.xml"
+        sta_inv = inventory.select(netcode, stacode)
+        # Correct HL channel codes to be HN
+        for net in sta_inv.networks:
+            for sta in net.stations:
+                for chan in sta.channels:
+                    if chan.code.startswith("HL"):
+                        chan.code = chan.code[0] + "N" + chan.code[2]
+        sta_inv.write(raw_path / xml_name, "STATIONXML")
+    print(f"{len(network.stations)} StationXML files written to {raw_path}.")
+
+    event_info = download_utils.download_comcat_event(args.event_id)
+    scalar_event.write_geojson(event_info, event_path)
+    event_file = event_path / constants.EVENT_FILE
+    msg = f"Created event file at {event_file}."
+    if not event_file.is_file():
+        msg = f"Error: Failed to create {event_file}."
+    print(msg)
+    download_utils.download_rupture_file(args.event_id, event_path)
+    rupture_file = event_path / "rupture.json"
+    msg = f"Created rupture file at {rupture_file}."
+    if not rupture_file.is_file():
+        msg = f"Error: Failed to create {rupture_file}."
+    print(msg)
+
+    event = ScalarEvent.from_json(event_file)
+    download_utils.get_strec_results(event, event_path)
+    msg = "Downloaded STREC results."
+    strec_file = event_path / "strec_results.json"
+    if not strec_file.is_file():
+        msg = "Failed to download STREC results."
+    print(msg)
+
+
+if __name__ == "__main__":
+    main()

{gmprocess-1.3.2 → gmprocess-2.0.0}/src/gmprocess/bin/fix_inventory.py

@@ -112,7 +112,7 @@ class FixInventory:
         ds = pyasdf.ASDFDataSet(self.file)
         for i, waveform in enumerate(ds.waveforms):
             tags = waveform.get_waveform_tags()
-            for tag in tags:
+            for _ in tags:
                 for net in self.inventories[i]:
                     for sta in net.stations:
                         sta.description = ""

gmprocess-2.0.0/src/gmprocess/bin/gmprocess_config.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+import argparse
+from pathlib import Path
+
+from ruamel.yaml import YAML
+
+from gmprocess.io.asdf.stream_workspace import StreamWorkspace
+
+
+def save_config(ws, out_file):
+    group_name = "config/config"
+    config_exists = group_name in ws.dataset._auxiliary_data_group
+    if config_exists:
+        fname = Path(out_file)
+        yaml = YAML()
+        yaml.indent(mapping=4)
+        yaml.preserve_quotes = True
+        with open(fname, "a", encoding="utf-8") as yf:
+            yaml.dump(ws.config, yf)
+        print("gmprocess_config:")
+        print(f"  Input file: {ws.filename}")
+        print(f"  Output file: {fname}")
+    else:
+        print("No config in workspace.")
+
+
+def update_config(ws, in_file):
+    in_config_path = Path(in_file)
+    with open(in_config_path, "r", encoding="utf-8") as f:
+        yaml = YAML()
+        yaml.preserve_quotes = True
+        in_config = yaml.load(f)
+
+    ws.add_config(config=in_config, force=True)
+    print("gmprocess_config:")
+    print(f"  Input file: {in_file}")
+    print(f"  Updated config: {ws.filename}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Help to fix confs in workspace files.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--workspace",
+        required=True,
+        help="Path to workspace file.",
+        type=str,
+    )
+    io_group = parser.add_mutually_exclusive_group(required=True)
+    io_group.add_argument(
+        "--save",
+        help="Path to file to save contents of workspace config file.",
+        type=str,
+    )
+    io_group.add_argument(
+        "--update",
+        help="Path to config file to overwrite in workspace.",
+        type=str,
+    )
+    args = parser.parse_args()
+    ws = StreamWorkspace.open(args.workspace)
+    if args.save is not None:
+        save_config(ws, args.save)
+    if args.update is not None:
+        update_config(ws, args.update)
+    ws.close()
+
+
+if __name__ == "__main__":
+    main()

gmprocess-2.0.0/src/gmprocess/bin/update_event_json.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+for efile in `find . -name event.json`; do
+    sed -e 's/"lat":/"latitude":/g; s/"lon":/"longitude":/g; s/"depth":/"depth_km":/g; s/"mag_type":/"magnitude_type":/g' $efile > tmp && mv tmp $efile
+done

{gmprocess-1.3.2 → gmprocess-2.0.0}/src/gmprocess/core/provenance.py

@@ -12,6 +12,7 @@ From SEIS-PROV docs regarding IDs:
 Note that it is important to recognize the difference between the above ID and the
 "prov:id" provenance property, which corresponds to the keys in Provenance.ACTIVITIES.
 """
+
 from abc import ABC, abstractmethod
 import logging
 from datetime import datetime
@@ -118,7 +119,7 @@ class LabelProvenance(Provenance):
         self.provenance_list.append(prov_dict)
 
     @classmethod
-    def from_provenance_document(cls, provdoc, label):
+    def from_provenance_document(cls, provdoc, label, config):
         """Add provenance entires from a provenance document.
 
         Args:
@@ -126,8 +127,10 @@ class LabelProvenance(Provenance):
                 Provenance document.
             label (str):
                 Processing label.
+            config (dict):
+                Dictionary of config options.
         """
-        prov_obj = cls(label)
+        prov_obj = cls(label, config=config)
         prov_obj._from_provenance_document(provdoc)
         return prov_obj