mt-metadata 0.3.5__py2.py3-none-any.whl → 0.3.7__py2.py3-none-any.whl

mt_metadata/__init__.py

@@ -39,7 +39,7 @@ you should only have to changes these dictionaries.
 
 __author__ = """Jared Peacock"""
 __email__ = "jpeacock@usgs.gov"
-__version__ = "0.3.5"
+__version__ = "0.3.7"
 
 # =============================================================================
 # Imports

mt_metadata/base/helpers.py

@@ -2,7 +2,7 @@
 """
 Created on Wed Dec 23 20:37:52 2020
 
-:copyright: 
+:copyright:
     Jared Peacock (jpeacock@usgs.gov)
 
 :license: MIT
@@ -637,11 +637,18 @@ def element_to_string(element):
 # Helper function to be sure everything is encoded properly
 # =============================================================================
 class NumpyEncoder(json.JSONEncoder):
+
     """
     Need to encode numpy ints and floats for json to work
     """
 
     def default(self, obj):
+        """
+
+        :param obj:
+        :type obj:
+        :return:
+        """
         if isinstance(
             obj,
             (
@@ -659,7 +666,7 @@ class NumpyEncoder(json.JSONEncoder):
             ),
         ):
             return int(obj)
-        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
+        elif isinstance(obj, (np.float16, np.float32, np.float64)):
             return float(obj)
         elif isinstance(obj, (np.ndarray)):
             if obj.dtype == complex:

mt_metadata/timeseries/filters/filtered.py

@@ -17,6 +17,7 @@ from mt_metadata.base.helpers import write_lines
 from mt_metadata.base import get_schema, Base
 from mt_metadata.timeseries.standards import SCHEMA_FN_PATHS
 from mt_metadata.utils.exceptions import MTSchemaError
+from typing import Optional, Union
 
 # =============================================================================
 attr_dict = get_schema("filtered", SCHEMA_FN_PATHS)
@@ -31,6 +32,14 @@ class Filtered(Base):
     __doc__ = write_lines(attr_dict)
 
     def __init__(self, **kwargs):
+        """
+        Constructor
+
+        :param kwargs:
+
+        TODO: Consider not setting self.applied = None, as this has the effect of self._applied = [True,]
+        """
+        self._applied_values_map = _applied_values_map()
         self._name = []
         self._applied = []
         self.name = None
@@ -53,7 +62,7 @@ class Filtered(Base):
         elif isinstance(names, list):
             self._name = [ss.strip().lower() for ss in names]
         elif isinstance(names, np.ndarray):
-            names = names.astype(np.unicode_)
+            names = names.astype(np.str_)
             self._name = [ss.strip().lower() for ss in names]
         else:
             msg = "names must be a string or list of strings not {0}, type {1}"
@@ -68,25 +77,52 @@ class Filtered(Base):
             self.logger.warning(msg)
 
     @property
-    def applied(self):
+    def applied(self) -> list:
         return self._applied
 
     @applied.setter
-    def applied(self, applied):
+    def applied(
+        self,
+        applied: Union[list, str, None, int, tuple, np.ndarray, bool],
+    ) -> None:
+        """
+        Sets the value of the booleans for whether each filter has been applied or not
+
+        :type applied: Union[list, str, None, int, tuple]
+        :param applied: The value to set self._applied.
+
+        Notes:
+        self._applied is a list, but we allow this to be assigned by single values as well,
+        such as None, True, 0. Supporting these other values makes the logic a little bit involved.
+        If a null value is received, the filters are assumed to be applied.
+        If a simple value, such as True, None, 0, etc. is not received, the input argument
+        applied (which is iterable) is first converted to `applied_list`.
+        The values in `applied_list` are then mapped to booleans.
+
+
+        """
+        # Handle cases where we did not pass an iterable
         if not hasattr(applied, "__iter__"):
-            if applied in [None, "none", "None", "NONE", "null"]:
-                self._applied = [True]
-                return
-            elif applied in [0, "0"]:
-                self._applied = [False]
-                return
+            self._applied = [self._applied_values_map[applied], ]
+            return
+
+        # the returned type from a hdf5 dataset is a numpy array.
+        if isinstance(applied, np.ndarray):
+            applied = applied.tolist()
 
         #sets an empty list to one default value
         if isinstance(applied, list) and len(applied) == 0:
-            self.applied = [True]
+            self._applied = [True]
             return
 
+        # Handle string case
         if isinstance(applied, str):
+            # Handle simple strings
+            if applied in self._applied_values_map.keys():
+                self._applied = [self._applied_values_map[applied], ]
+                return
+
+            # Handle string-lists (e.g. from json)
             if applied.find("[") >= 0:
                 applied = applied.replace("[", "").replace("]", "")
             if applied.count(",") > 0:
@@ -97,44 +133,20 @@ class Filtered(Base):
                 applied_list = [ss.lower() for ss in applied.split()]
         elif isinstance(applied, list):
             applied_list = applied
-            # set integer strings to integers ["0","1"]--> [0, 1]
-            for i, elt in enumerate(applied_list):
-                if elt in ["0", "1",]:
-                    applied_list[i] = int(applied_list[i])
-            # set integers to bools [0,1]--> [False, True]
-            for i, elt in enumerate(applied_list):
-                if elt in [0, 1,]:
-                    applied_list[i] = bool(applied_list[i])
-        elif isinstance(applied, bool):
-            applied_list = [applied]
-        # the returned type from a hdf5 dataset is a numpy array.
-        elif isinstance(applied, np.ndarray):
+        elif isinstance(applied, tuple):
             applied_list = list(applied)
-            if applied_list == []:
-                applied_list = [True]
         else:
-            msg = "applied must be a string or list of strings not {0}"
-            self.logger.error(msg.format(applied))
-            raise MTSchemaError(msg.format(applied))
-
-        bool_list = []
-        for app_bool in applied_list:
-            if app_bool is None:
-                bool_list.append(True)
-            elif isinstance(app_bool, str):
-                if app_bool.lower() in ["false", "0"]:
-                    bool_list.append(False)
-                elif app_bool.lower() in ["true", "1"]:
-                    bool_list.append(True)
-                else:
-                    msg = "Filter.applied must be [ True | False ], not {0}"
-                    self.logger.error(msg.format(app_bool))
-                    raise MTSchemaError(msg.format(app_bool))
-            elif isinstance(app_bool, (bool, np.bool_)):
-                bool_list.append(bool(app_bool))
-            else:
-                msg = "Filter.applied must be [True | False], not {0}"
-                self.logger.error(msg.format(app_bool))
+            msg = f"Input applied cannot be of type {type(applied)}"
+            self.logger.error(msg)
+            raise MTSchemaError(msg)
+
+        # Now we have a simple list -- map to bools
+        try:
+            bool_list = [self._applied_values_map[x] for x in applied_list]
+        except KeyError:
+            msg = f"A key in {applied_list} is not mapped to a boolean"
+            msg += "\n fix this by adding to _applied_values_map"
+            self.logger.error(msg)
         self._applied = bool_list
 
         # check for consistency
@@ -146,36 +158,82 @@ class Filtered(Base):
             self.logger.warning(msg)
 
 
-    def _check_consistency(self):
-        # check for consistency
-        if self._name != []:
-            if self._applied is None:
-                self.logger.warning("Need to input filter.applied")
-                return False
-            if len(self._name) == 1:
-                if len(self._applied) == 1:
-                    return True
-            elif len(self._name) > 1:
-                if len(self._applied) == 1:
-                    self.logger.debug(
-                        "Assuming all filters have been "
-                        + "applied as {0}".format(self._applied[0])
-                    )
-                    return True
-                elif len(self._applied) > 1:
-                    if len(self._applied) != len(self._name):
-                        self.logger.warning(
-                            "Applied and filter names "
-                            + "should be the same length. "
-                            + "Appied={0}, names={1}".format(
-                                len(self._applied), len(self._name)
-                            )
-                        )
-                        return False
-                    else:
-                        return True
-        elif self._name == [] and len(self._applied) > 0:
+    def _check_consistency(self) -> bool:
+        """
+        Logic to look for inconstencies in the configuration of the filter names and applied values.
+
+        In general, list of filter names should be same length as list of applied booleans.
+
+        Cases:
+        The filter has no name -- this could happen on intialization.
+
+        :return: bool
+	    True if OK, False if not.
+
+        """
+        # This inconsistency is ok -- the filter may not have been assigned a name yet
+        if self._name == [] and len(self._applied) > 0:
             self.logger.debug("Name probably not yet initialized -- skipping consitency check")
             return True
+
+        # Otherwise self._name != []
+
+        # Applied not assigned - this is not OK
+        if self._applied is None:
+            self.logger.warning("Need to input filter.applied")
+            return False
+
+        # Name and applied have same length, 1. This is OK
+        if len(self._name) == 1:
+            if len(self._applied) == 1:
+                return True
+
+        # Multiple filter names (name not of length 0 or 1)
+        if len(self._name) > 1:
+            # If only one applied boolean, we allow it.
+            # TODO: consider being less tolerant here
+            if len(self._applied) == 1:
+                msg = f"Assuming all filters have been applied as {self._applied[0]}"
+                self.logger.debug(msg)
+                self._applied = len(self.name) * [self._applied[0],]
+                msg = f"Explicitly set filter applied state to {self._applied[0]}"
+                self.logger.debug(msg)
+                return True
+            elif len(self._applied) > 1:
+                # need to check the lists are really the same length
+                if len(self._applied) != len(self._name):
+                    msg = "Applied and filter names should be the same length. "
+                    msg += f"Appied={len(self._applied)}, names={len(self._name)}"
+                    self.logger.warning(msg)
+                    return False
+                else:
+                    return True
         else:
+            # Some unknown configuration we have not yet encountered
+            msg = "Filter consistency check failed for an unknown reason"
+            self.logger.warning(msg)
             return False
+
+
+def _applied_values_map(
+    treat_null_values_as: Optional[bool] = True
+) -> dict:
+    """
+    helper function to simplify logic in applied setter.
+
+    Notes:
+    The logic in the setter was getting quite complicated handling many types.
+    A reasonable solution seemed to be to map each of the allowed values to a bool
+    via dict and then use this dict when setting applied values.
+
+    :return: dict
+    Mapping of all tolerated single-values for setting applied booleans
+    """
+    null_values = [None, "none", "None", "NONE", "null"]
+    null_values_map = {x: treat_null_values_as for x in null_values}
+    true_values = [True, 1, "1", "True", "true"]
+    true_values_map =  {x:True for x in true_values}
+    false_values = [False, 0, "0", "False", "false"]
+    false_values_map =  {x:False for x in false_values}
+    values_map = {**null_values_map, **true_values_map, **false_values_map}
+    return values_map

mt_metadata/timeseries/filters/frequency_response_table_filter.py

@@ -30,7 +30,6 @@ attr_dict.add_dict(
 # =============================================================================
 
 
-
 class FrequencyResponseTableFilter(FilterBase):
     """
     Phases should be in radians.
@@ -130,7 +129,6 @@ class FrequencyResponseTableFilter(FilterBase):
             self._empirical_phases = np.array(value, dtype=float)
 
             if self._empirical_phases.size > 0:
-
                 if self._empirical_phases.mean() > 1000 * np.pi / 2:
                     self.logger.warning(
                         "Phases appear to be in milli radians attempting to convert to radians"
@@ -216,12 +214,17 @@ class FrequencyResponseTableFilter(FilterBase):
         :rtype: np.ndarray
 
         """
-        if (
-            np.min(frequencies) < self.min_frequency
-            or np.max(frequencies) > self.max_frequency
-        ):
+        if np.min(frequencies) < self.min_frequency:
+            # if there is a dc component skip it.
+            if np.min(frequencies) != 0:
+                self.logger.warning(
+                    f"Extrapolating frequencies smaller ({np.min(frequencies)} Hz) "
+                    f"than table frequencies ({self.min_frequency} Hz)."
+                )
+        if np.max(frequencies) > self.max_frequency:
             self.logger.warning(
-                "Extrapolating, use values outside calibration frequencies with caution"
+                f"Extrapolating frequencies larger ({np.max(frequencies)} Hz) "
+                f"than table frequencies ({self.max_frequency} Hz)."
             )
 
         phase_response = interp1d(

mt_metadata/timeseries/station.py

@@ -59,6 +59,8 @@ attr_dict.add_dict(get_schema("copyright", SCHEMA_FN_PATHS), None)
 attr_dict["release_license"]["required"] = False
 attr_dict.add_dict(get_schema("citation", SCHEMA_FN_PATHS), None, keys=["doi"])
 attr_dict["doi"]["required"] = False
+
+
 # =============================================================================
 class Station(Base):
     __doc__ = write_lines(attr_dict)
@@ -316,3 +318,32 @@ class Station(Base):
                 else:
                     if self.time_period.end < max(end):
                         self.time_period.end = max(end)
+
+    def sort_runs_by_time(self, inplace=True, ascending=True):
+        """
+        return a list of runs sorted by start time in the order of ascending or
+        descending.
+
+        :param ascending: DESCRIPTION, defaults to True
+        :type ascending: TYPE, optional
+        :return: DESCRIPTION
+        :rtype: TYPE
+
+        """
+
+        run_ids = []
+        run_starts = []
+        for run_key, run_obj in self.runs.items():
+            run_ids.append(run_key)
+            run_starts.append(run_obj.time_period.start.split("+")[0])
+
+        index = np.argsort(np.array(run_starts, dtype=np.datetime64))
+
+        new_runs = ListDict()
+        for ii in index:
+            new_runs[run_ids[ii]] = self.runs[run_ids[ii]]
+
+        if inplace:
+            self.runs = new_runs
+        else:
+            return new_runs

mt_metadata/timeseries/stationxml/xml_channel_mt_channel.py

@@ -8,6 +8,8 @@ Created on Fri Feb 19 16:14:41 2021
 :license: MIT
 
 """
+import copy
+
 # =============================================================================
 # Imports
 # =============================================================================
@@ -24,6 +26,7 @@ from mt_metadata.timeseries.stationxml.utils import BaseTranslator
 from mt_metadata.utils.units import get_unit_object
 
 from obspy.core import inventory
+from obspy import UTCDateTime
 
 # =============================================================================
 
@@ -33,6 +36,15 @@ class XMLChannelMTChannel(BaseTranslator):
     translate back and forth between StationXML Channel and MT Channel
     """
 
+    understood_sensor_types = [
+        "logger",
+        "magnetometer",
+        "induction coil",
+        "coil",
+        "dipole",
+        "electrode"
+    ]
+
     def __init__(self):
         super().__init__()
 
@@ -109,7 +121,8 @@ class XMLChannelMTChannel(BaseTranslator):
         # fill channel filters
         mt_channel.filter.name = list(mt_filters.keys())
         mt_channel.filter.applied = [True] * len(list(mt_filters.keys()))
-
+        if UTCDateTime(mt_channel.time_period.end) < UTCDateTime(mt_channel.time_period.start):
+            mt_channel.time_period.end = '2200-01-01T00:00:00+00:00'
         return mt_channel, mt_filters
 
     def mt_to_xml(self, mt_channel, filters_dict, hard_code=True):
@@ -217,6 +230,8 @@ class XMLChannelMTChannel(BaseTranslator):
         :rtype: TYPE
 
         """
+        sensor.type = self._deduce_sensor_type(sensor)
+
         if not sensor.type:
             return mt_channel
 
@@ -566,3 +581,40 @@ class XMLChannelMTChannel(BaseTranslator):
         xml_channel.calibration_units_description = unit_obj.name
 
         return xml_channel
+
+
+    def _deduce_sensor_type(self, sensor):
+        """
+
+        :param sensor: Information about a sensor, usually extractes from FDSN XML
+        :type sensor: obspy.core.inventory.util.Equipment
+
+        :return:
+        """
+        original_sensor_type = sensor.type
+        # set sensor_type to be a string if it is None
+        if original_sensor_type is None:
+            sensor_type = ""  # make a string
+            msg = f"Sensor {sensor} does not have field type attr"
+            self.logger.debug(msg)
+        else:
+            sensor_type = copy.deepcopy(original_sensor_type)
+
+        if sensor_type.lower() in self.understood_sensor_types:
+            return sensor_type
+        else:
+            self.logger.warning(f" sensor {sensor} type {sensor.type} not in {self.understood_sensor_types}")
+
+        #  Try handling Bartington FGM at Earthscope ... this is a place holder for handling non-standard cases
+        if sensor.description == "Bartington 3-Axis Fluxgate Sensor":
+            sensor_type = "magnetometer"
+        elif sensor_type.lower() == "bartington":
+            sensor_type = "magnetometer"
+
+
+        # reset sensor_type to None it it was not handled
+        if not sensor_type:
+            sensor_type = original_sensor_type
+            self.logger.error("sensor type could not be resolved")
+
+        return sensor_type

mt_metadata/timeseries/stationxml/xml_inventory_mt_experiment.py

@@ -174,6 +174,7 @@ class XMLInventoryMTExperiment:
                     xml_station.site.country = ",".join(
                         [str(country) for country in mt_survey.country]
                     )
+                # need to sort the runs by time
                 for mt_run in mt_station.runs:
                     xml_station = self.add_run(
                         xml_station, mt_run, mt_survey.filters

mt_metadata/transfer_functions/__init__.py

@@ -1,3 +1,41 @@
+# Define allowed sets of channel labellings
+STANDARD_INPUT_CHANNELS = [
+    "hx",
+    "hy",
+]
+STANDARD_OUTPUT_CHANNELS = [
+    "ex",
+    "ey",
+    "hz",
+]
+
+CHANNEL_MAPS = {
+    "default": {"hx": "hx", "hy": "hy", "hz": "hz", "ex": "ex", "ey": "ey"},
+    "lemi12": {"hx": "bx", "hy": "by", "hz": "bz", "ex": "e1", "ey": "e2"},
+    "lemi34": {"hx": "bx", "hy": "by", "hz": "bz", "ex": "e3", "ey": "e4"},
+    "phoenix123": {"hx": "h1", "hy": "h2", "hz": "h3", "ex": "e1", "ey": "e2"},
+    "musgraves": {"hx": "bx", "hy": "by", "hz": "bz", "ex": "ex", "ey": "ey"},
+}
+
+
+def get_allowed_channel_names(standard_names):
+    """
+    :param standard_names: one of STANDARD_INPUT_NAMES, or STANDARD_OUTPUT_NAMES
+    :type standard_names: list
+    :return: allowed_names: list of channel names that are supported
+    :rtype: list
+    """
+    allowed_names = []
+    for ch in standard_names:
+        for _, channel_map in CHANNEL_MAPS.items():
+            allowed_names.append(channel_map[ch])
+    allowed_names = list(set(allowed_names))
+    return allowed_names
+
+
+ALLOWED_INPUT_CHANNELS = get_allowed_channel_names(STANDARD_INPUT_CHANNELS)
+ALLOWED_OUTPUT_CHANNELS = get_allowed_channel_names(STANDARD_OUTPUT_CHANNELS)
+
 from .core import TF
 
 __all__ = ["TF"]