mt-metadata 0.3.9__py2.py3-none-any.whl → 0.4.0__py2.py3-none-any.whl

mt_metadata/transfer_functions/processing/fourier_coefficients/decimation.py

@@ -1,45 +1,84 @@
 # -*- coding: utf-8 -*-
 """
-Created on Fri Feb 25 15:20:59 2022
+    This module contains the Decimation Metadata class.  This class interacts with a decimation JSON.
+    It contains the metadata to specify a transformation from time series to a Spectrogram, including
+    cascadng decimation info.
 
-@author: jpeacock
+    There are two main use cases for this class.  On the one hand, this can be used to specify a
+    set of processing parameters to create an FCDecimation, which can then be stored in an MTH5 archive.
+    On the other hand, this metadata gets stored along with Spectrograms in an MTH5 archive and can
+    be used to access the parameters associated with the spectrograms creation.
+
+    TODO: Consider renaming this class to FCDecmiation, to contrast with other Decimation objects,
+    or FCDecimationLevel to make it
+    Also see notes in mt_metadata issue 235.
+
+    Created on Fri Feb 25 15:20:59 2022
+
+    @author: jpeacock
 """
 # =============================================================================
 # Imports
 # =============================================================================
 from collections import OrderedDict
+from loguru import logger
+from typing import List, Optional
+
+from .standards import SCHEMA_FN_PATHS
 from mt_metadata.base.helpers import write_lines
 from mt_metadata.base import get_schema, Base
 from mt_metadata.timeseries import TimePeriod
-from mt_metadata.transfer_functions.processing.aurora import Window
-from mt_metadata.transfer_functions.processing.aurora.decimation_level import get_fft_harmonics
+from mt_metadata.transfer_functions.processing.short_time_fourier_transform import ShortTimeFourierTransform
+from mt_metadata.transfer_functions.processing.time_series_decimation import TimeSeriesDecimation
+# from mt_metadata.transfer_functions.processing.aurora.decimation_level import DecimationLevel as AuroraDecimationLevel
 from mt_metadata.transfer_functions.processing.fourier_coefficients import (
-    Channel,
+    Channel as FCChannel
 )
-from .standards import SCHEMA_FN_PATHS
 from mt_metadata.utils.list_dict import ListDict
 
+import numpy as np
 # =============================================================================
 attr_dict = get_schema("decimation", SCHEMA_FN_PATHS)
 attr_dict.add_dict(TimePeriod()._attr_dict, "time_period")
-attr_dict.add_dict(Window()._attr_dict, "window")
+attr_dict.add_dict(ShortTimeFourierTransform()._attr_dict, "short_time_fourier_transform")
+attr_dict.add_dict(TimeSeriesDecimation()._attr_dict, "time_series_decimation")
 
 # =============================================================================
 class Decimation(Base):
+    """
+        TODO: the name of this class could be changed to something more appropriate.
+        TODO: consider adding an attr decimation to access TimeSeriesDecimation more briefly.
+    """
     __doc__ = write_lines(attr_dict)
 
     def __init__(self, **kwargs):
-        self.window = Window()
+        """
+         Constructor.
+
+        :param kwargs: TODO: add doc here
+        """
         self.time_period = TimePeriod()
         self.channels = ListDict()
+        self.time_series_decimation = TimeSeriesDecimation()
+        self.short_time_fourier_transform = ShortTimeFourierTransform()
+
         super().__init__(attr_dict=attr_dict, **kwargs)
-        # if self.decimation_level == 0:
-        #     self.anti_alias_filter = None
 
-    def __len__(self):
+        if self.short_time_fourier_transform.per_window_detrend_type:
+            msg = f"per_window_detrend_type was set to {self.short_time_fourier_transform.per_window_detrend_type}"
+            msg += "however, this is not supported -- setting to empty string"
+            logger.debug(msg)
+            self.short_time_fourier_transform.per_window_detrend_type = ""
+
+    def __len__(self) -> int:
         return len(self.channels)
 
     def __add__(self, other):
+        """
+
+        :param other:
+        :return:
+        """
         if isinstance(other, Decimation):
             self.channels.extend(other.channels)
 
@@ -49,6 +88,21 @@ class Decimation(Base):
             self.logger.error(msg)
             raise TypeError(msg)
 
+    #----- Begin (Possibly Temporary) methods for integrating TimeSeriesDecimation, STFT Classes -----#
+
+    @property
+    def decimation(self) -> TimeSeriesDecimation:
+        """
+            Passthrough method to access self.time_series_decimation
+        """
+        return self.time_series_decimation
+
+    @property
+    def stft(self):
+        return self.short_time_fourier_transform
+
+    #----- End (Possibly Temporary) methods for integrating TimeSeriesDecimation, STFT Classes -----#
+
     def update(self, other, match=[]):
         """
         Update attribute values from another like element, skipping None
@@ -93,8 +147,8 @@ class Decimation(Base):
             self.add_channel(ch)
 
     @property
-    def channels_estimated(self):
-        """channels for fcs were estimated"""
+    def channels_estimated(self) -> list:
+        """channels for which fcs were estimated"""
         return [
             ch.component
             for ch in self.channels.values()
@@ -102,7 +156,7 @@ class Decimation(Base):
         ]
 
     @channels_estimated.setter
-    def channels_estimated(self, value):
+    def channels_estimated(self, value) -> None:
         """set channels esimated"""
 
         if value is None:
@@ -113,17 +167,17 @@ class Decimation(Base):
 
         for entry in value:
             if isinstance(entry, str):
-                self.add_channel(Channel(component=entry))
+                self.add_channel(FCChannel(component=entry))
             elif entry is None:
                 continue
-            elif isinstance(entry, Channel):
+            elif isinstance(entry, FCChannel):
                 self.add_channel(entry)
             else:
                 msg = f"entry must be a string or type FCChannel not {type(entry)}"
                 self.logger.error(msg)
                 raise ValueError(msg)
 
-    def has_channel(self, component):
+    def has_channel(self, component: str) -> bool:
         """
         Check to see if the channel already exists
 
@@ -144,15 +198,14 @@ class Decimation(Base):
         """
         if self.has_channel(component):
             return self.channels_estimated.index(component)
-        return None
 
-    def get_channel(self, component):
+    def get_channel(self, component: str) -> FCChannel:
         """
         Get a channel
 
         :param component: channel component to look for
         :type component: string
-        :return: channel object based on channel type
+        :return: FCChannel object based on channel type
         :rtype: :class:`mt_metadata.timeseries.Channel`
 
         """
@@ -160,7 +213,7 @@ class Decimation(Base):
         if self.has_channel(component):
             return self.channels[component]
 
-    def add_channel(self, channel_obj):
+    def add_channel(self, channel_obj: FCChannel) -> None:
         """
         Add a channel to the list, check if one exists if it does overwrite it
 
@@ -168,8 +221,8 @@ class Decimation(Base):
         :type channel_obj: :class:`mt_metadata.transfer_functions.processing.fourier_coefficients.Channel`
 
         """
-        if not isinstance(channel_obj, (Channel)):
-            msg = f"Input must be metadata.Channel not {type(channel_obj)}"
+        if not isinstance(channel_obj, (FCChannel)):
+            msg = f"Input must be metadata FCChannel not {type(channel_obj)}"
             self.logger.error(msg)
             raise ValueError(msg)
 
@@ -227,7 +280,7 @@ class Decimation(Base):
 
         for ii, channel in enumerate(value_list):
             try:
-                ch = Channel()
+                ch = FCChannel()
                 if hasattr(channel, "to_dict"):
                     channel = channel.to_dict()
                 ch.from_dict(channel)
@@ -268,26 +321,21 @@ class Decimation(Base):
                 if self.time_period.end < max(end):
                     self.time_period.end = max(end)
 
-    # Workarounds for pass-through usage of same decimation as aurora
-    @property
-    def factor(self):
-        return self.decimation_factor
 
-    @property
-    def sample_rate(self):
-        return self.sample_rate_decimation
+    def is_valid_for_time_series_length(self, n_samples_ts: int) -> bool:
+        """
+            Given a time series of len n_samples_ts, checks if there are sufficient samples to STFT.
 
-    def is_valid_for_time_series_length(self, n_samples_ts):
-        """Given a time series of len n_samples_ts, are there sufficient samples to STFT"""
+        """
         required_num_samples = (
-            self.window.num_samples
-            + (self.min_num_stft_windows - 1) * self.window.num_samples_advance
+            self.stft.window.num_samples
+            + (self.stft.min_num_stft_windows - 1) * self.stft.window.num_samples_advance
         )
         if n_samples_ts < required_num_samples:
             msg = (
                 f"{n_samples_ts} not enough samples for minimum of "
-                f"{self.min_num_stft_windows} stft windows of length "
-                f"{self.window.num_samples} and overlap {self.window.overlap}"
+                f"{self.stft.min_num_stft_windows} stft windows of length "
+                f"{self.stft.window.num_samples} and overlap {self.stft.window.overlap}"
             )
             self.logger.warning(msg)
             return False
@@ -295,158 +343,108 @@ class Decimation(Base):
             return True
 
     @property
-    def fft_frequencies(self):
-        return get_fft_harmonics(self.window.num_samples, self.sample_rate)
-
-
-    def has_fcs_for_aurora_processing(self, decimation_level, remote):
-        """
-
-        Parameters
-        ----------
-        decimation_level: mt_metadata.transfer_functions.processing.aurora.decimation_level.DecimationLevel
-        remote: bool
-
-        Iterates over parameters:
-        "channels_estimated", "anti_alias_filter", "sample_rate, "method", "prewhitening_type", "recoloring",
-        "pre_fft_detrend_type", "min_num_stft_windows", "window", "harmonic_indices",
-        Returns
-        -------
-
-        """
-        # "channels_estimated"
-        if remote:
-            required_channels = decimation_level.reference_channels
+    def fft_frequencies(self) -> np.ndarray:
+        """ Returns the one-sided fft frequencies (without Nyquist)"""
+        return self.stft.window.fft_harmonics(self.decimation.sample_rate)
+
+
+def fc_decimations_creator(
+    initial_sample_rate: float,
+    decimation_factors: Optional[list] = None,
+    max_levels: Optional[int] = 6,
+    time_period: Optional[TimePeriod] = None,
+) -> List[Decimation]:
+    """
+
+    Creates mt_metadata FCDecimation objects that parameterize Fourier coefficient decimation levels.
+
+    Note 1:  This does not yet work through the assignment of which bands to keep.  Refer to
+    mt_metadata.transfer_functions.processing.Processing.assign_bands() to see how this was done in the past
+
+    Parameters
+    ----------
+    initial_sample_rate: float
+        Sample rate of the "level0" data -- usually the sample rate during field acquisition.
+    decimation_factors: Optional[list]
+        The decimation factors that will be applied at each FC decimation level
+    max_levels: Optional[int]
+        The maximum number of decimation levels to allow
+    time_period: Optional[TimePeriod]
+        Provides the start and end times
+
+    Returns
+    -------
+    fc_decimations: list
+        Each element of the list is an object of type
+        mt_metadata.transfer_functions.processing.fourier_coefficients.Decimation,
+        (a.k.a. FCDecimation).
+
+        The order of the list corresponds the order of the cascading decimation
+          - No decimation levels are omitted.
+          - This could be changed in future by using a dict instead of a list,
+          - e.g. decimation_factors = dict(zip(np.arange(max_levels), decimation_factors))
+
+    """
+    if not decimation_factors:
+        # msg = "No decimation factors given, set default values to EMTF default values [1, 4, 4, 4, ..., 4]")
+        # logger.info(msg)
+        default_decimation_factor = 4
+        decimation_factors = max_levels * [default_decimation_factor]
+        decimation_factors[0] = 1
+
+    # See Note 1
+    fc_decimations = []
+    for i_dec_level, decimation_factor in enumerate(decimation_factors):
+        fc_dec = Decimation()
+        fc_dec.time_series_decimation.level = i_dec_level
+        fc_dec.id = f"{i_dec_level}"
+        fc_dec.time_series_decimation.factor = decimation_factor
+        if i_dec_level == 0:
+            current_sample_rate = 1.0 * initial_sample_rate
         else:
-            required_channels = decimation_level.local_channels
-
-        try:
-            assert set(required_channels).issubset(self.channels_estimated)
-        except AssertionError:
-            msg = (
-                f"required_channels for processing {required_channels} not available"
-                f"-- fc channels estimated are {self.channels_estimated}"
-            )
-            self.logger.info(msg)
-            return False
+            current_sample_rate /= decimation_factor
+        fc_dec.time_series_decimation.sample_rate = current_sample_rate
 
-        # anti_alias_filter
-        try:
-            assert self.anti_alias_filter == decimation_level.anti_alias_filter
-        except AssertionError:
-            cond1 = decimation_level.anti_alias_filter == "default"
-            cond2 = self.anti_alias_filter is None
-            if cond1 & cond2:
-                pass
+        if time_period:
+            if isinstance(time_period, TimePeriod):
+                fc_dec.time_period = time_period
             else:
                 msg = (
-                    "Antialias Filters Not Compatible -- need to add handling for "
-                    f"{msg} FCdec {self.anti_alias_filter} and "
-                    f"{msg} processing config:{decimation_level.anti_alias_filter}"
+                    f"Not sure how to assign time_period with type {type(time_period)}"
                 )
+                logger.info(msg)
                 raise NotImplementedError(msg)
 
-        # sample_rate
-        try:
-            assert (
-                self.sample_rate_decimation
-                == decimation_level.decimation.sample_rate
-            )
-        except AssertionError:
-            msg = (
-                f"Sample rates do not agree: fc {self.sample_rate_decimation} differs from "
-                f"processing config {decimation_level.decimation.sample_rate}"
-            )
-            self.logger.info(msg)
-            return False
-
-        # method (fft, wavelet, etc.)
-        try:
-            assert self.method == decimation_level.method
-        except AssertionError:
-            msg = (
-                "Transform methods do not agree "
-                f"{self.method} != {decimation_level.method}"
-            )
-            self.logger.info(msg)
-            return False
+        fc_decimations.append(fc_dec)
 
-        # prewhitening_type
-        try:
-            assert self.prewhitening_type == decimation_level.prewhitening_type
-        except AssertionError:
-            msg = (
-                "prewhitening_type does not agree "
-                f"{self.prewhitening_type} != {decimation_level.prewhitening_type}"
-            )
-            self.logger.info(msg)
-            return False
+    return fc_decimations
 
-        # recoloring
-        try:
-            assert self.recoloring == decimation_level.recoloring
-        except AssertionError:
-            msg = (
-                "recoloring does not agree "
-                f"{self.recoloring} != {decimation_level.recoloring}"
-            )
-            self.logger.info(msg)
-            return False
+def get_degenerate_fc_decimation(sample_rate: float) -> list:
+    """
+        WIP
 
-        # pre_fft_detrend_type
-        try:
-            assert (
-                self.pre_fft_detrend_type
-                == decimation_level.pre_fft_detrend_type
-            )
-        except AssertionError:
-            msg = (
-                "pre_fft_detrend_type does not agree "
-                f"{self.pre_fft_detrend_type} != {decimation_level.pre_fft_detrend_type}"
-            )
-            self.logger.info(msg)
-            return False
+        Makes a default fc_decimation list.
+        This "degenerate" config will only operate on the first decimation level.
+        This is useful for testing.  It could also be used in future on an MTH5 stored
+        time series in decimation levels already as separate runs.
 
-        # min_num_stft_windows
-        try:
-            assert (
-                self.min_num_stft_windows
-                == decimation_level.min_num_stft_windows
-            )
-        except AssertionError:
-            msg = (
-                "min_num_stft_windows do not agree "
-                f"{self.min_num_stft_windows} != {decimation_level.min_num_stft_windows}"
-            )
-            self.logger.info(msg)
-            return False
+    Parameters
+    ----------
+    sample_rate: float
+        The sample rate associated with the time-series to convert to spectrogram
 
-        # window
-        try:
-            assert self.window == decimation_level.window
-        except AssertionError:
-            msg = "window does not agree: "
-            msg = f"{msg} FC Group: {self.window} "
-            msg = f"{msg} Processing Config  {decimation_level.window}"
-            self.logger.info(msg)
-            return False
+    Returns
+    -------
+    output: list
+        List has only one element which is of type FCDecimation, aka.
 
-        if -1 in self.harmonic_indices:
-            # if harmonic_indices is -1, it means keep all so we can skip this check.
-            pass
-        else:
-            harmonic_indices_requested = decimation_level.harmonic_indices
-            fcdec_group_set = set(self.harmonic_indices)
-            processing_set = set(harmonic_indices_requested)
-            if processing_set.issubset(fcdec_group_set):
-                pass
-            else:
-                msg = (
-                    f"Processing FC indices {processing_set} is not contained "
-                    f"in FC indices {fcdec_group_set}"
-                )
-                self.logger.info(msg)
-                return False
+    """
+    output = fc_decimations_creator(
+        sample_rate,
+        decimation_factors=[
+            1,
+        ],
+        max_levels=1,
+    )
+    return output
 
-        # No checks were failed the FCDecimation supports the processing config
-        return True

mt_metadata/transfer_functions/processing/fourier_coefficients/fc.py

@@ -32,6 +32,8 @@ class FC(Base):
 
         self.time_period = TimePeriod()
         self.levels = ListDict()
+        self._decimation_levels = []
+        self._channels_estimated = []
 
         super().__init__(attr_dict=attr_dict, **kwargs)
 
@@ -97,7 +99,7 @@ class FC(Base):
         """list of decimation levels"""
         dl_list = []
         for dl in self.levels:
-            dl_list.append(dl.decimation_level)
+            dl_list.append(dl.decimation.level)
         dl_list = sorted(set([cc for cc in dl_list if cc is not None]))
         if self._decimation_levels == []:
             return dl_list
@@ -201,7 +203,7 @@ class FC(Base):
         if self.has_decimation_level(level):
             return self.levels[str(level)]
 
-    def add_decimation_level(self, decimation_level_obj):
+    def add_decimation_level(self, fc_decimation):
         """
         Add a decimation_level to the list, check if one exists if it does overwrite it
 
@@ -209,21 +211,21 @@ class FC(Base):
         :type decimation_level_obj: :class:`mt_metadata.transfer_functions.processing.fourier_coefficients.decimation_level`
 
         """
-        if not isinstance(decimation_level_obj, (Decimation)):
-            msg = f"Input must be metadata.decimation_level not {type(decimation_level_obj)}"
+        if not isinstance(fc_decimation, (Decimation)):
+            msg = f"Input must be metadata.decimation_level not {type(fc_decimation)}"
             self.logger.error(msg)
             raise ValueError(msg)
 
-        if self.has_decimation_level(decimation_level_obj.decimation_level):
-            self.levels[decimation_level_obj.decimation_level].update(
-                decimation_level_obj
+        if self.has_decimation_level(fc_decimation.decimation.level):
+            self.levels[fc_decimation.decimation.level].update(
+                fc_decimation
             )
             self.logger.debug(
-                f"ch {decimation_level_obj.level} already exists, updating metadata"
+                f"ch {fc_decimation.decimation.level} already exists, updating metadata"
             )
 
         else:
-            self.levels.append(decimation_level_obj)
+            self.levels.append(fc_decimation)
 
         self.update_time_period()
 

mt_metadata/transfer_functions/processing/fourier_coefficients/standards/decimation.json

@@ -1,15 +1,4 @@
 {
-    "decimation_level": {
-        "type": "integer",
-        "required": true,
-        "style": "number",
-        "units": null,
-        "description": "Decimation level, must be a non-negative integer starting at 0",
-        "options": [],
-        "alias": [],
-        "example": "1",
-		"default": null
-    },
     "id": {
         "type": "string",
         "required": true,
@@ -21,7 +10,7 @@
         "example": "1",
 		"default": null
     },
-	"channels_estimated": {
+    "channels_estimated": {
         "type": "string",
         "required": true,
         "style": "name list",
@@ -31,104 +20,5 @@
         "alias": [],
         "example": "[ex, hy]",
 		"default": null
-    },
-	"sample_rate_decimation": {
-        "type": "float",
-        "required": true,
-        "style": "number",
-        "units": "samples per second",
-        "description": "Sample rate of the decimation level.",
-        "options": [],
-        "alias": [],
-        "example": 60,
-		"default": 1
-    },
-	"decimation_factor": {
-        "type": "integer",
-        "required": true,
-        "style": "number",
-        "units": null,
-        "description": "Decimation factor between initial sample rate and decimation sample rate.",
-        "options": [],
-        "alias": [],
-        "example": 4,
-		"default": 1
-    },
-        "min_num_stft_windows": {
-        "type": "integer",
-        "required": true,
-        "style": "number",
-        "units": null,
-        "description": "How many FFT windows must be available for the time series to valid for STFT.",
-        "options": [],
-        "alias": [],
-        "example": 4,
-		"default": 2
-    },
-	"method": {
-        "type": "string",
-        "required": true,
-        "style": "controlled vocabulary",
-        "units": null,
-        "description": "Fourier transform method",
-        "options": ["fft", "wavelet", "other"],
-        "alias": [],
-        "example": "fft",
-		"default": "fft"
-    },
-	"anti_alias_filter": {
-        "type": "string",
-        "required": true,
-        "style": "free form",
-        "units": null,
-        "description": "Type of anti alias filter for decimation.",
-        "options": [],
-        "alias": [],
-        "example": "default",
-		"default": "default"
-    },
-	"pre_fft_detrend_type": {
-        "type": "string",
-        "required": true,
-        "style": "free form",
-        "units": null,
-        "description": "Type of detrend method before FFT.",
-        "options": [],
-        "alias": [],
-        "example": "linear",
-		"default": "linear"
-    },
-    "prewhitening_type": {
-        "type": "string",
-        "required": true,
-        "style": "controlled vocabulary",
-        "units": null,
-        "description": "Prewhitening method to be applied",
-        "options": ["first difference", "other"],
-        "alias": [],
-        "example": "first difference",
-        "default": "first difference"
-    },
-	"recoloring": {
-        "type": "bool",
-        "required": true,
-        "style": "free form",
-        "units": null,
-        "description": "Whether the data are recolored [True] or not [False].",
-        "options": [],
-        "alias": [],
-        "example": true,
-		"default": true
-    },
-	"harmonic_indices": {
-        "type": "integer",
-        "required": true,
-        "style": "number list",
-        "units": null,
-        "description": "List of harmonics indices kept, if all use -1",
-        "options": [],
-        "alias": [],
-        "example": [0, 4, 8],
-		"default": [-1]
     }
 }