acspype 0.1.0__py3-none-any.whl

acspype/__init__.py

@@ -0,0 +1,5 @@
+from .dev import ACSDev
+from .tscor import ACSTSCor
+from .stream import ACSStream
+from .processing import parse_packet, calibrate_packet
+

acspype/core.py

@@ -0,0 +1,28 @@
+import numpy as np
+
+NUM_PAT = "[+-]?[0-9]*[.]?[0-9]+"  # REGEX for any number, float or int, positive or negative.
+
+PACKET_REGISTRATION = b'\xff\x00\xff\x00'  # Start of every ACS packet.
+PAD_BYTE = b'\x00'  # End of every ACS packet.
+WVL_BYTE_OFFSET = 4 + 2 + 1 + 1 + 1 + 3 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 4 + 1  # See Process Data section in ACS manual.
+NUM_CHECKSUM_BYTES = 2
+PACKET_HEAD = '!4cHBBl7HIBB'  # struct descriptor for the static header of a packet.
+PACKET_TAIL = 'Hx' # struct descriptor for the static tail of a packet.
+LPR = len(PACKET_REGISTRATION)
+
+class DefaultSerial:
+    BAUDRATE: int = 115200
+    BYTESIZE: int = 8
+    PARITY: str = 'N'
+    STOPBITS: int = 1
+    FLOWCONTROL: int = 0
+    TIMEOUT: int = 3
+
+# Raw pressure counts are no longer output by an ACS and can be safely ignored. The reserved_1 and reserved_2 variables are single byte variables that are not used by the ACS and can be ignored.
+ACS_VARS_TO_IGNORE = ['raw_pressure', 'reserved_1', 'reserved_2']
+
+#---------- File Creation ----------#
+ENCODING = {'time': {'units': 'nanoseconds since 1900-01-01'}}  # xr.Dataset to netcdf encoding for time
+
+#---------- PHYSICAL QUANTITIES ----------#
+EST_FLOW_CELL_VOLUME = 30 # in mL, from the ACS Protocol Document, Rev Q.

acspype/dev.py

@@ -0,0 +1,229 @@
+from datetime import datetime
+import numpy as np
+import re
+from scipy import interpolate
+import xarray as xr
+
+from acspype.core import NUM_PAT
+
+
+class ACSDev:
+    """
+    A class for parsing ACS .dev files and putting them into a format that is easier to work with for larger or
+    multiple file datasets.
+
+    Generally, users will not call individual functions, but rather use the class to obtain attributes, which are
+    created at class instantiation or convert the data to an xarray dataset using the to_xarray function.
+    """
+
+    def __init__(self, filepath: str) -> None:
+        """
+        Run the following functions at instantiation to parse the .dev file and store the info as class attributes.
+
+        :param filepath: The filepath to the .dev file.
+        :return: None
+        """
+
+        self._filepath = filepath
+        self.__read_dev()
+        self.__parse_metadata()
+        self.__parse_tbins()
+        self.__parse_offsets()
+        self.__build_interp_funcs()
+        self.__check_parse()
+
+
+    def __read_dev(self) -> None:
+        """
+        Import the .dev file as a text file.
+        The file contents are stored as a list of strings in the class attribute self._content.
+
+        :return: None
+        """
+
+        with open(self._filepath, 'r') as _file:
+            self._content = _file.readlines()
+
+
+    def __parse_metadata(self) -> None:
+        """
+        Parse the .dev file for individual sensor metadata.
+        Sensor specific metadata are stored as class attributes.
+
+        :return: None
+        """
+
+        metadata_lines = [line for line in self._content if 'C and A offset' not in line]
+        for line in metadata_lines:
+            if 'ACS Meter' in line:
+                self.sensor_type = re.findall('(.*?)\n', line)[0]
+            elif 'Serial' in line:
+                self.sn_hexdec = re.findall('(.*?)\t', line)[0]
+                self.sn = 'ACS-' + str(int(self.sn_hexdec[-6:], 16)).zfill(5)  # Convert to sn shown on product sticker.
+            elif 'structure version' in line:
+                self.structure_version = int(re.findall(f'({NUM_PAT})\t', line)[0])
+            elif 'tcal' in line or 'Tcal' in line:
+                self.tcal, self.ical = [float(v) for v in re.findall(f': ({NUM_PAT}) C', line)]
+                cal_date_str = re.findall('file on (.*?)[.]', line)[0].replace(' ', '')
+                try:  # Sometimes the file date is entered as yyyy or yy. This should handle both cases.
+                    self.cal_date = datetime.strptime(cal_date_str, '%m/%d/%Y').strftime('%Y-%m-%d')
+                except:
+                    self.cal_date = datetime.strptime(cal_date_str, '%m/%d/%y').strftime('%Y-%m-%d')
+            elif 'Depth calibration' in line:
+                (self.depth_cal_1,
+                 self.depth_cal_2) = [float(v) for v in re.findall(f'({NUM_PAT})', line)]
+            elif 'Baud' in line:
+                self.baudrate = int(re.findall(f'({NUM_PAT})\t', line)[0])
+            elif 'Path' in line:
+                self.path_length = float(re.findall(f'({NUM_PAT})\t', line)[0])
+            elif 'wavelengths' in line:
+                self.num_wavelength = int(re.findall(f'({NUM_PAT})\t', line)[0])
+            elif 'number of temperature bins' in line:
+                self.num_tbin = int(re.findall(f'({NUM_PAT})\t', line)[0])
+            elif 'maxANoise' in line:
+                (self.max_a_noise, self.max_c_noise, self.max_a_nonconform, self.max_c_nonconform,
+                 self.max_a_difference, self.max_c_difference, self.min_a_counts,
+                 self.min_c_counts, self.min_r_counts, self.max_temp_sd,
+                 self.max_depth_sd) = [float(v) for v in re.findall(f'({NUM_PAT})\t', line)]
+
+
+    def __parse_tbins(self) -> None:
+        """
+        Parse the .dev file for temperature bin information.
+
+        :return: None
+        """
+        tbin_line = [line for line in self._content if '; temperature bins' in line][0]
+        tbins = tbin_line.split('\t')
+        tbins = [v for v in tbins if v]  # Toss empty strings.
+        tbins = [v for v in tbins if v != '\n']  # Toss newline characters.
+        self.tbin = np.array([float(v) for v in tbins if 'temperature bins' not in v])  # Convert to float and toss comment.
+
+
+    def __parse_offsets(self) -> None:
+        """
+        Parse the .dev file for a and c offsets. Data are saved as class attributes for access at a later time.
+
+        :return: None
+        """
+
+        offset_lines = [line for line in self._content if 'C and A offset' in line]
+
+        # Create holder arrays to loop over and append data to.
+        c_wvls = []
+        a_wvls = []
+        c_offs = []
+        a_offs = []
+        c_deltas = []
+        a_deltas = []
+        wavelength_color_schemes = []
+
+        for line in offset_lines:
+            offsets, c_delta, a_delta = line.split('\t\t')[:-1]
+            c_wvl, a_wvl, wvl_color, c_off, a_off = offsets.split('\t')
+
+            # Convert strings to proper pythonic datatypes.
+            c_wvl = float(c_wvl.replace('C', ''))
+            a_wvl = float(a_wvl.replace('A', ''))
+            c_off = float(c_off)
+            a_off = float(a_off)
+            c_delta = [float(v) for v in c_delta.split('\t')]
+            a_delta = [float(v) for v in a_delta.split('\t')]
+
+            # Append files to holder arrays.
+            c_wvls.append(c_wvl)
+            a_wvls.append(a_wvl)
+            c_offs.append(c_off)
+            a_offs.append(a_off)
+            c_deltas.append(c_delta)
+            a_deltas.append(a_delta)
+            wavelength_color_schemes.append(wvl_color)
+
+        # Convert holder arrays to numpy arrays.
+        self.c_wavelength = np.array(c_wvls)
+        self.a_wavelength = np.array(a_wvls)
+        self.c_offset = np.array(c_offs)
+        self.a_offset = np.array(a_offs)
+        self.c_delta_t = np.array(c_deltas)
+        self.a_delta_t = np.array(a_deltas)
+        self.wavelength_color_schemes = wavelength_color_schemes
+
+
+    def __build_interp_funcs(self) -> None:
+        """
+        Build interpolation functions for the a and c delta_t values and store as class attributes.
+
+        :return: None
+        """
+        self.func_a_delta_t = interpolate.interp1d(self.tbin, self.a_delta_t, axis=1)
+        self.func_c_delta_t = interpolate.interp1d(self.tbin, self.c_delta_t, axis=1)
+        self.delta_t_interp_method = 'scipy.interpolate.interp1d'
+
+
+    def __check_parse(self) -> None:
+        """
+        Verify that the shape of the data is as expected.
+
+        :return: None
+        """
+
+        if len(self.a_wavelength) != self.num_wavelength:
+            raise ValueError('Mismatch between number of wavelengths extracted for A and expected from file.'
+                             'Please verify the .dev file integrity.')
+        if len(self.c_wavelength) != self.num_wavelength:
+            raise ValueError('Mismatch between number of wavelengths extracted for C and expected from file.'
+                             'Please verify the .dev file integrity.')
+        if len(self.c_wavelength) != len(self.a_wavelength):
+            raise ValueError('Mismatch between number of wavelengths extracted for A and C.'
+                             'Please verify the .dev file integrity.')
+        if np.array(self.a_delta_t).shape != (len(self.a_wavelength), self.num_tbin):
+            raise ValueError('Mismatch between length of A wavelengths and number of temperature bins.'
+                             'Please verify the .dev file integrity.')
+        if np.array(self.c_delta_t).shape != (len(self.a_wavelength), self.num_tbin):
+            raise ValueError('Mismatch between length of C wavelengths and number of temperature bins.'
+                             'Please verify the .dev file integrity.')
+
+
+    def to_xarray(self) -> xr.Dataset:
+        """
+        Convert the parsed .dev file files to an xarray dataset
+
+        Returns: An appropriately dimensioned xarray dataset containing device file files.
+        """
+        ds = xr.Dataset()
+        ds = ds.assign_coords({'a_wavelength': self.a_wavelength,
+                               'c_wavelength': self.c_wavelength,
+                               'temperature_bin': self.tbin})
+
+        ds['a_offset'] = (['a_wavelength'], self.a_offset)
+        ds['a_delta_t'] = (['a_wavelength', 'temperature_bin'], self.a_delta_t)
+
+        ds['c_offset'] = (['c_wavelength'], np.array(self.c_offset))
+        ds['c_delta_t'] = (['c_wavelength', 'temperature_bin'], self.c_delta_t)
+
+        ds.attrs['device_filepath'] = self._filepath
+        ds.attrs['sensor_type'] = self.sensor_type
+        ds.attrs['serial_number_hexdec'] = self.sn_hexdec
+        ds.attrs['serial_number'] = self.sn
+        ds.attrs['device_file_structure_version'] = self.structure_version
+        ds.attrs['tcal'] = self.tcal
+        ds.attrs['ical'] = self.ical
+        ds.attrs['calibration_date'] = self.cal_date
+        ds.attrs['depth_cal_1'] = self.depth_cal_1
+        ds.attrs['depth_cal_2'] = self.depth_cal_2
+        ds.attrs['baudrate'] = self.baudrate
+        ds.attrs['path_length'] = self.path_length
+        ds.attrs['number_of_wavelength_bins'] = self.num_wavelength
+        ds.attrs['number_of_temperature_bins'] = self.num_tbin
+        ds.attrs['max_a_noise'] = self.max_a_noise
+        ds.attrs['max_c_noise'] = self.max_c_noise
+        ds.attrs['max_a_nonconform'] = self.max_a_nonconform
+        ds.attrs['max_c_nonconform'] = self.max_c_nonconform
+        ds.attrs['max_a_difference'] = self.max_a_difference
+        ds.attrs['max_c_difference'] = self.max_c_difference
+        ds.attrs['min_a_counts'] = self.min_a_counts
+        ds.attrs['min_c_counts'] = self.min_c_counts
+        ds.attrs['min_r_counts'] = self.min_r_counts
+        ds.attrs['max_temp_sd'] = self.max_temp_sd
+        ds.attrs['max_depth_sd'] = self.max_depth_sd
+        return ds

acspype/discontinuity.py

@@ -0,0 +1,142 @@
+import numpy as np
+from scipy.interpolate import CubicSpline
+from typing import Union
+import xarray as xr
+
+
+def find_discontinuity_index(a_wavelengths: Union[list,tuple, np.array, xr.DataArray],
+                             c_wavelengths: Union[list,tuple, np.array, xr.DataArray],
+                             min_band: int = 535, max_band: int = 600) -> int:
+    """
+
+    This code is modified from the OPTAA processing utilities in the ooi-data-explorations repo.
+    https://github.com/IanTBlack/ooi-data-explorations/blob/master/python/ooi_data_explorations/uncabled/utilities/utilities_optaa.py#L104
+
+    Find the last wavelength index of the first filter based on wavelength differences.
+    This function assumes that the discontinuity occurs between 535 nm and 600 nm, which is buffered from the values in the ACS Protocol Document, Rev Q.
+
+    :param a_wavelengths: Absorption wavelengths
+    :param c_wavelengths: Attenuation wavelengths
+    :return: The last wavelength index at the discontinuity.
+    """
+
+    # Copy and convert to numpy arrays because we are paranoid about global variables.
+    a_wavelengths = np.array(a_wavelengths).copy()
+    c_wavelengths = np.array(c_wavelengths).copy()
+
+    # Set values outside the range to NaN
+    a_wavelengths[(a_wavelengths < min_band) | (a_wavelengths > max_band)] = np.nan
+    c_wavelengths[(c_wavelengths < min_band) | (c_wavelengths > max_band)] = np.nan
+
+    # Find the index of the discontinuity
+    didx = int(np.nanargmin(np.diff(a_wavelengths) + np.diff(c_wavelengths)))
+    return didx
+
+
+
+def _compute_discontinuity_offset(values: Union[list, tuple, np.array],
+                                  wavelength: Union[list, tuple, np.array],
+                                  didx: int) -> float:
+    """
+    This code is modified from the OPTAA processing utilities in the ooi-data-explorations repo.
+    https://github.com/IanTBlack/ooi-data-explorations/blob/master/python/ooi_data_explorations/uncabled/utilities/utilities_optaa.py#L212
+
+    Compute the scalar discontinuity offset to be applied to the second half of an ACS spectra.
+    NOTE: If the input values contain an inf value, the function will return -999. This is to prevent math errors associated with creating a cubic spline on infinite values.
+    Spectra with infinite values should be removed at some point in the processing pipeline.
+
+    :param values: The incoming absorption or attenuation values. It is highly recommended that these values be
+        representative of ACS data that have been converted to 'geophysical' units (1/m) and corrected for the effects
+        of internal temperature on output. That is to say, the recommended input is the measured (a_m and c_m) in the
+        ACS protocol documents and manual. acspype inputs would be a_m_discontinuity and c_m_discontinuity.
+    :param wavelength: The wavelength bins of the values.
+    :param didx: The index of discontinuity.
+    :return: The discontinuity offset for the second half of the spectrum.
+    """
+    _wavelength = np.copy(wavelength)
+    _values = np.copy(values)
+    _didx = int(np.copy(didx))
+
+    x = _wavelength[_didx - 2:_didx + 1]
+    y = _values[_didx - 2:_didx + 1]
+
+    if True in np.isinf(y):
+        return -999
+    else:
+        cubic_spline = CubicSpline(x, y)
+        interp = cubic_spline(_wavelength[_didx + 1], extrapolate=True)
+        offset = interp - _values[_didx + 1]
+        return offset
+
+
+def _apply_discontinuity_offset(values: Union[list, tuple, np.array],
+                                offset: float,
+                                didx: int) -> np.array:
+
+    """
+    This code is modified from the OPTAA processing utilities in the ooi-data-explorations repo.
+    https://github.com/IanTBlack/ooi-data-explorations/blob/master/python/ooi_data_explorations/uncabled/utilities/utilities_optaa.py#L212
+
+    Apply a pre-determined discontinuity offset to values after the discontinuity index.
+
+    :param values: The measured values to apply the discontinuity offset to.
+    :param offset: The scalar offset to apply to the values after the discontinuity index.
+    :param didx: The discontinuity index computed from find_discontinuity_index.
+    :return: A discontinuity-corrected spectra.
+    """
+
+    _values = np.copy(values)
+    _offset = np.copy(offset)
+    _didx = int(np.copy(didx))
+    _values[_didx + 1:] = _values[_didx + 1:] + _offset
+    return _values
+
+
+def compute_discontinuity_offset(measured, wavelength_dim, discontinuity_index):
+    """
+    This is a wrapper function for _compute_discontinuity_offset that is vectorized for Xarray.
+
+    :param measured: The measured values
+    :param wavelength_dim: The dimension to calculate the offset on.
+    :param discontinuity_index
+    :return: The scalar offset for a given spectrum.
+    """
+
+    offset = xr.apply_ufunc(_compute_discontinuity_offset, measured,
+                            kwargs = {'wavelength': measured[wavelength_dim].values, 'didx': discontinuity_index},
+                            input_core_dims = [[wavelength_dim]],
+                            output_core_dims = [[]],
+                            vectorize = True)
+    return offset
+
+
+def apply_discontinuity_offset(measured, offset, wavelength_dim, discontinuity_index):
+    """
+    This is a wrapper function for _apply_discontinuity_offset that is vectorized for Xarray.
+    :param measured: The measured values.
+    :param offset: The pre-determined discontinuity offset.
+    :param wavelength_dim: The wavelength dimension to apply the offset to.
+    :param discontinuity_index: The index of discontinuity.
+    :return: Spectra with the discontinuity offset applied.
+    """
+
+    dc = xr.apply_ufunc(_apply_discontinuity_offset, measured, offset,
+                        kwargs = {'didx': discontinuity_index},
+                        input_core_dims=[[wavelength_dim],[]],
+                        output_core_dims = [[wavelength_dim]],
+                        vectorize = True)
+    return dc
+
+
+def discontinuity_correction(measured, wavelength_dim, discontinuity_index):
+    """
+    This is a convenience function for computing the discontinuity offset and applying it to the measured values in Xarray.
+    :param measured: The measured values.
+    :param wavelength_dim: The wavelength dimension to apply the correction to.
+    :param discontinuity_index: The index of discontinuity.
+    :return:
+    """
+
+    offset = compute_discontinuity_offset(measured, wavelength_dim, discontinuity_index)
+    dc = apply_discontinuity_offset(measured, offset, wavelength_dim, discontinuity_index)
+    return dc, offset

acspype/experimental.py

@@ -0,0 +1,64 @@
+import xarray as xr
+
+
+def compute_chl_alh(absorption: xr.DataArray, alh_star: float = 0.0104) -> xr.DataArray:
+    """
+    Compute chlorophyll-a from absorption line height via Roesler and Barnard 2013.
+    https://www.sciencedirect.com/science/article/pii/S2211122013000509
+
+    :param absorption: Absorption data with wavelength as a coordinate.
+    :param alh_star: Absorption line height coefficient value, default is 0.0104,
+        which is the average from Table 1 in Roesler and Barnard 2013.
+    :return: Chlorophyll-a concentration in mg/m^3.
+    """
+
+    a650 = absorption.sel(wavelength=650, method='nearest')
+    a676 = absorption.sel(wavelength=676, method='nearest')
+    a715 = absorption.sel(wavelength=715, method='nearest')
+
+    abl = ((a715 - a650) / (715 - 650)) * (676 - 650) + a650  #EQ 1 in Roesler and Barnard 2013
+    alh = a676 - abl  #EQ 2 in Roesler and Barnard 2013
+    chl_alh = alh / alh_star #EQ 3 in Roesler and Barnard 2013
+
+    chl_alh.attrs['alh_star'] = alh_star
+    chl_alh.attrs['method'] = 'Roesler and Barnard, 2013'
+    chl_alh.attrs['ancillary_variables'] = str(absorption.name)
+    return chl_alh
+
+#
+# def compute_poc(attenuation: xr.DataArray, method = 'gardner_et_al_2006'):
+#     """
+#     Compute particulate organic carbon (POC) from attenuation based on linear models defined by the issued method.
+#
+#     :param attenuation: The particulate attenuation coefficient (c_p).
+#     :param method: The method to use for POC calculation. Options are:
+#         gardner_et_al_2006
+#         behrenfeld_and_boss_2006
+#         stramski_et_al_2008
+#         cetenic_et_al_2012
+#         goni_et_al_2021 -> Oregon Coast, August, 2011, Sigma 22-23
+#     :return: POC in mg/m3
+#     """
+#
+#     if method == 'gardner_et_al_2006':
+#         m = 381  #POC to c_p slope in units of mgC/m2
+#         c_p = attenuation.sel(wavelength=660, method='nearest')
+#         b = 9.4
+#     elif method == 'behrenfeld_and_boss_2006':
+#         m = 585
+#         c_p = attenuation.sel(wavelength=660, method='nearest')
+#         b = 7.6
+#     elif method == 'stramski_et_al_2008':
+#         m = 458
+#         c_p = attenuation.sel(wavelength=660, method='nearest')
+#         b = 0
+#     elif method == 'cetenic_et_al_2012':
+#         m = 391
+#         c_p = attenuation.sel(wavelength=660, method='nearest')
+#         b = -5.8
+#     elif method == 'goni_et_al_2021':
+#         m = 38.9* 12.01
+#         c_p = attenuation.sel(wavelength=650, method='nearest')
+#         b = 0
+#     poc = m * c_p + b
+#     return poc