imap-processing 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

imap_processing/spice/spin.py

@@ -0,0 +1,233 @@
+"""Functions for retrieving spin-table data."""
+
+import os
+from pathlib import Path
+from typing import Union
+
+import numpy as np
+import pandas as pd
+from numpy import typing as npt
+
+from imap_processing.spice.geometry import (
+    SpiceFrame,
+    get_spacecraft_to_instrument_spin_phase_offset,
+)
+
+
+def get_spin_data() -> pd.DataFrame:
+    """
+    Read spin file using environment variable and return spin data.
+
+    SPIN_DATA_FILEPATH environment variable would be a fixed value.
+    It could be s3 filepath that can be used to download the data
+    through API or it could be path EFS or Batch volume mount path.
+
+    Spin data should contain the following fields:
+        * spin_number
+        * spin_start_sec
+        * spin_start_subsec
+        * spin_period_sec
+        * spin_period_valid
+        * spin_phase_valid
+        * spin_period_source
+        * thruster_firing
+
+    Returns
+    -------
+    spin_data : pandas.DataFrame
+        Spin data.
+    """
+    spin_data_filepath = os.getenv("SPIN_DATA_FILEPATH")
+    if spin_data_filepath is not None:
+        path_to_spin_file = Path(spin_data_filepath)
+    else:
+        # Handle the case where the environment variable is not set
+        raise ValueError("SPIN_DATA_FILEPATH environment variable is not set.")
+
+    spin_df = pd.read_csv(path_to_spin_file, comment="#")
+    # Combine spin_start_sec and spin_start_subsec to get the spin start
+    # time in seconds. The spin start subseconds are in milliseconds.
+    spin_df["spin_start_time"] = (
+        spin_df["spin_start_sec"] + spin_df["spin_start_subsec"] / 1e3
+    )
+
+    return spin_df
+
+
+def interpolate_spin_data(query_met_times: Union[float, npt.NDArray]) -> pd.DataFrame:
+    """
+    Interpolate spin table data to the queried MET times.
+
+    All columns in the spin table csv file are interpolated to the previous
+    table entry. A sc_spin_phase column is added that is the computed spacecraft
+    spin phase at the queried MET times. Note that spin phase is by definition,
+    in the interval [0, 1) where 1 is equivalent to 360 degrees.
+
+    Parameters
+    ----------
+    query_met_times : float or np.ndarray
+        Query times in Mission Elapsed Time (MET).
+
+    Returns
+    -------
+    spin_df : pandas.DataFrame
+        Spin table data with the spin-phase column added and one row
+        interpolated for each queried MET time. Output columns are:
+        * spin_number
+        * spin_start_sec
+        * spin_start_subsec
+        * spin_period_sec
+        * spin_period_valid
+        * spin_phase_valid
+        * spin_period_source
+        * thruster_firing
+        * spin_start_met
+        * sc_spin_phase
+    """
+    spin_df = get_spin_data()
+
+    # Ensure query_met_times is an array
+    query_met_times = np.asarray(query_met_times)
+    is_scalar = query_met_times.ndim == 0
+    if is_scalar:
+        # Force scalar to array because np.asarray() will not
+        # convert scalar to array
+        query_met_times = np.atleast_1d(query_met_times)
+
+    # Make sure input times are within the bounds of spin data
+    spin_df_start_time = spin_df["spin_start_time"].values[0]
+    spin_df_end_time = (
+        spin_df["spin_start_time"].values[-1] + spin_df["spin_period_sec"].values[-1]
+    )
+    input_start_time = query_met_times.min()
+    input_end_time = query_met_times.max()
+    if input_start_time < spin_df_start_time or input_end_time >= spin_df_end_time:
+        raise ValueError(
+            f"Query times, {query_met_times} are outside of the spin data range, "
+            f"{spin_df_start_time, spin_df_end_time}."
+        )
+
+    # Find all spin time that are less or equal to query_met_times.
+    # To do that, use side right, a[i-1] <= v < a[i], in the searchsorted.
+    # Eg.
+    # >>> df['a']
+    # array([0, 15, 30, 45, 60])
+    # >>> np.searchsorted(df['a'], [0, 13, 15, 32, 70], side='right')
+    # array([1, 1, 2, 3, 5])
+    last_spin_indices = (
+        np.searchsorted(spin_df["spin_start_time"], query_met_times, side="right") - 1
+    )
+    # Generate a dataframe with one row per query time
+    out_df = spin_df.iloc[last_spin_indices]
+
+    # Calculate spin phase
+    spin_phases = (query_met_times - out_df["spin_start_time"].values) / out_df[
+        "spin_period_sec"
+    ].values
+
+    # Check for invalid spin phase using below checks:
+    # 1. Check that the spin phase is in valid range, [0, 1).
+    # 2. Check invalid spin phase using spin_phase_valid,
+    #   spin_period_valid columns.
+    invalid_spin_phase_range = (spin_phases < 0) | (spin_phases >= 1)
+
+    invalid_spins = (out_df["spin_phase_valid"].values == 0) | (
+        out_df["spin_period_valid"].values == 0
+    )
+    bad_spin_phases = invalid_spin_phase_range | invalid_spins
+    spin_phases[bad_spin_phases] = np.nan
+
+    # Add spin_phase column to output dataframe
+    out_df["sc_spin_phase"] = spin_phases
+
+    return out_df
+
+
+def get_spin_angle(
+    spin_phases: Union[float, npt.NDArray],
+    degrees: bool = False,
+) -> Union[float, npt.NDArray]:
+    """
+    Convert spin_phases to radians or degrees.
+
+    Parameters
+    ----------
+    spin_phases : float or np.ndarray
+        Instrument or spacecraft spin phases. Spin phase is a
+        floating point number in the range [0, 1) corresponding to the
+        spin angle / 360.
+    degrees : bool
+        If degrees parameter is True, return angle in degrees otherwise return angle in
+        radians. Default is False.
+
+    Returns
+    -------
+    spin_phases : float or np.ndarray
+        Spin angle in degrees or radians for the input query times.
+    """
+    if np.any(spin_phases < 0) or np.any(spin_phases >= 1):
+        raise ValueError(
+            f"Spin phases, {spin_phases} are outside of the expected spin phase range, "
+            f"[0, 1) "
+        )
+    if degrees:
+        # Convert to degrees
+        return spin_phases * 360
+    else:
+        # Convert to radians
+        return spin_phases * 2 * np.pi
+
+
+def get_spacecraft_spin_phase(
+    query_met_times: Union[float, npt.NDArray],
+) -> Union[float, npt.NDArray]:
+    """
+    Get the spacecraft spin phase for the input query times.
+
+    Formula to calculate spin phase:
+        spin_phase = (query_met_times - spin_start_time) / spin_period_sec
+
+    Parameters
+    ----------
+    query_met_times : float or np.ndarray
+        Query times in Mission Elapsed Time (MET).
+
+    Returns
+    -------
+    spin_phase : float or np.ndarray
+        Spin phase for the input query times.
+    """
+    spin_df = interpolate_spin_data(query_met_times)
+    if np.asarray(query_met_times).ndim == 0:
+        return spin_df["sc_spin_phase"].values[0]
+    return spin_df["sc_spin_phase"].values
+
+
+def get_instrument_spin_phase(
+    query_met_times: Union[float, npt.NDArray], instrument: SpiceFrame
+) -> Union[float, npt.NDArray]:
+    """
+    Get the instrument spin phase for the input query times.
+
+    Formula to calculate spin phase:
+        instrument_spin_phase = (spacecraft_spin_phase + instrument_spin_offset) % 1
+
+    Parameters
+    ----------
+    query_met_times : float or np.ndarray
+        Query times in Mission Elapsed Time (MET).
+    instrument : SpiceFrame
+        Instrument frame to calculate spin phase for.
+
+    Returns
+    -------
+    spin_phase : float or np.ndarray
+        Instrument spin phase for the input query times. Spin phase is a
+        floating point number in the range [0, 1) corresponding to the
+        spin angle / 360.
+    """
+    spacecraft_spin_phase = get_spacecraft_spin_phase(query_met_times)
+    instrument_spin_phase_offset = get_spacecraft_to_instrument_spin_phase_offset(
+        instrument
+    )
+    return (spacecraft_spin_phase + instrument_spin_phase_offset) % 1

imap_processing/spice/time.py

@@ -1,4 +1,4 @@
-"""Time conversion functions that rely on SPICE."""
+"""Time conversion functions that rely on SPICEYPY."""
 
 import typing
 from collections.abc import Collection, Iterable
@@ -6,7 +6,7 @@ from typing import Union
 
 import numpy as np
 import numpy.typing as npt
-import spiceypy as spice
+import spiceypy
 
 from imap_processing.spice import IMAP_SC_ID
 from imap_processing.spice.kernels import ensure_spice
@@ -19,8 +19,38 @@ TICK_DURATION = 2e-5  # 20 microseconds as defined in imap_sclk_0000.tsc
 # TODO: Implement a function for converting CDF epoch to UTC correctly.
 #     see github ticket #1208
 # The UTC string was generated by:
-# >>> spiceypy.et2utc(0, "ISOC", 9)
-J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816072737", "ns")
+# >>> spiceypy.et2utc(spiceypy.unitim(0, "TT", "ET"), "ISOC", 9)
+TTJ2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns")
+
+
+@typing.no_type_check
+def _vectorize(pyfunc: typing.Callable, **vectorize_kwargs) -> typing.Callable:
+    """
+    Convert 0-D arrays from numpy.vectorize to scalars.
+
+    For details on numpy.vectorize, see
+    https://numpy.org/doc/stable/reference/generated/numpy.vectorize.html
+
+    Parameters
+    ----------
+    pyfunc : callable
+        A python function or method.
+    **vectorize_kwargs :
+        Keyword arguments to pass to numpy.vectorize.
+
+    Returns
+    -------
+    out : callable
+        A vectorized function.
+    """
+    vectorized_func = np.vectorize(pyfunc, **vectorize_kwargs)
+
+    def wrapper(*args, **kwargs):  # numpydoc ignore=GL08
+        # Calling the vectorized function with [()] will convert 0-D arrays
+        # to scalars
+        return vectorized_func(*args, **kwargs)[()]
+
+    return wrapper
 
 
 def met_to_sclkticks(met: npt.ArrayLike) -> npt.NDArray[float]:
@@ -40,11 +70,11 @@ def met_to_sclkticks(met: npt.ArrayLike) -> npt.NDArray[float]:
     return np.asarray(met, dtype=float) / TICK_DURATION
 
 
-def met_to_j2000ns(
+def met_to_ttj2000ns(
     met: npt.ArrayLike,
 ) -> npt.NDArray[np.int64]:
     """
-    Convert mission elapsed time (MET) to nanoseconds since J2000.
+    Convert mission elapsed time (MET) to terrestrial time nanoseconds since J2000.
 
     Parameters
     ----------
@@ -54,7 +84,8 @@ def met_to_j2000ns(
     Returns
     -------
     numpy.ndarray[numpy.int64]
-        The mission elapsed time converted to nanoseconds since the J2000 epoch.
+        The mission elapsed time converted to nanoseconds since the J2000 epoch
+        in the terrestrial time (TT) timescale.
 
     Notes
     -----
@@ -67,27 +98,35 @@ def met_to_j2000ns(
     it is preferable to use the higher accuracy method.
     """
     sclk_ticks = met_to_sclkticks(met)
-    return np.asarray(_sct2e_wrapper(sclk_ticks) * 1e9, dtype=np.int64)
+    return np.asarray(sct_to_ttj2000s(sclk_ticks) * 1e9, dtype=np.int64)
 
 
-def j2000ns_to_j2000s(j2000ns: npt.ArrayLike) -> npt.NDArray[float]:
+@typing.no_type_check
+@ensure_spice
+def ttj2000ns_to_et(tt_ns: npt.ArrayLike) -> npt.NDArray[float]:
     """
-    Convert the J2000 epoch nanoseconds to J2000 epoch seconds.
+    Convert TT J2000 epoch nanoseconds to TDB J2000 epoch seconds.
 
-    The common CDF coordinate `epoch` stores J2000 nanoseconds. SPICE requires
-    J2000 seconds be used. This is a common function to do that conversion.
+    The common CDF coordinate `epoch` stores terrestrial time (TT) J2000
+    nanoseconds. SPICE requires Barycentric Dynamical Time (TDB, aka ET) J2000
+    floating point seconds for most geometry related functions. This is a common
+    function to do that conversion.
 
     Parameters
     ----------
-    j2000ns : float, numpy.ndarray
-        Number of nanoseconds since the J2000 epoch.
+    tt_ns : float, numpy.ndarray
+        Number of nanoseconds since the J2000 epoch in the TT timescale.
 
     Returns
     -------
     numpy.ndarray[float]
-        Number of seconds since the J2000 epoch.
+        Number of seconds since the J2000 epoch in the TDB timescale.
     """
-    return np.asarray(j2000ns, dtype=np.float64) / 1e9
+    tt_seconds = np.asarray(tt_ns, dtype=np.float64) / 1e9
+    vectorized_unitim = _vectorize(
+        spiceypy.unitim, otypes=[float], excluded=["insys", "outsys"]
+    )
+    return vectorized_unitim(tt_seconds, "TT", "ET")
 
 
 @typing.no_type_check
@@ -112,8 +151,8 @@ def met_to_utc(met: npt.ArrayLike, precision: int = 9) -> npt.NDArray[str]:
         fractional seconds precision as specified by the precision keyword.
     """
     sclk_ticks = met_to_sclkticks(met)
-    et = _sct2e_wrapper(sclk_ticks)
-    return spice.et2utc(et, "ISOC", prec=precision)
+    et = sct_to_et(sclk_ticks)
+    return spiceypy.et2utc(et, "ISOC", prec=precision)
 
 
 def met_to_datetime64(
@@ -132,14 +171,12 @@ def met_to_datetime64(
     numpy.ndarray[str]
         The mission elapsed time converted to UTC string.
     """
-    if isinstance(met, typing.Iterable):
-        return np.asarray([np.datetime64(utc) for utc in met_to_utc(met)])
-    return np.datetime64(met_to_utc(met))
+    return np.array(met_to_utc(met), dtype=np.datetime64)[()]
 
 
 @typing.no_type_check
 @ensure_spice
-def _sct2e_wrapper(
+def sct_to_et(
     sclk_ticks: Union[float, Collection[float]],
 ) -> Union[float, np.ndarray]:
     """
@@ -159,10 +196,40 @@ def _sct2e_wrapper(
     ephemeris_time: np.ndarray
         Ephemeris time, seconds past J2000.
     """
-    if isinstance(sclk_ticks, Collection):
-        return np.array([spice.sct2e(IMAP_SC_ID, s) for s in sclk_ticks])
-    else:
-        return spice.sct2e(IMAP_SC_ID, sclk_ticks)
+    vectorized_sct2e = _vectorize(spiceypy.sct2e, otypes=[float], excluded=[0])
+    return vectorized_sct2e(IMAP_SC_ID, sclk_ticks)
+
+
+@typing.no_type_check
+@ensure_spice
+def sct_to_ttj2000s(
+    sclk_ticks: Union[float, Iterable[float]],
+) -> Union[float, np.ndarray]:
+    """
+    Convert encoded spacecraft clock "ticks" to terrestrial time (TT).
+
+    Decorated wrapper for chained spiceypy functions `unitim(sct2e(), "ET", "TT")`
+    that vectorizes the functions in addition to wrapping with the @ensure_spice
+    automatic kernel furnishing functionality.
+    https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/sct2e_c.html
+    https://naif.jpl.nasa.gov/pub/naif/toolkit_docs/C/cspice/unitim_c.html
+
+    Parameters
+    ----------
+    sclk_ticks : Union[float, Iterable[float]]
+        Input sclk ticks value(s) to be converted to ephemeris time.
+
+    Returns
+    -------
+    terrestrial_time: np.ndarray[float]
+        Terrestrial time, seconds past J2000.
+    """
+
+    def conversion(sclk_ticks):  # numpydoc ignore=GL08
+        return spiceypy.unitim(spiceypy.sct2e(IMAP_SC_ID, sclk_ticks), "ET", "TT")
+
+    vectorized_func = _vectorize(conversion, otypes=[float])
+    return vectorized_func(sclk_ticks)
 
 
 @typing.no_type_check
@@ -187,10 +254,8 @@ def str_to_et(
     ephemeris_time: np.ndarray
         Ephemeris time, seconds past J2000.
     """
-    if isinstance(time_str, str):
-        return spice.str2et(time_str)
-    else:
-        return np.array([spice.str2et(t) for t in time_str])
+    vectorized_str2et = _vectorize(spiceypy.str2et, otypes=[float])
+    return vectorized_str2et(time_str)
 
 
 @typing.no_type_check
@@ -231,4 +296,4 @@ def et_to_utc(
     utc_time : str or np.ndarray
         UTC time(s).
     """
-    return spice.et2utc(et, format_str, precision, utclen)
+    return spiceypy.et2utc(et, format_str, precision, utclen)

imap_processing/swapi/l1/swapi_l1.py

@@ -9,6 +9,7 @@ import xarray as xr
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
+from imap_processing.cdf.utils import load_cdf
 from imap_processing.quality_flags import SWAPIFlags
 from imap_processing.swapi.swapi_utils import SWAPIAPID, SWAPIMODE
 from imap_processing.utils import packet_file_to_datasets
@@ -485,12 +486,19 @@ def process_swapi_science(
     # ===================================================================
     # Quality flags
     # ===================================================================
-    quality_flags_data = np.zeros((total_full_sweeps, 72), np.uint16)
+    quality_flags_data = np.zeros((total_full_sweeps, 72), dtype=np.uint16)
 
     # Add science data quality flags
-    quality_flags_data[pcem_compression_flags == 1] |= SWAPIFlags.SWP_PCEM_COMP.value
-    quality_flags_data[scem_compression_flags == 1] |= SWAPIFlags.SWP_SCEM_COMP.value
-    quality_flags_data[coin_compression_flags == 1] |= SWAPIFlags.SWP_COIN_COMP.value
+    # Have to match datatype to bitwise OR
+    quality_flags_data[pcem_compression_flags == 1] |= np.uint16(
+        SWAPIFlags.SWP_PCEM_COMP
+    )
+    quality_flags_data[scem_compression_flags == 1] |= np.uint16(
+        SWAPIFlags.SWP_SCEM_COMP
+    )
+    quality_flags_data[coin_compression_flags == 1] |= np.uint16(
+        SWAPIFlags.SWP_COIN_COMP
+    )
 
     # Add housekeeping-derived quality flags
     # --------------------------------------
@@ -536,10 +544,10 @@ def process_swapi_science(
         # Use getattr to dynamically access the flag in SWAPIFlags class
         flag_to_set = getattr(SWAPIFlags, flag_name)
         # set the quality flag for each data
-        quality_flags_data[current_flag == 1] |= flag_to_set.value
+        quality_flags_data[current_flag == 1] |= np.uint16(flag_to_set)
 
     swp_flags = xr.DataArray(
-        quality_flags_data,
+        quality_flags_data.astype(np.uint16),
         dims=["epoch", "energy"],
         attrs=cdf_manager.get_variable_attributes("flags_default"),
     )
@@ -555,7 +563,7 @@ def process_swapi_science(
         epoch_values,
         name="epoch",
         dims=["epoch"],
-        attrs=cdf_manager.get_variable_attributes("epoch"),
+        attrs=cdf_manager.get_variable_attributes("epoch", check_schema=False),
     )
 
     # There are 72 energy steps
@@ -563,14 +571,14 @@ def process_swapi_science(
         np.arange(72),
         name="energy",
         dims=["energy"],
-        attrs=cdf_manager.get_variable_attributes("energy"),
+        attrs=cdf_manager.get_variable_attributes("energy", check_schema=False),
     )
     # LABL_PTR_1 should be CDF_CHAR.
     energy_label = xr.DataArray(
         energy.values.astype(str),
         name="energy_label",
         dims=["energy_label"],
-        attrs=cdf_manager.get_variable_attributes("energy_label"),
+        attrs=cdf_manager.get_variable_attributes("energy_label", check_schema=False),
     )
 
     # Add other global attributes
@@ -691,14 +699,14 @@ def process_swapi_science(
     return dataset
 
 
-def swapi_l1(file_path: str, data_version: str) -> xr.Dataset:
+def swapi_l1(dependencies: list, data_version: str) -> xr.Dataset:
     """
     Will process SWAPI level 0 data to level 1.
 
     Parameters
     ----------
-    file_path : str
-        Path to SWAPI L0 file.
+    dependencies : list
+        Input dependencies needed for L1 processing.
     data_version : str
         Version of the data product being created.
 
@@ -710,27 +718,48 @@ def swapi_l1(file_path: str, data_version: str) -> xr.Dataset:
     xtce_definition = (
         f"{imap_module_directory}/swapi/packet_definitions/swapi_packet_definition.xml"
     )
-    datasets = packet_file_to_datasets(
-        file_path, xtce_definition, use_derived_value=False
-    )
+    l0_unpacked_dict = {}
+    l1_hk_ds = None
+    for file_path in dependencies:
+        if file_path.suffix == ".pkts":
+            l0_unpacked_dict = packet_file_to_datasets(
+                file_path, xtce_definition, use_derived_value=False
+            )
+        if file_path.suffix == ".cdf":
+            l1_hk_ds = load_cdf(file_path)
+
     processed_data = []
 
-    for apid, ds_data in datasets.items():
-        # Right now, we only process SWP_HK and SWP_SCI
-        # other packets are not process in this processing pipeline
-        # If appId is science, then the file should contain all data of science appId
+    # Right now, we only process SWP_HK and SWP_SCI.
+    # Other apId are not processed in this processing pipeline.
+
+    # Len of dependencies is 2 and l0_unpacked_dict[SWAPIAPID.SWP_HK] is not None
+    if (
+        len(dependencies) == 2
+        and l0_unpacked_dict.get(SWAPIAPID.SWP_SCI, None) is not None
+    ):
+        # process science data
+        sci_dataset = process_swapi_science(
+            l0_unpacked_dict[SWAPIAPID.SWP_SCI], l1_hk_ds, data_version
+        )
+        processed_data.append(sci_dataset)
+
+    elif len(dependencies) == 1 and l0_unpacked_dict[SWAPIAPID.SWP_HK]:
+        hk_ds = l0_unpacked_dict[SWAPIAPID.SWP_HK]
+        # Add HK datalevel attrs
+        imap_attrs = ImapCdfAttributes()
+        imap_attrs.add_instrument_global_attrs("swapi")
+        imap_attrs.add_global_attribute("Data_version", data_version)
+        imap_attrs.add_instrument_variable_attrs(instrument="swapi", level=None)
+        hk_ds.attrs.update(imap_attrs.get_global_attributes("imap_swapi_l1_hk"))
+        hk_common_attrs = imap_attrs.get_variable_attributes("hk_attrs")
+        hk_ds["epoch"].attrs.update(
+            imap_attrs.get_variable_attributes("epoch", check_schema=False)
+        )
 
-        if apid == SWAPIAPID.SWP_SCI.value:
-            sci_dataset = process_swapi_science(
-                ds_data, datasets[SWAPIAPID.SWP_HK], data_version
-            )
-            processed_data.append(sci_dataset)
-        if apid == SWAPIAPID.SWP_HK.value:
-            # Add HK datalevel attrs
-            hk_attrs = ImapCdfAttributes()
-            hk_attrs.add_instrument_global_attrs("swapi")
-            hk_attrs.add_global_attribute("Data_version", data_version)
-            ds_data.attrs.update(hk_attrs.get_global_attributes("imap_swapi_l1_hk"))
-            processed_data.append(ds_data)
+        # Add attrs to HK data variables
+        for var_name in hk_ds.data_vars:
+            hk_ds[var_name].attrs.update(hk_common_attrs)
+        processed_data.append(hk_ds)
 
     return processed_data