imap-processing 0.18.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

imap_processing/hit/l1b/hit_l1b.py

@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
 # TODO review logging levels to use (debug vs. info)
 
 
-def hit_l1b(dependencies: dict) -> list[xr.Dataset]:
+def hit_l1b(dependency: str | xr.Dataset, l1b_descriptor: str) -> xr.Dataset:
     """
     Will process HIT data to L1B.
 
@@ -34,54 +34,56 @@ def hit_l1b(dependencies: dict) -> list[xr.Dataset]:
 
     Parameters
     ----------
-    dependencies : dict
-        Dictionary of dependencies that are L1A xarray datasets
-        for science data and a file path string to an L0 file
-        for housekeeping data.
+    dependency : Union[str, xr.Dataset]
+        Dependency is either an L1A xarray dataset to process
+        science data or a file path string to an L0 file to
+        process housekeeping data.
+    l1b_descriptor : str
+        The descriptor for the L1B dataset to create.
 
     Returns
     -------
-    processed_data : list[xarray.Dataset]
-        List of four L1B datasets.
+    l1b_dataset : xarray.Dataset
+        The processed L1B dataset.
     """
     # Create the attribute manager for this data level
     attr_mgr = get_attribute_manager("l1b")
 
+    l1b_dataset = None
+
     # Create L1B datasets
-    l1b_datasets: list = []
-    if "imap_hit_l0_raw" in dependencies:
+    if l1b_descriptor == "hk":
         # Unpack ccsds file to xarray datasets
-        packet_file = dependencies["imap_hit_l0_raw"]
+        packet_file = dependency
         datasets_by_apid = get_datasets_by_apid(packet_file, derived=True)
-        # TODO: update to raise error after all APIDs are included in the same
-        #  raw files. currently science and housekeeping are in separate files.
         if HitAPID.HIT_HSKP in datasets_by_apid:
             # Process housekeeping to L1B.
-            l1b_datasets.append(
-                process_housekeeping_data(
-                    datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1b_hk"
-                )
+            l1b_dataset = process_housekeeping_data(
+                datasets_by_apid[HitAPID.HIT_HSKP], attr_mgr, "imap_hit_l1b_hk"
             )
             logger.info("HIT L1B housekeeping dataset created")
-    if "imap_hit_l1a_counts" in dependencies:
+    elif l1b_descriptor in ["standard-rates", "summed-rates", "sectored-rates"]:
         # Process science data to L1B datasets
-        l1a_counts_dataset = dependencies["imap_hit_l1a_counts"]
-        l1b_datasets.extend(process_science_data(l1a_counts_dataset, attr_mgr))
-        logger.info("HIT L1B science datasets created")
+        l1b_dataset = process_science_data(dependency, l1b_descriptor, attr_mgr)
+        logger.info("HIT L1B science dataset created")
+    else:
+        logger.error(f"Unsupported descriptor for L1B processing: {l1b_descriptor}")
+        raise ValueError(f"Unsupported descriptor: {l1b_descriptor}")
 
-    return l1b_datasets
+    return l1b_dataset
 
 
 def process_science_data(
-    l1a_counts_dataset: xr.Dataset, attr_mgr: ImapCdfAttributes
-) -> list[xr.Dataset]:
+    l1a_counts_dataset: xr.Dataset, descriptor: str, attr_mgr: ImapCdfAttributes
+) -> xr.Dataset:
     """
     Will create L1B science datasets for CDF products.
 
-    Process L1A raw counts data to create L1B science data for
-    CDF creation. This function will create three L1B science
+    This function processes L1A counts data to L1B science
+    data for CDF creation. There are three L1B science
     datasets: standard rates, summed rates, and sectored rates.
-    It will also update dataset attributes, coordinates and
+    This function creates one dataset based on the descriptor
+    provided. It will also update dataset attributes, coordinates and
     data variable dimensions according to specifications in
     a CDF yaml file.
 
@@ -89,46 +91,44 @@ def process_science_data(
     ----------
     l1a_counts_dataset : xr.Dataset
         The L1A counts dataset.
+    descriptor : str
+        The descriptor for the L1B dataset to create
+        (e.g., "standard-rates", "summed-rates", "sectored-rates").
     attr_mgr : AttributeManager
         The attribute manager for the L1B data level.
 
     Returns
     -------
-    dataset : list
-        The processed L1B science datasets as xarray datasets.
+    dataset : xarray.Dataset
+        A processed L1B science dataset.
     """
     logger.info("Creating HIT L1B science datasets")
 
-    # TODO: Write functions to create the following datasets
-    #  Process sectored rates dataset
+    dataset = None
+    logical_source = None
 
     # Calculate fractional livetime from the livetime counter
     livetime = l1a_counts_dataset["livetime_counter"] / LIVESTIM_PULSES
     livetime = livetime.rename("livetime")
 
-    # Process counts data to L1B datasets
-    l1b_datasets: dict = {
-        "imap_hit_l1b_standard-rates": process_standard_rates_data(
-            l1a_counts_dataset, livetime
-        ),
-        "imap_hit_l1b_summed-rates": process_summed_rates_data(
-            l1a_counts_dataset, livetime
-        ),
-        "imap_hit_l1b_sectored-rates": process_sectored_rates_data(
-            l1a_counts_dataset, livetime
-        ),
-    }
+    # Process counts data to an L1B dataset based on the descriptor
+    if descriptor == "standard-rates":
+        dataset = process_standard_rates_data(l1a_counts_dataset, livetime)
+        logical_source = "imap_hit_l1b_standard-rates"
+    elif descriptor == "summed-rates":
+        dataset = process_summed_rates_data(l1a_counts_dataset, livetime)
+        logical_source = "imap_hit_l1b_summed-rates"
+    elif descriptor == "sectored-rates":
+        dataset = process_sectored_rates_data(l1a_counts_dataset, livetime)
+        logical_source = "imap_hit_l1b_sectored-rates"
 
     # Update attributes and dimensions
-    for logical_source, dataset in l1b_datasets.items():
+    if dataset and logical_source:
         dataset.attrs = attr_mgr.get_global_attributes(logical_source)
-
-        # TODO: Add CDF attributes to yaml once they're defined for L1B science data
-        # Assign attributes and dimensions to each data array in the Dataset
+        # TODO: Add CDF attributes to yaml
         for field in dataset.data_vars.keys():
             try:
-                # Create a dict of dimensions using the DEPEND_I keys in the
-                # attributes
+                # Create a dict of dimensions using the DEPEND_I keys in the attributes
                 dims = {
                     key: value
                     for key, value in attr_mgr.get_variable_attributes(field).items()
@@ -137,7 +137,6 @@ def process_science_data(
                 dataset[field].attrs = attr_mgr.get_variable_attributes(field)
                 dataset[field].assign_coords(dims)
             except KeyError:
-                print(f"Field {field} not found in attribute manager.")
                 logger.warning(f"Field {field} not found in attribute manager.")
 
         # Skip schema check for epoch to prevent attr_mgr from adding the
@@ -145,10 +144,9 @@ def process_science_data(
         dataset.epoch.attrs = attr_mgr.get_variable_attributes(
             "epoch", check_schema=False
         )
-
         logger.info(f"HIT L1B dataset created for {logical_source}")
 
-    return list(l1b_datasets.values())
+    return dataset
 
 
 def initialize_l1b_dataset(l1a_counts_dataset: xr.Dataset, coords: list) -> xr.Dataset:
@@ -359,79 +357,11 @@ def process_summed_rates_data(
     return l1b_summed_rates_dataset
 
 
-def subset_data_for_sectored_counts(
-    l1a_counts_dataset: xr.Dataset, livetime: xr.DataArray
-) -> tuple[xr.Dataset, xr.DataArray]:
-    """
-    Subset data for complete sets of sectored counts and corresponding livetime values.
-
-    A set of sectored data starts with hydrogen and ends with iron and correspond to
-    the mod 10 values 0-9. The livetime values from the previous 10 minutes are used
-    to calculate the rates for each set since those counts are transmitted 10 minutes
-    after they were collected. Therefore, only complete sets of sectored counts where
-    livetime from the previous 10 minutes are available are included in the output.
-
-    Parameters
-    ----------
-    l1a_counts_dataset : xr.Dataset
-        The L1A counts dataset.
-    livetime : xr.DataArray
-        1D array of livetime values calculated from the livetime counter.
-
-    Returns
-    -------
-    tuple[xr.Dataset, xr.DataArray]
-        Dataset of complete sectored counts and corresponding livetime values.
-    """
-    # Identify 10-minute intervals of complete sectored counts.
-    bin_size = 10
-    mod_10 = l1a_counts_dataset.hdr_minute_cnt.values % 10
-    pattern = np.arange(bin_size)
-
-    # Use sliding windows to find pattern matches
-    matches = np.all(
-        np.lib.stride_tricks.sliding_window_view(mod_10, bin_size) == pattern, axis=1
-    )
-    start_indices = np.where(matches)[0]
-
-    # Filter out start indices that are less than or equal to the bin size
-    # since the previous 10 minutes are needed for calculating rates
-    if start_indices.size == 0:
-        logger.error(
-            "No data to process - valid start indices not found for "
-            "complete sectored counts."
-        )
-        raise ValueError("No valid start indices found for complete sectored counts.")
-    else:
-        start_indices = start_indices[start_indices >= bin_size]
-
-    # Subset data for complete sets of sectored counts.
-    # Each set of sectored counts is 10 minutes long, so we take the indices
-    # starting from the start indices and extend to the bin size of 10.
-    # This creates a 1D array of indices that correspond to the complete
-    # sets of sectored counts which is used to filter the L1A dataset and
-    # create the L1B sectored rates dataset.
-    data_indices = np.concatenate(
-        [np.arange(idx, idx + bin_size) for idx in start_indices]
-    )
-    l1b_sectored_rates_dataset = l1a_counts_dataset.isel(epoch=data_indices)
-
-    # Subset livetime values corresponding to the previous 10 minutes
-    # for each start index. This ensures the livetime data aligns correctly
-    # with the sectored counts for rate calculations.
-    livetime_indices = np.concatenate(
-        [np.arange(idx - bin_size, idx) for idx in start_indices]
-    )
-    livetime = livetime.isel(epoch=livetime_indices)
-
-    return l1b_sectored_rates_dataset, livetime
-
-
 def process_sectored_rates_data(
     l1a_counts_dataset: xr.Dataset, livetime: xr.DataArray
 ) -> xr.Dataset:
     """
-    Will process L1B sectored rates data from L1A raw counts data.
+    Will process L1A raw counts data into L1B sectored rates.
 
     A complete set of sectored counts is taken over 10 science frames (10 minutes)
     where each science frame contains counts for one species and energy range.
@@ -451,10 +381,13 @@ def process_sectored_rates_data(
     rotation is split into 15 inclination ranges). See equation 11 in the algorithm
     document.
 
+    NOTE: The L1A counts dataset has complete sets of sectored counts and livetime is
+    already shifted to 10 minutes before the counts. This was handled in L1A processing.
+
     Parameters
     ----------
     l1a_counts_dataset : xr.Dataset
-        The L1A counts dataset.
+        The L1A counts dataset containing sectored counts.
 
     livetime : xr.DataArray
         1D array of livetime values calculated from the livetime counter.
@@ -477,11 +410,6 @@ def process_sectored_rates_data(
         if any(str(var).startswith(f"{p}_") for p in particles)
     ]
 
-    # Subset data for complete sets of sectored counts and corresponding livetime values
-    l1a_counts_dataset, livetime = subset_data_for_sectored_counts(
-        l1a_counts_dataset, livetime
-    )
-
     # Sum livetime over 10 minute intervals
     livetime_10min = sum_livetime_10min(livetime)
 

imap_processing/hit/l2/hit_l2.py

@@ -27,16 +27,16 @@ logger = logging.getLogger(__name__)
 #  - review logging levels to use (debug vs. info)
 
 
-def hit_l2(dependency_sci: xr.Dataset, dependencies_anc: list) -> list[xr.Dataset]:
+def hit_l2(dependency_sci: xr.Dataset, dependencies_anc: list) -> xr.Dataset:
     """
-    Will process HIT data to L2.
+    Will process HIT L1B data to L2.
 
     Processes dependencies needed to create L2 data products.
 
     Parameters
     ----------
     dependency_sci : xr.Dataset
-        L1B xarray science dataset that is either summed rates
+        L1B dataset that is either summed rates
         standard rates or sector rates.
 
     dependencies_anc : list
@@ -44,8 +44,8 @@ def hit_l2(dependency_sci: xr.Dataset, dependencies_anc: list) -> list[xr.Datase
 
     Returns
     -------
-    processed_data : list[xarray.Dataset]
-        List of one L2 dataset.
+    l2_dataset : xarray.Dataset
+        The processed L2 dataset from the dependency dataset provided.
     """
     logger.info("Creating HIT L2 science dataset")
 
@@ -74,7 +74,7 @@ def hit_l2(dependency_sci: xr.Dataset, dependencies_anc: list) -> list[xr.Datase
 
         logger.info(f"HIT L2 dataset created for {logical_source}")
 
-    return [l2_dataset]
+    return l2_dataset
 
 
 def add_cdf_attributes(

imap_processing/ialirt/calculate_ingest.py

@@ -0,0 +1,219 @@
+"""Packet ingest and tcp connection times for each station."""
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from imap_processing.ialirt.constants import STATIONS
+
+logger = logging.getLogger(__name__)
+
+
+def find_tcp_connections(
+    start_file_creation: datetime,
+    end_file_creation: datetime,
+    lines: list,
+    realtime_summary: dict,
+) -> dict:
+    """
+    Find tcp connection time ranges for ground station from log lines.
+
+    Parameters
+    ----------
+    start_file_creation : datetime
+        File creation time of last file minus 48 hrs.
+    end_file_creation : datetime
+        File creation time of last file.
+    lines : list
+        All lines of log files.
+    realtime_summary : dict
+        Input dictionary containing ingest parameters.
+
+    Returns
+    -------
+    realtime_summary : dict
+        Output dictionary with tcp connection info.
+    """
+    current_starts: dict[str, datetime | None] = {}
+
+    for line in lines:
+        if "antenna partner connection is" not in line:
+            continue
+
+        timestamp_str = line.split(" ")[0]
+        msg = " ".join(line.split(" ")[1:])
+        station = msg.split(" antenna")[0]
+
+        if station not in realtime_summary["connection_times"]:
+            realtime_summary["connection_times"][station] = []
+        if station not in realtime_summary["stations"]:
+            realtime_summary["stations"].append(station)
+
+        timestamp = datetime.strptime(timestamp_str, "%Y/%j-%H:%M:%S.%f")
+
+        if f"{station} antenna partner connection is up." in line:
+            current_starts[station] = timestamp
+
+        elif f"{station} antenna partner connection is down!" in line:
+            start = current_starts.get(station)
+            if start is not None:
+                realtime_summary["connection_times"][station].append(
+                    {
+                        "start": datetime.isoformat(start),
+                        "end": datetime.isoformat(timestamp),
+                    }
+                )
+                current_starts[station] = None
+            else:
+                # No matching "up"
+                realtime_summary["connection_times"][station].append(
+                    {
+                        "start": datetime.isoformat(start_file_creation),
+                        "end": datetime.isoformat(timestamp),
+                    }
+                )
+                current_starts[station] = None
+
+    # Handle hanging "up" at the end of file
+    for station, start in current_starts.items():
+        if start is not None:
+            realtime_summary["connection_times"][station].append(
+                {
+                    "start": datetime.isoformat(start),
+                    "end": datetime.isoformat(end_file_creation),
+                }
+            )
+
+    # Filter out connection windows that are completely outside the time window
+    for station in realtime_summary["connection_times"]:
+        realtime_summary["connection_times"][station] = [
+            window
+            for window in realtime_summary["connection_times"][station]
+            if datetime.fromisoformat(window["end"]) >= start_file_creation
+            and datetime.fromisoformat(window["start"]) <= end_file_creation
+        ]
+
+    return realtime_summary
+
+
+def packets_created(start_file_creation: datetime, lines: list) -> list:
+    """
+    Find timestamps when packets were created based on log lines.
+
+    Parameters
+    ----------
+    start_file_creation : datetime
+        File creation time of last file minus 48 hrs.
+    lines : list
+        All lines of log files.
+
+    Returns
+    -------
+    packet_times : list
+        List of datetime objects when packets were created.
+    """
+    packet_times = []
+
+    for line in lines:
+        if "Renamed iois_1_packets" in line:
+            timestamp_str = line.split(" ")[0]
+            timestamp = datetime.strptime(timestamp_str, "%Y/%j-%H:%M:%S.%f")
+            # Possible that data extends further than 48 hrs in the past.
+            if timestamp >= start_file_creation:
+                packet_times.append(timestamp)
+
+    return packet_times
+
+
+def format_ingest_data(last_filename: str, log_lines: list) -> dict:
+    """
+    Format TCP connection and packet ingest data from multiple log files.
+
+    Parameters
+    ----------
+    last_filename : str
+        Log file that is last chronologically.
+    log_lines : list[str]
+        Combined lines from all log files (assumed already sorted by time).
+
+    Returns
+    -------
+    realtime_summary : dict
+        Structured output with TCP connection windows per station
+        and global packet ingest timestamps.
+
+    Notes
+    -----
+    Example output:
+    {
+      "summary": "I-ALiRT Real-time Ingest Summary",
+      "generated": "2025-08-07T21:36:09Z",
+      "time_format": "UTC (ISOC)",
+      "stations": [
+        "Kiel"
+      ],
+      "time_range": [
+        "2025-07-30T23:00:00",
+        "2025-07-31T02:00:00"
+      ],
+      "packet_ingest": [
+        "2025-07-31T00:00:00",
+        "2025-07-31T02:01:00"
+      ],
+      "connection_times": {
+        "Kiel": [
+          {
+            "start": "2025-07-30T23:00:00",
+            "end": "2025-07-31T00:15:00"
+          },
+          {
+            "start": "2025-07-31T02:00:00",
+            "end": "2025-07-31T02:00:00"
+          }
+        ]
+      }
+    }
+
+    where time_range is the overall time range of the data,
+    packet_ingest contains timestamps when packets were finalized,
+    and tcp contains connection windows for each station.
+    """
+    # File creation time.
+    last_timestamp_str = last_filename.split(".")[2]
+    last_timestamp_str = last_timestamp_str.replace("_", ":")
+    end_of_time = datetime.strptime(last_timestamp_str, "%Y-%jT%H:%M:%S")
+
+    # File creation time of last file minus 48 hrs.
+    start_of_time = datetime.strptime(last_timestamp_str, "%Y-%jT%H:%M:%S") - timedelta(
+        hours=48
+    )
+
+    realtime_summary: dict[str, Any] = {
+        "summary": "I-ALiRT Real-time Ingest Summary",
+        "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "time_format": "UTC (ISOC)",
+        "stations": list(STATIONS),
+        "time_range": [
+            start_of_time.isoformat(),
+            end_of_time.isoformat(),
+        ],  # Overall time range of the data
+        "packet_ingest": [],  # Global packet ingest times
+        "connection_times": {
+            station: [] for station in list(STATIONS)
+        },  # Per-station TCP connection windows
+    }
+
+    # TCP connection data for each station
+    realtime_summary = find_tcp_connections(
+        start_of_time, end_of_time, log_lines, realtime_summary
+    )
+
+    # Global packet ingest timestamps
+    packet_times = packets_created(start_of_time, log_lines)
+    realtime_summary["packet_ingest"] = [
+        pkt_time.isoformat() for pkt_time in packet_times
+    ]
+
+    logger.info(f"Created ingest files for {realtime_summary['time_range']}")
+
+    return realtime_summary

imap_processing/ialirt/constants.py

@@ -33,10 +33,12 @@ class IalirtSwapiConstants:
     boltz = 1.380649e-23  # Boltzmann constant, J/K
     at_mass = 1.6605390666e-27  # atomic mass, kg
     prot_mass = 1.007276466621 * at_mass  # mass of proton, kg
-    eff_area = 3.3e-5 * 1e-4  # effective area, meters squared
+    eff_area = 1.633e-4 * 1e-4  # effective area, cm2 to meters squared
     az_fov = np.deg2rad(30)  # azimuthal width of the field of view, radians
     fwhm_width = 0.085  # FWHM of energy width
     speed_ew = 0.5 * fwhm_width  # speed width of energy passband
+    e_charge = 1.602176634e-19  # electronic charge, [C]
+    speed_coeff = np.sqrt(2 * e_charge / prot_mass) / 1e3
 
 
 class StationProperties(NamedTuple):
@@ -48,7 +50,15 @@ class StationProperties(NamedTuple):
     min_elevation_deg: float  # minimum elevation angle in degrees
 
 
-# Verified by Kiel Observatory staff.
+# Verified by Kiel and KSWC Observatory staff.
+# Notes: the KSWC station is not yet operational,
+# but will have the following properties:
+# "KSWC": StationProperties(
+#     longitude=126.2958,  # degrees East
+#     latitude=33.4273,  # degrees North
+#     altitude=0.1,  # approx 100 meters
+#     min_elevation_deg=5,  # 5 degrees is the requirement
+# ),
 STATIONS = {
     "Kiel": StationProperties(
         longitude=10.1808,  # degrees East

imap_processing/ialirt/generate_coverage.py

@@ -11,8 +11,21 @@ from imap_processing.spice.time import et_to_utc, str_to_et
 # Logger setup
 logger = logging.getLogger(__name__)
 
-# TODO: get a list of all potential DSN stations.
-ALL_STATIONS = [*STATIONS.keys(), "DSS-55", "DSS-56", "DSS-74", "DSS-75"]
+ALL_STATIONS = [
+    *STATIONS.keys(),
+    "DSS-24",
+    "DSS-25",
+    "DSS-26",
+    "DSS-34",
+    "DSS-35",
+    "DSS-36",
+    "DSS-53",
+    "DSS-54",
+    "DSS-55",
+    "DSS-56",
+    "DSS-74",
+    "DSS-75",
+]
 
 
 def generate_coverage(

imap_processing/ialirt/l0/ialirt_spice.py

@@ -56,7 +56,10 @@ def get_rotation_matrix(axis: NDArray, angle: NDArray) -> NDArray:
     """
     angle_rad = np.radians(angle)
     rot_matrices = np.array(
-        [spice.axisar(z, float(phase)) for z, phase in zip(axis, angle_rad)]
+        [
+            spice.axisar(z, float(phase))
+            for z, phase in zip(axis, angle_rad, strict=False)
+        ]
     )
 
     return rot_matrices
@@ -130,6 +133,7 @@ def transform_instrument_vectors_to_inertial(
     spin_phase: NDArray,
     sc_inertial_right: NDArray,
     sc_inertial_decline: NDArray,
+    # TODO: Use correct IMAP_MAG_I or IMAP_MAG_O frame here
     instrument_frame: SpiceFrame = SpiceFrame.IMAP_MAG,
     spacecraft_frame: SpiceFrame = SpiceFrame.IMAP_SPACECRAFT,
 ) -> NDArray:
@@ -185,7 +189,7 @@ def transform_instrument_vectors_to_inertial(
     vectors = np.array(
         [
             spice.mxv(rot.T.copy(), vec)
-            for rot, vec in zip(total_rotations, instrument_vectors)
+            for rot, vec in zip(total_rotations, instrument_vectors, strict=False)
         ]
     )