imap-processing 1.0.35__tar.gz → 1.0.36__tar.gz

{imap_processing-1.0.35 → imap_processing-1.0.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: imap-processing
-Version: 1.0.35
+Version: 1.0.36
 Summary: IMAP Science Operations Center Processing
 License-Expression: MIT
 License-File: LICENSE

{imap_processing-1.0.35 → imap_processing-1.0.36}/imap_processing/_version.py

@@ -1,3 +1,3 @@
 # These version placeholders will be replaced later during substitution.
-__version__ = "1.0.35"
-__version_tuple__ = (1, 0, 35)
+__version__ = "1.0.36"
+__version_tuple__ = (1, 0, 36)

{imap_processing-1.0.35 → imap_processing-1.0.36}/imap_processing/cdf/config/imap_hi_variable_attrs.yaml

@@ -446,8 +446,10 @@ hi_de_ccsds_qf:
   VALIDMIN: 0
   VAR_NOTES: >
     Bitwise quality flag for CCSDS packet. Bit 0 (value 1): packet was full
-    (contained 664 events). Bit 2 (value 4): packet contained events from an
-    invalid spin.
+    (contained 664 events). Bit 1 (value 2): ESA (inner/outer) voltage did not
+    match any known esa_energy_step. Bit 2 (value 4): packet contained events
+    from an invalid spin. Bit 3 (value 8): detector high voltage (CEM/MCP/U-Can/
+    deflector) did not match the pointing's classified gain configuration.
 
 # ======= L1C PSET Section =======
 
@@ -477,6 +479,24 @@ hi_pset_esa_energy_step:
   VAR_TYPE: support_data
   dtype: uint8
 
+hi_pset_geometric_factor:
+  <<: *default_float32
+  CATDESC: Geometric factor for the detector gain configuration active during this pointing
+  DEPEND_0: epoch
+  DEPEND_1: esa_energy_step
+  DISPLAY_TYPE: no_plot
+  FIELDNAM: Geometric factor
+  FORMAT: E12.5
+  LABLAXIS: Geometric Factor
+  LABL_PTR_1: esa_energy_step_label
+  UNITS: cm^2 sr
+  VALIDMAX: 1.0
+  VALIDMIN: 0.0
+  VAR_NOTES: >
+    Geometric factor per esa_energy_step, looked up from the gain-configuration
+    ancillary file using the l1b_de's classified gain configuration.
+  VAR_TYPE: support_data
+
 hi_pset_calibration_prod:
   CATDESC: Coincidence types are combined into various calibration products
   FIELDNAM: Calibration product number

{imap_processing-1.0.35 → imap_processing-1.0.36}/imap_processing/cli.py

@@ -18,6 +18,7 @@ import logging
 import sys
 from abc import ABC, abstractmethod
 from pathlib import Path
+from time import sleep
 from typing import final
 
 import imap_data_access
@@ -53,6 +54,7 @@ from imap_processing.cdf.utils import load_cdf, write_cdf
 # In code:
 #   call cdf.utils.write_cdf
 from imap_processing.codice import codice_l1a, codice_l1b, codice_l2
+from imap_processing.ena_maps.utils.naming import MapDescriptor
 from imap_processing.glows.l1a.glows_l1a import glows_l1a
 from imap_processing.glows.l1b.glows_l1b import glows_l1b, glows_l1b_de
 from imap_processing.glows.l2.glows_l2 import glows_l2
@@ -95,6 +97,9 @@ from imap_processing.utils import (
 
 logger = logging.getLogger(__name__)
 
+RETRY_EXIT_CODE = 75  # Exit code indicating the job should be retried
+# (e.g., imap-data-access 503 SlowDown).
+
 
 def _parse_args() -> argparse.Namespace:
     """
@@ -201,8 +206,9 @@ def _parse_args() -> argparse.Namespace:
         '{"major_version": 2, "minor_version": 1}}'
         "}'"
         "    A path to a JSON file containing this same information may also be"
-        "passed in. If dependency is a string ending in '.json', it will be interpreted"
-        " as such a file path."
+        " passed in. If dependency is a string ending in '.json', it will be"
+        " interpreted as such a file path: an existing local file is used as-is,"
+        " otherwise the file is downloaded from the IMAP SDC."
     )
 
     parser = argparse.ArgumentParser(prog="imap_cli", description=description)
@@ -288,7 +294,10 @@ def _parse_args() -> argparse.Namespace:
         logger.info(
             f"Interpreting dependency argument as a JSON file: {args.dependency}"
         )
-        dependency_filepath = download(args.dependency)
+        if not Path(args.dependency).exists():
+            dependency_filepath = download(args.dependency)
+        else:
+            dependency_filepath = args.dependency
         with open(dependency_filepath) as f:
             args.dependency = f.read()
 
@@ -468,6 +477,8 @@ class ProcessInstrument(ABC):
         """
         Upload data products to the IMAP SDC.
 
+        If a 503 SlowDown error is reported from the Upload API, a retry will be made
+
         Parameters
         ----------
         products : list[Path]
@@ -479,18 +490,42 @@ class ProcessInstrument(ABC):
                 return
 
             for filename in products:
-                try:
-                    logger.info(f"Uploading file: {filename}")
-                    imap_data_access.upload(filename)
-                except IMAPDataAccessError as e:
-                    message = str(e)
-                    if "FileAlreadyExists" in message and "409" in message:
-                        logger.warning("Skipping upload of existing file, %s", filename)
-                        continue
-                    else:
+                max_retries = 3
+
+                for attempt in range(max_retries):
+                    try:
+                        logger.info(f"Uploading file: {filename}")
+                        imap_data_access.upload(filename)
+                        break
+
+                    except IMAPDataAccessError as e:
+                        message = str(e)
+
+                        if "FileAlreadyExists" in message and "409" in message:
+                            logger.warning(
+                                "Skipping upload of existing file, %s", filename
+                            )
+                            break
+                        elif "503" in message and "SlowDown" in message:
+                            if attempt < max_retries - 1:
+                                logger.warning(
+                                    "Upload busy. Waiting 5 seconds before retrying..."
+                                )
+                                sleep(5)
+                                continue
+
+                            logger.error(
+                                f"Upload failed after {max_retries} attempts. Exiting "
+                                f"with code {RETRY_EXIT_CODE}."
+                            )
+                            sys.exit(RETRY_EXIT_CODE)
+
                         logger.error(f"Upload failed with error: {message}")
-                except Exception as e:
-                    logger.error(f"Upload failed unknown error: {e}")
+                        raise
+
+                    except Exception as e:
+                        logger.error(f"Upload failed unknown error: {e}")
+                        raise
 
     @final
     def process(self) -> None:
@@ -937,9 +972,17 @@ class Hi(ProcessInstrument):
                 l1b_hk_file = dependencies.get_file_paths(
                     source="hi", data_type="l1b", descriptor="hk"
                 )[0]
-                esa_energies_csv = dependencies.get_file_paths(data_type="ancillary")[0]
+                esa_energies_csv = dependencies.get_file_paths(
+                    data_type="ancillary", descriptor="esa-energies"
+                )[0]
+                gain_config_csv = dependencies.get_file_paths(
+                    data_type="ancillary", descriptor="gain-configuration"
+                )[0]
                 datasets = hi_l1b.annotate_direct_events(
-                    load_cdf(l1a_de_file), load_cdf(l1b_hk_file), esa_energies_csv
+                    load_cdf(l1a_de_file),
+                    load_cdf(l1b_hk_file),
+                    esa_energies_csv,
+                    gain_config_csv,
                 )
         elif self.data_level == "l1c":
             if "pset" in self.descriptor:
@@ -957,10 +1000,10 @@ class Hi(ProcessInstrument):
                 anc_dependencies = dependencies.get_processing_inputs(
                     data_type="ancillary"
                 )
-                if len(anc_dependencies) != 2:
+                if len(anc_dependencies) != 3:
                     raise ValueError(
-                        f"Expected two ancillary dependencies (cal-prod and "
-                        f"backgrounds). Got "
+                        f"Expected three ancillary dependencies (cal-prod, "
+                        f"backgrounds, and gain-configuration). Got "
                         f"{[anc_dep.descriptor for anc_dep in anc_dependencies]}"
                     )
 
@@ -972,14 +1015,16 @@ class Hi(ProcessInstrument):
                     for dep in anc_dependencies
                 }
 
-                # Verify we have both required ancillary files
+                # Verify we have all required ancillary files
                 if (
                     "cal-prod" not in anc_path_dict
                     or "backgrounds" not in anc_path_dict
+                    or "gain-configuration" not in anc_path_dict
                 ):
                     raise ValueError(
-                        f"Missing required ancillary files. Expected 'cal-prod' and "
-                        f"'backgrounds', got {list(anc_path_dict.keys())}"
+                        f"Missing required ancillary files. Expected 'cal-prod', "
+                        f"'backgrounds', and 'gain-configuration', got "
+                        f"{list(anc_path_dict.keys())}"
                     )
 
                 # Load goodtimes dependency
@@ -997,6 +1042,7 @@ class Hi(ProcessInstrument):
                     anc_path_dict["cal-prod"],
                     load_cdf(goodtimes_paths[0]),
                     anc_path_dict["backgrounds"],
+                    anc_path_dict["gain-configuration"],
                 )
         elif self.data_level == "l2":
             science_paths = dependencies.get_file_paths(source="hi", data_type="l1c")
@@ -1208,52 +1254,109 @@ class Idex(ProcessInstrument):
 class Lo(ProcessInstrument):
     """Process IMAP-Lo."""
 
+    @staticmethod
+    def _pointings_at_pivot_angle(
+        dependencies: ProcessingInputCollection, map_pivot_angle: int
+    ) -> set[int]:
+        """
+        Find the pointings that were taken at the pivot angle of a map.
+
+        A pointing's pivot angle is recorded in its goodtimes product, which is
+        also the product the other inputs of that pointing are selected by.
+
+        Parameters
+        ----------
+        dependencies : ProcessingInputCollection
+            Object containing dependencies to process.
+        map_pivot_angle : int
+            The pivot angle [degrees] of the map being made.
+
+        Returns
+        -------
+        set[int]
+            The repointings whose pivot angle is that of the map.
+        """
+        at_pivot_angle = set()
+        for goodtimes_path in dependencies.get_file_paths(
+            source="lo", descriptor="goodtimes"
+        ):
+            goodtimes = load_cdf(goodtimes_path)
+            repointing = imap_data_access.ScienceFilePath(
+                goodtimes_path.name
+            ).repointing
+            if "pivot" not in goodtimes:
+                logger.info(f"Dropping {goodtimes_path.name} - no pivot angle.")
+                continue
+
+            pivot_angle = np.atleast_1d(goodtimes["pivot"].values)[0]
+            if (
+                abs(pivot_angle - map_pivot_angle)
+                < LoConstants.PSET_PIVOT_ANGLE_TOLERANCE
+            ):
+                at_pivot_angle.add(repointing)
+            else:
+                logger.info(
+                    f"Dropping repoint{repointing}, its pivot angle {pivot_angle} "
+                    f"is not the {map_pivot_angle} degree pivot angle of the map."
+                )
+
+        return at_pivot_angle
+
     def pre_processing(self) -> ProcessingInputCollection:
         """
         Complete pre-processing.
 
-        Extends the base pre-processing by filtering Lo PSET science inputs to
-        only those whose ``pivot_angle`` is within
-        ``LoConstants.PSET_PIVOT_ANGLE_TOLERANCE`` of
-        ``LoConstants.PSET_PIVOT_ANGLE``. PSET files that fall outside this
-        range are dropped before processing begins.
+        Extends the base pre-processing by dropping, for map products, the Lo
+        science inputs of the pointings that were not taken at the pivot angle
+        of the map being made. A pointing is dropped whole: its goodtimes give
+        the pivot angle, and its other inputs go with them.
+
+        Filtering here, rather than during processing, keeps the `Parents`
+        attribute of the produced map limited to the files it was made from.
 
         Returns
         -------
         dependencies : ProcessingInputCollection
             Object containing dependencies to process.
         """
-        datasets = super().pre_processing()
-        new_datasets = ProcessingInputCollection()
+        dependencies = super().pre_processing()
+        if self.data_level != "l2":
+            return dependencies
+
+        try:
+            map_pivot_angle = MapDescriptor.from_string(self.descriptor).sensor
+        except ValueError:
+            # Not a map product, so there is no pivot angle to select inputs with
+            logger.info(
+                f"Not filtering inputs by pivot angle, {self.descriptor} is not a "
+                f"map descriptor."
+            )
+            return dependencies
+
+        if not isinstance(map_pivot_angle, int):
+            # A map of no particular pivot angle, e.g. "ilo-ena-h-sf-nsp-ram-..."
+            return dependencies
+
+        at_pivot_angle = self._pointings_at_pivot_angle(dependencies, map_pivot_angle)
 
-        for processing_input in datasets.get_processing_inputs():
+        filtered_dependencies = ProcessingInputCollection()
+        for processing_input in dependencies.get_processing_inputs():
             if (
-                processing_input.source == "lo"
-                and processing_input.descriptor == "pset"
+                processing_input.input_type != ProcessingInputType.SCIENCE_FILE
+                or processing_input.source != "lo"
             ):
-                valid_filenames = []
-                for imap_file_path in processing_input.imap_file_paths:
-                    pset = load_cdf(imap_file_path.construct_path())
-                    if "pivot_angle" in pset:
-                        if (
-                            abs(
-                                pset["pivot_angle"].item()
-                                - LoConstants.PSET_PIVOT_ANGLE
-                            )
-                            < LoConstants.PSET_PIVOT_ANGLE_TOLERANCE
-                        ):
-                            valid_filenames.append(str(imap_file_path.filename))
-                        else:
-                            logger.info(
-                                f"Dropping pset {imap_file_path.filename} because "
-                                f"pivot angle is not in range."
-                            )
-                if valid_filenames:
-                    new_datasets.add(type(processing_input)(*valid_filenames))
-            else:
-                new_datasets.add(processing_input)
+                filtered_dependencies.add(processing_input)
+                continue
 
-        return new_datasets
+            kept_filenames = [
+                str(imap_file_path.filename)
+                for imap_file_path in processing_input.imap_file_paths
+                if imap_file_path.repointing in at_pivot_angle
+            ]
+            if kept_filenames:
+                filtered_dependencies.add(type(processing_input)(*kept_filenames))
+
+        return filtered_dependencies
 
     def do_processing(
         self, dependencies: ProcessingInputCollection
@@ -1319,18 +1422,26 @@ class Lo(ProcessInstrument):
             datasets = lo_l1c.lo_l1c(data_dict, anc_dependencies)
 
         elif self.data_level == "l2":
-            data_dict = {}
-            science_files = dependencies.get_file_paths(source="lo", descriptor="pset")
             anc_dependencies = dependencies.get_file_paths(data_type="ancillary")
 
-            # Load all pset files into datasets
-            if not science_files:
-                logger.info("No valid psets found for L2 processing.")
-                return datasets
+            # Load every pointing of the map window, grouped into the products
+            # of each pointing.
+            sci_dependencies: dict[int, dict[str, xr.Dataset]] = {}
+            for descriptor in lo_l2.REQUIRED_PRODUCTS:
+                for file in dependencies.get_file_paths(
+                    source="lo", data_type="l1b", descriptor=descriptor
+                ):
+                    repointing = imap_data_access.ScienceFilePath(file.name).repointing
+                    if repointing is None:
+                        logger.warning(
+                            f"Dropping {file.name}, it covers no single pointing."
+                        )
+                        continue
+                    sci_dependencies.setdefault(repointing, {})[descriptor] = load_cdf(
+                        file
+                    )
 
-            psets = [load_cdf(file) for file in science_files]
-            data_dict[psets[0].attrs["Logical_source"]] = psets
-            datasets = lo_l2.lo_l2(data_dict, anc_dependencies, self.descriptor)
+            datasets = lo_l2.lo_l2(sci_dependencies, anc_dependencies, self.descriptor)
         return datasets
 
 

{imap_processing-1.0.35 → imap_processing-1.0.36}/imap_processing/ena_maps/ena_maps.py

@@ -1498,11 +1498,25 @@ class RectangularSkyMap(AbstractSkyMap):
         # Now set global attributes
         map_attrs = cdf_attrs.get_global_attributes(f"imap_{instrument}_{level}_enamap")
         map_attrs["Spacing_degrees"] = str(self.spacing_deg)
-        for key in ["Data_type", "Logical_source", "Logical_source_description"]:
+        for key in ["Data_type", "Logical_source"]:
             map_attrs[key] = map_attrs[key].format(
                 descriptor=descriptor,
                 sensor=sensor,
             )
+        # Use the MapDescriptor to generate the Logical_source_description
+        # when possible, but preserve previous behavior for non-descriptor
+        # strings so later validation still raises the intended errors.
+        try:
+            md = naming.MapDescriptor.from_string(descriptor)
+        except ValueError:
+            map_attrs["Logical_source_description"] = map_attrs[
+                "Logical_source_description"
+            ].format(
+                descriptor=descriptor,
+                sensor=sensor,
+            )
+        else:
+            map_attrs["Logical_source_description"] = md.to_logical_source_description()
         # Always add the following attributes to the map
         map_attrs.update(
             {
@@ -1804,8 +1818,10 @@ class HealpixSkyMap(AbstractSkyMap):
             )
         # Log the mean pixel value and the number of subdivisions for debugging
         logger.debug(
-            f"    Mean pixel value at Number of subdivisions: {num_subdivisions}: "
-            f"array of shape {mean_pixel_value.shape}: {mean_pixel_value}"
+            "    Mean pixel value at Number of subdivisions: %s: array of shape %s: %s",
+            num_subdivisions,
+            mean_pixel_value.shape,
+            mean_pixel_value,
         )
         return mean_pixel_value