imap-processing 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

imap_processing/cdf/config/imap_swe_global_cdf_attrs.yaml

@@ -12,28 +12,34 @@ imap_swe_l1a_sci:
   <<: *instrument_base
   Data_type: L1A_SCI>Level-1A Science data
   Logical_source: imap_swe_l1a_sci
-  Logical_source_description: SWE Instrument Level-1A Science Data
+  Logical_source_description: IMAP SWE Instrument Level-1A Science Data
 
 imap_swe_l1a_hk:
   <<: *instrument_base
   Data_type: L1A_HK>Level-1A Housekeeping data
   Logical_source: imap_swe_l1a_hk
-  Logical_source_description: SWE Instrument Level-1A Housekeeping Data
+  Logical_source_description: IMAP SWE Instrument Level-1A Housekeeping Data
 
 imap_swe_l1a_cem-raw:
   <<: *instrument_base
   Data_type: L1A_CEM-RAW>Level-1A CEM Raw data
   Logical_source: imap_swe_l1a_cem-raw
-  Logical_source_description: SWE Instrument Level-1A CEM Raw Data
+  Logical_source_description: IMAP SWE Instrument Level-1A CEM Raw Data
 
 imap_swe_l1b_sci:
   <<: *instrument_base
   Data_type: L1B_SCI>Level-1B Science data
   Logical_source: imap_swe_l1b_sci
-  Logical_source_description: SWE Instrument Level-1B Science Data
+  Logical_source_description: IMAP SWE Instrument Level-1B Science Data
+
+imap_swe_l1b_hk:
+  <<: *instrument_base
+  Data_type: L1B_HK>Level-1B Housekeeping data
+  Logical_source: imap_swe_l1b_hk
+  Logical_source_description: IMAP SWE Instrument Level-1B Housekeeping Data
 
 imap_swe_l2_sci:
   <<: *instrument_base
   Data_type: L2_SCI>Level-2 Science data
   Logical_source: imap_swe_l2_sci
-  Logical_source_description: SWE Instrument Level-2 Science Data
+  Logical_source_description: IMAP SWE Instrument Level-2 Science Data

imap_processing/cdf/config/imap_swe_l1b_variable_attrs.yaml

@@ -299,4 +299,26 @@ cksum:
   CATDESC: Checksum
   FIELDNAM: Checksum
   FORMAT: I5
-  VALIDMAX: 65535
+  VALIDMAX: 65535
+
+# <=== HK Variables ===>
+# L1B HK data attrs for data with string values
+l1b_hk_string_attrs:
+  CATDESC: Housekeeping derived data
+  FIELDNAM: Housekeeing Human Readable State
+  FORMAT: A80
+  VAR_TYPE: metadata
+  DEPEND_0: epoch
+
+l1b_hk_attrs:
+  CATDESC: SWE HK data
+  FIELDNAM: SWE Housekeeping Data
+  LABLAXIS: Values
+  DEPEND_0: epoch
+  FILLVAL: -9223372036854775808
+  FORMAT: I19
+  UNITS: ' '
+  VALIDMIN: 0
+  VALIDMAX: 9223372036854769664
+  VAR_TYPE: support_data
+  DISPLAY_TYPE: time_series

imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml

@@ -334,6 +334,7 @@ spin_start_time:
   LABLAXIS: spin start time
   # TODO: come back to format
   UNITS: s
+  DEPEND_0: spin_number
 
 spin_period:
   <<: *default
@@ -341,6 +342,7 @@ spin_period:
   FIELDNAM: spin_period
   LABLAXIS: spin_period
   UNITS: s
+  DEPEND_0: spin_number
 
 spin_rate:
   <<: *default
@@ -348,6 +350,7 @@ spin_rate:
   FIELDNAM: spin_rate
   LABLAXIS: spin_rate
   UNITS: rpm
+  DEPEND_0: spin_number
 
 rate_start_pulses:
   <<: *default
@@ -424,6 +427,7 @@ quality_attitude:
   LABLAXIS: quality attitude
   # TODO: come back to format
   UNITS: " "
+  DEPEND_0: spin_number
 
 quality_instruments:
   <<: *default_uint16

imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml

@@ -48,7 +48,7 @@ exposure_factor:
   UNITS: seconds
 
 sensitivity:
-  <<: *default
+  <<: *default_float32
   CATDESC: Calibration/sensitivity factor.
   FIELDNAM: sensitivity
   LABLAXIS: sensitivity
@@ -90,7 +90,7 @@ shcoarse:
 energy_bin_delta:
   <<: *default_float32
   CATDESC: Difference between the energy bin edges.
-  DEPEND_0: energy_bin_geometric_mean
+  DEPEND_1: energy_bin_geometric_mean
   FIELDNAM: energy_bin_delta
   LABLAXIS: energy bin delta
   UNITS: keV

imap_processing/cli.py

@@ -47,14 +47,11 @@ from imap_processing.cdf.utils import load_cdf, write_cdf
 #   from imap_processing import cdf
 # In code:
 #   call cdf.utils.write_cdf
-from imap_processing.codice import codice_l1a, codice_l1b
+from imap_processing.codice import codice_l1a, codice_l1b, codice_l2
 from imap_processing.glows.l1a.glows_l1a import glows_l1a
 from imap_processing.glows.l1b.glows_l1b import glows_l1b
 from imap_processing.glows.l2.glows_l2 import glows_l2
-from imap_processing.hi.l1a import hi_l1a
-from imap_processing.hi.l1b import hi_l1b
-from imap_processing.hi.l1c import hi_l1c
-from imap_processing.hi.l2 import hi_l2
+from imap_processing.hi import hi_l1a, hi_l1b, hi_l1c, hi_l2
 from imap_processing.hit.l1a.hit_l1a import hit_l1a
 from imap_processing.hit.l1b.hit_l1b import hit_l1b
 from imap_processing.hit.l2.hit_l2 import hit_l2
@@ -66,6 +63,8 @@ from imap_processing.idex.idex_l2c import idex_l2c
 from imap_processing.lo.l1a import lo_l1a
 from imap_processing.lo.l1b import lo_l1b
 from imap_processing.lo.l1c import lo_l1c
+from imap_processing.lo.l2 import lo_l2
+from imap_processing.mag.constants import DataMode
 from imap_processing.mag.l1a.mag_l1a import mag_l1a
 from imap_processing.mag.l1b.mag_l1b import mag_l1b
 from imap_processing.mag.l1c.mag_l1c import mag_l1c
@@ -77,6 +76,7 @@ from imap_processing.swapi.l2.swapi_l2 import swapi_l2
 from imap_processing.swapi.swapi_utils import read_swapi_lut_table
 from imap_processing.swe.l1a.swe_l1a import swe_l1a
 from imap_processing.swe.l1b.swe_l1b import swe_l1b
+from imap_processing.swe.l2.swe_l2 import swe_l2
 from imap_processing.ultra.l1a import ultra_l1a
 from imap_processing.ultra.l1b import ultra_l1b
 from imap_processing.ultra.l1c import ultra_l1c
@@ -416,6 +416,7 @@ class ProcessInstrument(ABC):
                     repointing=file_path.repointing,
                     version=file_path.version,
                     extension="cdf",
+                    table="science",
                 )
                 if existing_file:
                     raise ProcessInstrument.ImapFileExistsError(
@@ -610,27 +611,34 @@ class Codice(ProcessInstrument):
         print(f"Processing CoDICE {self.data_level}")
         datasets: list[xr.Dataset] = []
 
-        dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="codice")
+            if len(science_files) != 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for CoDICE L1a:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"CoDICE L1A requires exactly one input science file, received: "
+                    f"{science_files}."
                 )
             # process data
-            science_files = dependencies.get_file_paths(source="codice")
             datasets = codice_l1a.process_codice_l1a(science_files[0])
 
         if self.data_level == "l1b":
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="codice")
+            if len(science_files) != 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for CoDICE L1b:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"CoDICE L1B requires exactly one input science file, received: "
+                    f"{science_files}."
                 )
             # process data
+            datasets = [codice_l1b.process_codice_l1b(science_files[0])]
+
+        if self.data_level == "l2":
             science_files = dependencies.get_file_paths(source="codice")
-            dependency = load_cdf(science_files[0])
-            datasets = [codice_l1b.process_codice_l1b(dependency)]
+            if len(science_files) != 1:
+                raise ValueError(
+                    f"CoDICE L2 requires exactly one input science file, received: "
+                    f"{science_files}."
+                )
+            datasets = [codice_l2.process_codice_l2(science_files[0])]
 
         return datasets
 
@@ -657,33 +665,32 @@ class Glows(ProcessInstrument):
         print(f"Processing GLOWS {self.data_level}")
         datasets: list[xr.Dataset] = []
 
-        dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="glows")
+            if len(science_files) != 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for GLOWS L1A:"
-                    f"{dependency_list}. Expected only one input dependency."
+                    f"GLOWS L1A requires exactly one input science file, received: "
+                    f"{science_files}."
                 )
-            science_files = dependencies.get_file_paths(source="glows")
             datasets = glows_l1a(science_files[0])
 
         if self.data_level == "l1b":
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="glows")
+            if len(science_files) != 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for GLOWS L1B:"
-                    f"{dependency_list}. Expected at least one input dependency."
+                    f"GLOWS L1A requires exactly one input science file, received: "
+                    f"{science_files}."
                 )
-            science_files = dependencies.get_file_paths(source="glows")
             input_dataset = load_cdf(science_files[0])
             datasets = [glows_l1b(input_dataset)]
 
         if self.data_level == "l2":
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="glows")
+            if len(science_files) != 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for GLOWS L2:"
-                    f"{dependency_list}. Expected only one input dependency."
+                    f"GLOWS L1A requires exactly one input science file, received: "
+                    f"{science_files}."
                 )
-            science_files = dependencies.get_file_paths(source="glows")
             input_dataset = load_cdf(science_files[0])
             datasets = glows_l2(input_dataset)
 
@@ -781,25 +788,23 @@ class Hit(ProcessInstrument):
 
         dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
-            if len(dependency_list) > 1:
+            # 1 science files and 2 spice files
+            if len(dependency_list) > 3:
                 raise ValueError(
                     f"Unexpected dependencies found for HIT L1A:"
                     f"{dependency_list}. Expected only one dependency."
                 )
             # process data to L1A products
-            science_files = dependencies.get_file_paths(source="hit")
+            science_files = dependencies.get_file_paths(source="hit", descriptor="raw")
             datasets = hit_l1a(science_files[0])
 
         elif self.data_level == "l1b":
-            if len(dependency_list) > 1:
-                raise ValueError(
-                    f"Unexpected dependencies found for HIT L1B:"
-                    f"{dependency_list}. Expected only one dependency."
-                )
             data_dict = {}
-            # TODO: Check this and update with new features as needed.
+            # TODO: Sean removed the file number error handling to work with the
+            #  new SPICE dependencies for SIT-4. Need to review and make changes
+            #  if needed.
             l0_files = dependencies.get_file_paths(source="hit", descriptor="raw")
-            l1a_files = dependencies.get_file_paths(source="hit")
+            l1a_files = dependencies.get_file_paths(source="hit", data_type="l1a")
             if len(l0_files) > 0:
                 # Add path to CCSDS file to process housekeeping
                 data_dict["imap_hit_l0_raw"] = l0_files[0]
@@ -938,42 +943,50 @@ class Lo(ProcessInstrument):
         """
         print(f"Processing IMAP-Lo {self.data_level}")
         datasets: list[xr.Dataset] = []
-        dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
             # L1A packet / products are 1 to 1. Should only have
             # one dependency file
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="lo", data_type="l0")
+            if len(science_files) > 1:
                 raise ValueError(
                     f"Unexpected dependencies found for IMAP-Lo L1A:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"{science_files}. Expected only one dependency."
                 )
-            science_files = dependencies.get_file_paths(source="lo")
             datasets = lo_l1a.lo_l1a(science_files[0])
 
         elif self.data_level == "l1b":
             data_dict = {}
-            # TODO: Check this and update with new features as needed.
-            for input_type in dependencies.processing_input:
-                science_files = dependencies.get_file_paths(
-                    source="lo", descriptor=input_type.descriptor
-                )
-                dataset = load_cdf(science_files[0])
+            science_files = dependencies.get_file_paths(source="lo", data_type="l1a")
+            logger.info(f"Science files for L1B: {science_files}")
+            for file in science_files:
+                dataset = load_cdf(file)
                 data_dict[dataset.attrs["Logical_source"]] = dataset
             datasets = lo_l1b.lo_l1b(data_dict)
 
         elif self.data_level == "l1c":
             data_dict = {}
-            for input_type in dependencies.processing_input:
-                science_files = dependencies.get_file_paths(
-                    source="lo", descriptor=input_type.descriptor
-                )
-                dataset = load_cdf(science_files[0])
+            anc_dependencies: list = dependencies.get_file_paths(
+                source="lo", descriptor="goodtimes"
+            )
+            science_files = dependencies.get_file_paths(source="lo", descriptor="de")
+            for file in science_files:
+                dataset = load_cdf(file)
                 data_dict[dataset.attrs["Logical_source"]] = dataset
-                # TODO: add dependencies to S3 and dependency tree
-                #  setting to empty for now
-            anc_depedencies: list = []
-            datasets = lo_l1c.lo_l1c(data_dict, anc_depedencies)
+            datasets = lo_l1c.lo_l1c(data_dict, anc_dependencies)
 
+        elif self.data_level == "l2":
+            data_dict = {}
+            # TODO: Add ancillary descriptors when maps using them are
+            #  implemented.
+            anc_dependencies = dependencies.get_file_paths(
+                source="lo",
+            )
+            science_files = dependencies.get_file_paths(source="lo", descriptor="pset")
+            psets = []
+            for file in science_files:
+                psets.append(load_cdf(file))
+            data_dict[psets[0].attrs["Logical_source"]] = psets
+            datasets = lo_l2.lo_l2(data_dict, anc_dependencies)
         return datasets
 
 
@@ -1071,12 +1084,14 @@ class Mag(ProcessInstrument):
             # TODO: Ensure that parent_files attribute works with that
             input_data = load_cdf(science_files[0])
 
+            descriptor_no_frame = str.split(self.descriptor, "-")[0]
+
             # We expect either a norm or a burst input descriptor.
-            offsets_desc = f"l2-offsets-{self.descriptor}"
+            offsets_desc = f"l2-{descriptor_no_frame}-offsets"
             offsets = dependencies.get_processing_inputs(descriptor=offsets_desc)
 
             calibration = dependencies.get_processing_inputs(
-                descriptor="l2-calibration-matrices"
+                descriptor="l2-calibration"
             )
 
             if (
@@ -1104,6 +1119,7 @@ class Mag(ProcessInstrument):
                 offset_dataset,
                 input_data,
                 current_day,
+                mode=DataMode(descriptor_no_frame.upper()),
             )
 
         return datasets
@@ -1254,17 +1270,20 @@ class Swe(ProcessInstrument):
                 raise ValueError(
                     f"Unexpected dependencies found for SWE L1A:"
                     f"{dependency_list}. Expected only two dependencies."
+                    "L0 data and time kernels."
                 )
             science_files = dependencies.get_file_paths(source="swe")
             datasets = swe_l1a(str(science_files[0]))
             # Right now, we only process science data. Therefore,
             # we expect only one dataset to be returned.
 
-        elif self.data_level == "l1b":
-            if len(dependency_list) != 4:
+        elif self.data_level == "l1b" and self.descriptor == "sci":
+            if len(dependency_list) != 5:
                 raise ValueError(
                     f"Unexpected dependencies found for SWE L1B:"
-                    f"{dependency_list}. Expected exactly four dependencies."
+                    f"{dependency_list}. Expected exactly five dependencies."
+                    "L1A science, in-fligth cal, esa LUT, EU conversion and "
+                    "time kernels."
                 )
 
             science_files = dependencies.get_file_paths("swe", "sci")
@@ -1274,6 +1293,30 @@ class Swe(ProcessInstrument):
                 )
 
             datasets = swe_l1b(dependencies)
+        elif self.data_level == "l1b" and self.descriptor == "hk":
+            if len(dependency_list) != 2:
+                raise ValueError(
+                    f"Unexpected dependencies found for SWE L1B HK:"
+                    f"{dependency_list}. Expected exactly two dependencies."
+                    "L0 data and time kernels."
+                )
+            # process data
+            datasets = swe_l1b(dependencies)
+        elif self.data_level == "l2":
+            if len(dependency_list) != 2:
+                raise ValueError(
+                    f"Unexpected dependencies found for SWE L2:"
+                    f"{dependency_list}. Expected exactly two dependencies."
+                    "L1B science and spin data."
+                )
+            # process data
+            science_files = dependencies.get_file_paths(source="swe", descriptor="sci")
+            if len(science_files) > 1:
+                raise ValueError(
+                    "Multiple science files processing is not supported for SWE L2."
+                )
+            l1b_datasets = load_cdf(science_files[0])
+            datasets = [swe_l2(l1b_datasets)]
         else:
             print("Did not recognize data level. No processing done.")
 
@@ -1302,31 +1345,53 @@ class Ultra(ProcessInstrument):
         print(f"Processing IMAP-Ultra {self.data_level}")
         datasets: list[xr.Dataset] = []
 
-        dependency_list = dependencies.processing_input
         if self.data_level == "l1a":
-            # File path is expected output file path
-            if len(dependency_list) > 1:
+            science_files = dependencies.get_file_paths(source="ultra")
+            if len(science_files) != 1:
                 raise ValueError(
-                    f"Unexpected dependencies found for ULTRA L1A:"
-                    f"{dependency_list}. Expected only one dependency."
+                    f"Unexpected science_files found for ULTRA L1A:"
+                    f"{science_files}. Expected only one dependency."
                 )
-            science_files = dependencies.get_file_paths(source="ultra")
             datasets = ultra_l1a.ultra_l1a(science_files[0])
-
         elif self.data_level == "l1b":
-            data_dict = {}
-            for dep in dependency_list:
-                dataset = load_cdf(dep.imap_file_paths[0])
-                data_dict[dataset.attrs["Logical_source"]] = dataset
-            datasets = ultra_l1b.ultra_l1b(data_dict)
-
+            science_files = dependencies.get_file_paths(source="ultra", data_type="l1a")
+            l1a_dict = {
+                dataset.attrs["Logical_source"]: dataset
+                for dataset in [load_cdf(sci_file) for sci_file in science_files]
+            }
+            science_files = dependencies.get_file_paths(source="ultra", data_type="l1b")
+            l1b_dict = {
+                dataset.attrs["Logical_source"]: dataset
+                for dataset in [load_cdf(sci_file) for sci_file in science_files]
+            }
+            combined = {**l1a_dict, **l1b_dict}
+            anc_paths = dependencies.get_file_paths(data_type="ancillary")
+            ancillary_files = {}
+            for path in anc_paths:
+                ancillary_files[path.stem.split("_")[2]] = path
+            datasets = ultra_l1b.ultra_l1b(combined, ancillary_files)
         elif self.data_level == "l1c":
-            data_dict = {}
-            for dep in dependency_list:
-                dataset = load_cdf(dep.imap_file_paths[0])
-                data_dict[dataset.attrs["Logical_source"]] = dataset
-            datasets = ultra_l1c.ultra_l1c(data_dict)
-
+            science_files = dependencies.get_file_paths(source="ultra", data_type="l1a")
+            l1a_dict = {
+                dataset.attrs["Logical_source"]: dataset
+                for dataset in [load_cdf(sci_file) for sci_file in science_files]
+            }
+            science_files = dependencies.get_file_paths(source="ultra", data_type="l1b")
+            l1b_dict = {
+                dataset.attrs["Logical_source"]: dataset
+                for dataset in [load_cdf(sci_file) for sci_file in science_files]
+            }
+            combined = {**l1a_dict, **l1b_dict}
+            anc_paths = dependencies.get_file_paths(data_type="ancillary")
+            ancillary_files = {}
+            for path in anc_paths:
+                ancillary_files[path.stem.split("_")[2]] = path
+            spice_paths = dependencies.get_file_paths(data_type="spice")
+            if spice_paths:
+                has_spice = True
+            else:
+                has_spice = False
+            datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, has_spice)
         elif self.data_level == "l2":
             all_pset_filepaths = dependencies.get_file_paths(
                 source="ultra", descriptor="pset"