imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (172) hide show
  1. imap_processing/__init__.py +1 -1
  2. imap_processing/_version.py +2 -2
  3. imap_processing/ccsds/excel_to_xtce.py +36 -2
  4. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
  5. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
  6. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
  7. imap_processing/cdf/config/imap_hi_variable_attrs.yaml +136 -9
  8. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +14 -0
  9. imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +63 -1
  10. imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
  11. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +14 -7
  12. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +577 -235
  13. imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +326 -0
  14. imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +33 -23
  15. imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
  16. imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
  17. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +137 -79
  18. imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
  19. imap_processing/cdf/imap_cdf_manager.py +31 -27
  20. imap_processing/cdf/utils.py +3 -5
  21. imap_processing/cli.py +25 -14
  22. imap_processing/codice/codice_l1a.py +153 -63
  23. imap_processing/codice/constants.py +10 -10
  24. imap_processing/codice/decompress.py +10 -11
  25. imap_processing/codice/utils.py +1 -0
  26. imap_processing/glows/l1a/glows_l1a.py +1 -2
  27. imap_processing/glows/l1b/glows_l1b.py +3 -3
  28. imap_processing/glows/l1b/glows_l1b_data.py +59 -37
  29. imap_processing/glows/l2/glows_l2_data.py +123 -0
  30. imap_processing/hi/l1a/hi_l1a.py +4 -4
  31. imap_processing/hi/l1a/histogram.py +107 -109
  32. imap_processing/hi/l1a/science_direct_event.py +92 -225
  33. imap_processing/hi/l1b/hi_l1b.py +85 -11
  34. imap_processing/hi/l1c/hi_l1c.py +23 -1
  35. imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +3994 -0
  36. imap_processing/hi/utils.py +1 -1
  37. imap_processing/hit/hit_utils.py +221 -0
  38. imap_processing/hit/l0/constants.py +118 -0
  39. imap_processing/hit/l0/decom_hit.py +100 -156
  40. imap_processing/hit/l1a/hit_l1a.py +170 -184
  41. imap_processing/hit/l1b/hit_l1b.py +33 -153
  42. imap_processing/ialirt/l0/process_codicelo.py +153 -0
  43. imap_processing/ialirt/l0/process_hit.py +5 -5
  44. imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +281 -0
  45. imap_processing/ialirt/process_ephemeris.py +212 -0
  46. imap_processing/idex/idex_l1a.py +65 -84
  47. imap_processing/idex/idex_l1b.py +192 -0
  48. imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -0
  49. imap_processing/idex/packet_definitions/idex_packet_definition.xml +97 -595
  50. imap_processing/lo/l0/decompression_tables/decompression_tables.py +17 -1
  51. imap_processing/lo/l0/lo_science.py +45 -13
  52. imap_processing/lo/l1a/lo_l1a.py +76 -8
  53. imap_processing/lo/packet_definitions/lo_xtce.xml +8344 -1849
  54. imap_processing/mag/l0/decom_mag.py +4 -3
  55. imap_processing/mag/l1a/mag_l1a.py +12 -13
  56. imap_processing/mag/l1a/mag_l1a_data.py +1 -2
  57. imap_processing/mag/l1b/mag_l1b.py +90 -7
  58. imap_processing/spice/geometry.py +156 -16
  59. imap_processing/spice/time.py +144 -2
  60. imap_processing/swapi/l1/swapi_l1.py +4 -4
  61. imap_processing/swapi/l2/swapi_l2.py +1 -1
  62. imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +1535 -446
  63. imap_processing/swe/l1b/swe_l1b_science.py +8 -8
  64. imap_processing/swe/l2/swe_l2.py +134 -17
  65. imap_processing/tests/ccsds/test_data/expected_output.xml +2 -1
  66. imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
  67. imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
  68. imap_processing/tests/codice/conftest.py +1 -17
  69. imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
  70. imap_processing/tests/codice/test_codice_l0.py +8 -2
  71. imap_processing/tests/codice/test_codice_l1a.py +127 -107
  72. imap_processing/tests/codice/test_codice_l1b.py +1 -0
  73. imap_processing/tests/codice/test_decompress.py +7 -7
  74. imap_processing/tests/conftest.py +100 -58
  75. imap_processing/tests/glows/conftest.py +6 -0
  76. imap_processing/tests/glows/test_glows_l1b.py +9 -9
  77. imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
  78. imap_processing/tests/hi/test_data/l0/H90_NHK_20241104.bin +0 -0
  79. imap_processing/tests/hi/test_data/l0/H90_sci_cnt_20241104.bin +0 -0
  80. imap_processing/tests/hi/test_data/l0/H90_sci_de_20241104.bin +0 -0
  81. imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
  82. imap_processing/tests/hi/test_hi_l1b.py +73 -3
  83. imap_processing/tests/hi/test_hi_l1c.py +10 -2
  84. imap_processing/tests/hi/test_l1a.py +31 -58
  85. imap_processing/tests/hi/test_science_direct_event.py +58 -0
  86. imap_processing/tests/hi/test_utils.py +4 -3
  87. imap_processing/tests/hit/test_data/sci_sample1.ccsds +0 -0
  88. imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +95 -36
  89. imap_processing/tests/hit/test_hit_l1a.py +299 -179
  90. imap_processing/tests/hit/test_hit_l1b.py +231 -24
  91. imap_processing/tests/hit/test_hit_utils.py +218 -0
  92. imap_processing/tests/hit/validation_data/hskp_sample_eu.csv +89 -0
  93. imap_processing/tests/hit/validation_data/sci_sample_raw1.csv +29 -0
  94. imap_processing/tests/ialirt/test_data/l0/apid01152.tlm +0 -0
  95. imap_processing/tests/ialirt/test_data/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
  96. imap_processing/tests/ialirt/unit/test_process_codicelo.py +106 -0
  97. imap_processing/tests/ialirt/unit/test_process_ephemeris.py +109 -0
  98. imap_processing/tests/ialirt/unit/test_process_hit.py +9 -6
  99. imap_processing/tests/idex/conftest.py +2 -2
  100. imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
  101. imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
  102. imap_processing/tests/idex/test_idex_l0.py +4 -4
  103. imap_processing/tests/idex/test_idex_l1a.py +8 -2
  104. imap_processing/tests/idex/test_idex_l1b.py +126 -0
  105. imap_processing/tests/lo/test_lo_l1a.py +7 -16
  106. imap_processing/tests/lo/test_lo_science.py +69 -5
  107. imap_processing/tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts +0 -0
  108. imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv +1999 -0
  109. imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
  110. imap_processing/tests/mag/test_mag_l1b.py +97 -7
  111. imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
  112. imap_processing/tests/spice/test_geometry.py +115 -9
  113. imap_processing/tests/spice/test_time.py +135 -6
  114. imap_processing/tests/swapi/test_swapi_decom.py +75 -69
  115. imap_processing/tests/swapi/test_swapi_l1.py +4 -4
  116. imap_processing/tests/swe/conftest.py +33 -0
  117. imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
  118. imap_processing/tests/swe/test_swe_l1b.py +29 -8
  119. imap_processing/tests/swe/test_swe_l2.py +64 -8
  120. imap_processing/tests/test_utils.py +2 -2
  121. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -3314
  122. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
  123. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
  124. imap_processing/tests/ultra/unit/test_de.py +113 -0
  125. imap_processing/tests/ultra/unit/test_spatial_utils.py +125 -0
  126. imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
  127. imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
  128. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +55 -35
  129. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +10 -68
  130. imap_processing/ultra/constants.py +12 -3
  131. imap_processing/ultra/l1b/de.py +168 -30
  132. imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
  133. imap_processing/ultra/l1b/ultra_l1b_extended.py +46 -80
  134. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +60 -144
  135. imap_processing/ultra/utils/spatial_utils.py +221 -0
  136. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/METADATA +15 -14
  137. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/RECORD +142 -139
  138. imap_processing/cdf/cdf_attribute_manager.py +0 -322
  139. imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
  140. imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  141. imap_processing/hi/l0/decom_hi.py +0 -24
  142. imap_processing/hi/packet_definitions/hi_packet_definition.xml +0 -482
  143. imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
  144. imap_processing/hit/l0/data_classes/science_packet.py +0 -259
  145. imap_processing/hit/l0/utils/hit_base.py +0 -57
  146. imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
  147. imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  148. imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
  149. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
  150. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
  151. imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
  152. imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
  153. imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
  154. imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
  155. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
  156. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
  157. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
  158. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
  159. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
  160. imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
  161. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
  162. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
  163. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
  164. imap_processing/tests/hi/test_decom.py +0 -55
  165. imap_processing/tests/hi/test_l1a_sci_de.py +0 -72
  166. imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
  167. imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
  168. /imap_processing/{hi/l0/__init__.py → tests/glows/test_glows_l2_data.py} +0 -0
  169. /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
  170. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/LICENSE +0 -0
  171. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/WHEEL +0 -0
  172. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/entry_points.txt +0 -0
@@ -2,10 +2,19 @@
2
2
  Perform IDEX L1a Processing.
3
3
 
4
4
  This module processes decommutated IDEX packets and creates L1a data products.
5
+
6
+ Examples
7
+ --------
8
+ .. code-block:: python
9
+
10
+ from imap_processing.idex.idex_l1a import PacketParser
11
+
12
+ l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
13
+ l1a_data = PacketParser(l0_file, data_version)
14
+ l1a_data.write_l1a_cdf()
5
15
  """
6
16
 
7
17
  import logging
8
- from collections import namedtuple
9
18
  from enum import IntEnum
10
19
  from pathlib import Path
11
20
  from typing import Union
@@ -22,59 +31,6 @@ from imap_processing.utils import convert_to_binary_string
22
31
 
23
32
  logger = logging.getLogger(__name__)
24
33
 
25
- # TODO: Generate quicklook plots
26
-
27
- # Create a large dictionary of values from the FPGA header that need to be
28
- # captured into the CDF file. They are lumped together because they share
29
- # similar attributes.
30
- # Notes about the variables are set here, acting as comments and will also be
31
- # placed into the CDF in the VAR_NOTES attribute.
32
- TRIGGER_DESCRIPTION = namedtuple(
33
- "TRIGGER_DESCRIPTION",
34
- ["name", "packet_name"],
35
- )
36
- TRIGGER_DESCRIPTION_DICT = {
37
- trigger.name: trigger
38
- for trigger in [
39
- TRIGGER_DESCRIPTION("event_number", "IDX__TXHDREVTNUM"),
40
- TRIGGER_DESCRIPTION("tof_high_trigger_level", "IDX__TXHDRHGTRIGLVL"),
41
- TRIGGER_DESCRIPTION("tof_high_trigger_num_max_1_2", "IDX__TXHDRHGTRIGNMAX12"),
42
- TRIGGER_DESCRIPTION("tof_high_trigger_num_min_1_2", "IDX__TXHDRHGTRIGNMIN12"),
43
- TRIGGER_DESCRIPTION("tof_high_trigger_num_min_1", "IDX__TXHDRHGTRIGNMIN1"),
44
- TRIGGER_DESCRIPTION("tof_high_trigger_num_max_1", "IDX__TXHDRHGTRIGNMAX1"),
45
- TRIGGER_DESCRIPTION("tof_high_trigger_num_min_2", "IDX__TXHDRHGTRIGNMIN2"),
46
- TRIGGER_DESCRIPTION("tof_high_trigger_num_max_2", "IDX__TXHDRHGTRIGNMAX2"),
47
- TRIGGER_DESCRIPTION("tof_low_trigger_level", "IDX__TXHDRLGTRIGLVL"),
48
- TRIGGER_DESCRIPTION("tof_low_trigger_num_max_1_2", "IDX__TXHDRLGTRIGNMAX12"),
49
- TRIGGER_DESCRIPTION("tof_low_trigger_num_min_1_2", "IDX__TXHDRLGTRIGNMIN12"),
50
- TRIGGER_DESCRIPTION("tof_low_trigger_num_min_1", "IDX__TXHDRLGTRIGNMIN1"),
51
- TRIGGER_DESCRIPTION("tof_low_trigger_num_max_1", "IDX__TXHDRLGTRIGNMAX1"),
52
- TRIGGER_DESCRIPTION("tof_low_trigger_num_min_2", "IDX__TXHDRLGTRIGNMIN2"),
53
- TRIGGER_DESCRIPTION("tof_low_trigger_num_max_2", "IDX__TXHDRLGTRIGNMAX2"),
54
- TRIGGER_DESCRIPTION("tof_mid_trigger_level", "IDX__TXHDRMGTRIGLVL"),
55
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_1_2", "IDX__TXHDRMGTRIGNMAX12"),
56
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_1_2", "IDX__TXHDRMGTRIGNMIN12"),
57
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_1", "IDX__TXHDRMGTRIGNMIN1"),
58
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_1", "IDX__TXHDRMGTRIGNMAX1"),
59
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_2", "IDX__TXHDRMGTRIGNMIN2"),
60
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_2", "IDX__TXHDRMGTRIGNMAX2"),
61
- TRIGGER_DESCRIPTION("low_sample_coincidence_mode_blocks", "IDX__TXHDRLSTRIGCMBLOCKS"), # noqa
62
- TRIGGER_DESCRIPTION("low_sample_trigger_polarity", "IDX__TXHDRLSTRIGPOL"),
63
- TRIGGER_DESCRIPTION("low_sample_trigger_level", "IDX__TXHDRLSTRIGLVL"),
64
- TRIGGER_DESCRIPTION("low_sample_trigger_num_min", "IDX__TXHDRLSTRIGNMIN"),
65
- TRIGGER_DESCRIPTION("low_sample_trigger_mode", "IDX__TXHDRLSTRIGMODE"),
66
- TRIGGER_DESCRIPTION("tof_low_trigger_mode", "IDX__TXHDRLSTRIGMODE"),
67
- TRIGGER_DESCRIPTION("tof_mid_trigger_mode", "IDX__TXHDRMGTRIGMODE"),
68
- TRIGGER_DESCRIPTION("tof_high_trigger_mode", "IDX__TXHDRHGTRIGMODE"),
69
- TRIGGER_DESCRIPTION("detector_voltage", "IDX__TXHDRHVPSHKCH0"),
70
- TRIGGER_DESCRIPTION("sensor_voltage", "IDX__TXHDRHVPSHKCH1"),
71
- TRIGGER_DESCRIPTION("target_voltage", "IDX__TXHDRHVPSHKCH2"),
72
- TRIGGER_DESCRIPTION("reflectron_voltage", "IDX__TXHDRHVPSHKCH3"),
73
- TRIGGER_DESCRIPTION("rejection_voltage", "IDX__TXHDRHVPSHKCH4"),
74
- TRIGGER_DESCRIPTION("detector_current", "IDX__TXHDRHVPSHKCH5"),
75
- ]
76
- } # fmt: skip
77
-
78
34
 
79
35
  class Scitype(IntEnum):
80
36
  """Define parameters for IDEX Science Type."""
@@ -102,15 +58,6 @@ class PacketParser:
102
58
  The path and filename to the L0 file to read.
103
59
  data_version : str
104
60
  The version of the data product being created.
105
-
106
- Examples
107
- --------
108
- .. code-block:: python
109
-
110
- from imap_processing.idex.idex_l1a import PacketParser
111
- l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20230725_v001.pkts"
112
- l1a_data = PacketParser(l0_file, data_version)
113
- l1a_data.write_l1a_cdf()
114
61
  """
115
62
 
116
63
  def __init__(self, packet_file: Union[str, Path], data_version: str) -> None:
@@ -158,6 +105,23 @@ class PacketParser:
158
105
  idex_attrs = get_idex_attrs(data_version)
159
106
  self.data.attrs = idex_attrs.get_global_attributes("imap_idex_l1a_sci")
160
107
 
108
+ # NOTE: LABL_PTR_1 should be CDF_CHAR.
109
+ self.data["time_low_sr_label"] = xr.DataArray(
110
+ self.data.time_low_sr_dim.values.astype(str),
111
+ name="time_low_sr_label",
112
+ dims=["time_low_sr_label"],
113
+ attrs=idex_attrs.get_variable_attributes("time_low_sr_label"),
114
+ )
115
+
116
+ self.data["time_high_sr_label"] = xr.DataArray(
117
+ self.data.time_high_sr_dim.values.astype(str),
118
+ name="time_high_sr_label",
119
+ dims=["time_high_sr_label"],
120
+ attrs=idex_attrs.get_variable_attributes("time_high_sr_label"),
121
+ )
122
+
123
+ logger.info("IDEX L1A science data processing completed.")
124
+
161
125
 
162
126
  class RawDustEvent:
163
127
  """
@@ -246,13 +210,15 @@ class RawDustEvent:
246
210
  self.high_sample_trigger_time = 0
247
211
  self._set_sample_trigger_times(header_packet)
248
212
 
249
- # Iterate through the trigger description dictionary and pull out the values
250
- self.trigger_values = {
251
- trigger.name: header_packet[trigger.packet_name].raw_value
252
- for trigger in TRIGGER_DESCRIPTION_DICT.values()
213
+ # Iterate through every telemetry item not in the header and pull out the values
214
+ self.telemetry_items = {
215
+ key.lower(): val
216
+ for key, val in header_packet.items()
217
+ if key not in header_packet.header.keys()
253
218
  }
219
+
254
220
  logger.debug(
255
- f"trigger_values:\n{self.trigger_values}"
221
+ f"telemetry_items:\n{self.telemetry_items}"
256
222
  ) # Log values here in case of error
257
223
 
258
224
  # Initialize the binary data received from future packets
@@ -353,12 +319,28 @@ class RawDustEvent:
353
319
  The header has information about the number of blocks before triggering,
354
320
  rather than the number of samples before triggering.
355
321
  """
356
- # Retrieve the number of samples of high gain delay
357
- high_gain_delay = packet["IDX__TXHDRADC0IDELAY"]
322
+ # Retrieve the number of samples for high gain delay
323
+
324
+ # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits
325
+ # representing the high gain sample delay and the first 2 bits used for padding.
326
+ # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits
327
+ # 20 positions to the right, and the mask (0b1111111111) keeps only the least
328
+ # significant 10 bits.
329
+ high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111
330
+ n_blocks = packet["IDX__TXHDRBLOCKS"]
358
331
 
359
332
  # Retrieve number of low/high sample pre-trigger blocks
360
- num_low_sample_pretrigger_blocks = packet["IDX__TXHDRLSPREBLOCKS"]
361
- num_high_sample_pretrigger_blocks = packet["IDX__TXHDRHSPREBLOCKS"]
333
+
334
+ # packet['IDX__TXHDRBLOCKS'] is a 32-bit value:
335
+ # Bits 21-26 represent the number of low sampling pre-trigger blocks.
336
+ # We can extract this by shifting right by 6 bits and applying a mask to keep
337
+ # the last 6 bits.
338
+ # Bits 13-16 represent the number of high sampling pre-trigger blocks.
339
+ # We can extract this by shifting right by 16 bits and applying a mask to keep
340
+ # the last 4 bits.
341
+
342
+ num_low_sample_pretrigger_blocks = (n_blocks >> 6) & 0b111111
343
+ num_high_sample_pretrigger_blocks = (n_blocks >> 16) & 0b1111
362
344
 
363
345
  # Calculate the low and high sample trigger times based on the high gain delay
364
346
  # and the number of high sample/low sample pretrigger blocks
@@ -411,7 +393,7 @@ class RawDustEvent:
411
393
  Will process the low sample waveform.
412
394
 
413
395
  Parse a binary string representing a low sample waveform
414
- Data arrives in 32 bit chunks, divided up into:
396
+ Data arrives in 32-bit chunks, divided up into:
415
397
  * 8 bits of padding
416
398
  * 2x12 bits of integer data.
417
399
 
@@ -514,55 +496,54 @@ class RawDustEvent:
514
496
  dataset : xarray.Dataset
515
497
  A Dataset object containing the data from a single impact.
516
498
  """
517
- # Create object for CDF attrs
499
+ # Create an object for CDF attrs
518
500
  idex_attrs = self.cdf_attrs
519
501
 
520
- # Gather the huge number of trigger info metadata
502
+ # Gather the huge amount of metadata info
521
503
  trigger_vars = {}
522
- for var, value in self.trigger_values.items():
523
- trigger_desc = TRIGGER_DESCRIPTION_DICT[var]
504
+ for var, value in self.telemetry_items.items():
524
505
  trigger_vars[var] = xr.DataArray(
525
506
  name=var,
526
507
  data=[value],
527
508
  dims=("epoch"),
528
- attrs=idex_attrs.get_variable_attributes(trigger_desc.name),
509
+ attrs=idex_attrs.get_variable_attributes(var),
529
510
  )
530
511
 
531
512
  # Process the 6 primary data variables
532
513
  tof_high_xr = xr.DataArray(
533
514
  name="TOF_High",
534
515
  data=[self._parse_high_sample_waveform(self.TOF_High_bits)],
535
- dims=("epoch", "time_high_ssr_dim"),
516
+ dims=("epoch", "time_high_sr_dim"),
536
517
  attrs=idex_attrs.get_variable_attributes("tof_high_attrs"),
537
518
  )
538
519
  tof_low_xr = xr.DataArray(
539
520
  name="TOF_Low",
540
521
  data=[self._parse_high_sample_waveform(self.TOF_Low_bits)],
541
- dims=("epoch", "time_high_sr"),
522
+ dims=("epoch", "time_high_sr_dim"),
542
523
  attrs=idex_attrs.get_variable_attributes("tof_low_attrs"),
543
524
  )
544
525
  tof_mid_xr = xr.DataArray(
545
526
  name="TOF_Mid",
546
527
  data=[self._parse_high_sample_waveform(self.TOF_Mid_bits)],
547
- dims=("epoch", "time_high_sr"),
528
+ dims=("epoch", "time_high_sr_dim"),
548
529
  attrs=idex_attrs.get_variable_attributes("tof_mid_attrs"),
549
530
  )
550
531
  target_high_xr = xr.DataArray(
551
532
  name="Target_High",
552
533
  data=[self._parse_low_sample_waveform(self.Target_High_bits)],
553
- dims=("epoch", "time_low_sr"),
534
+ dims=("epoch", "time_low_sr_dim"),
554
535
  attrs=idex_attrs.get_variable_attributes("target_high_attrs"),
555
536
  )
556
537
  target_low_xr = xr.DataArray(
557
538
  name="Target_Low",
558
539
  data=[self._parse_low_sample_waveform(self.Target_Low_bits)],
559
- dims=("epoch", "time_low_sr"),
540
+ dims=("epoch", "time_low_sr_dim"),
560
541
  attrs=idex_attrs.get_variable_attributes("target_low_attrs"),
561
542
  )
562
543
  ion_grid_xr = xr.DataArray(
563
544
  name="Ion_Grid",
564
545
  data=[self._parse_low_sample_waveform(self.Ion_Grid_bits)],
565
- dims=("epoch", "time_low_sr"),
546
+ dims=("epoch", "time_low_sr_dim"),
566
547
  attrs=idex_attrs.get_variable_attributes("ion_grid_attrs"),
567
548
  )
568
549
 
@@ -0,0 +1,192 @@
1
+ """
2
+ Perform IDEX L1b Processing.
3
+
4
+ Examples
5
+ --------
6
+ .. code-block:: python
7
+
8
+ from imap_processing.idex.idex_l1a import PacketParser
9
+ from imap_processing.idex.idex_l1b import idex_l1b
10
+
11
+ l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
12
+ l1a_data = PacketParser(l0_file, data_version)
13
+ l1b_data = idex_l1b(l1a_data, data_version)
14
+ write_cdf(l1b_data)
15
+ """
16
+
17
+ import logging
18
+ from enum import Enum
19
+
20
+ import pandas as pd
21
+ import xarray as xr
22
+
23
+ from imap_processing import imap_module_directory
24
+ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
25
+ from imap_processing.utils import convert_raw_to_eu
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+
30
+ class ConversionFactors(float, Enum):
31
+ """Enum class for conversion factor values."""
32
+
33
+ TOF_High = 2.89e-4
34
+ TOF_Low = 5.14e-4
35
+ TOF_Mid = 1.13e-2
36
+ Target_Low = 1.58e1
37
+ Target_High = 1.63e-1
38
+ Ion_Grid = 7.46e-4
39
+
40
+
41
+ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
42
+ """
43
+ Will process IDEX l1a data to create l1b data products.
44
+
45
+ Parameters
46
+ ----------
47
+ l1a_dataset : xarray.Dataset
48
+ IDEX L1a dataset to process.
49
+ data_version : str
50
+ Version of the data product being created.
51
+
52
+ Returns
53
+ -------
54
+ l1b_dataset : xarray.Dataset
55
+ The``xarray`` dataset containing the science data and supporting metadata.
56
+ """
57
+ logger.info(
58
+ f"Running IDEX L1B processing on dataset: {l1a_dataset.attrs['Logical_source']}"
59
+ )
60
+
61
+ # create the attribute manager for this data level
62
+ idex_attrs = ImapCdfAttributes()
63
+ idex_attrs.add_instrument_global_attrs(instrument="idex")
64
+ idex_attrs.add_instrument_variable_attrs(instrument="idex", level="l1b")
65
+ idex_attrs.add_global_attribute("Data_version", data_version)
66
+
67
+ var_information_path = (
68
+ f"{imap_module_directory}/idex/idex_variable_unpacking_and_eu_conversion.csv"
69
+ )
70
+ # Read in csv that contains instrument variable setting information
71
+ var_information_df = pd.read_csv(var_information_path)
72
+
73
+ processed_vars = unpack_instrument_settings(
74
+ l1a_dataset, var_information_df, idex_attrs
75
+ )
76
+
77
+ waveforms_converted = convert_waveforms(l1a_dataset, idex_attrs)
78
+
79
+ epoch_da = xr.DataArray(
80
+ l1a_dataset["epoch"],
81
+ name="epoch",
82
+ dims=["epoch"],
83
+ attrs=idex_attrs.get_variable_attributes("epoch"),
84
+ )
85
+ # Create l1b Dataset
86
+ l1b_dataset = xr.Dataset(
87
+ coords={"epoch": epoch_da},
88
+ data_vars=processed_vars | waveforms_converted,
89
+ attrs=idex_attrs.get_global_attributes("imap_idex_l1b_sci"),
90
+ )
91
+ # Convert variables
92
+ l1b_dataset = convert_raw_to_eu(
93
+ l1b_dataset,
94
+ conversion_table_path=var_information_path,
95
+ packet_name="IDEX_SCI",
96
+ )
97
+ vars_to_copy = [
98
+ "shcoarse",
99
+ "shfine",
100
+ "time_high_sr",
101
+ "time_low_sr",
102
+ "time_high_sr_label",
103
+ "time_low_sr_label",
104
+ ]
105
+ # Copy arrays from the l1a_dataset that do not need l1b processing
106
+ for var in vars_to_copy:
107
+ l1b_dataset[var] = l1a_dataset[var].copy()
108
+
109
+ # TODO: Add TriggerMode and TriggerLevel attr
110
+ # TODO: Spice data?
111
+
112
+ logger.info("IDEX L1B science data processing completed.")
113
+
114
+ return l1b_dataset
115
+
116
+
117
+ def unpack_instrument_settings(
118
+ l1a_dataset: xr.Dataset,
119
+ var_information_df: pd.DataFrame,
120
+ idex_attrs: ImapCdfAttributes,
121
+ ) -> dict[str, xr.DataArray]:
122
+ """
123
+ Unpack raw telemetry data from the l1a dataset into individual variables.
124
+
125
+ Parameters
126
+ ----------
127
+ l1a_dataset : xarray.Dataset
128
+ IDEX L1a dataset containing the 6 waveform arrays.
129
+ var_information_df : pd.DataFrame
130
+ Pandas data frame that contains information about each variable
131
+ (e.g., bit-size, starting bit, and padding). This is used to unpack raw
132
+ telemetry data from the input dataset (`l1a_dataset`).
133
+ idex_attrs : ImapCdfAttributes
134
+ CDF attribute manager object.
135
+
136
+ Returns
137
+ -------
138
+ telemetry_data : dict
139
+ A dictionary where the keys are the instrument setting array names and the
140
+ values are the unpacked xr.DataArrays.
141
+ """
142
+ telemetry_data = {}
143
+
144
+ for _, row in var_information_df.iterrows():
145
+ unpacked_name = row["mnemonic"]
146
+
147
+ # Create binary mask of the size of the variable in bits
148
+ mask = (1 << row["unsigned_nbits"]) - 1
149
+ # Determine the number of bits to shift
150
+ shift = row["starting_bit"] - row["nbits_padding_before"]
151
+ # Get the unpacked value by shifting the data to align the desired bits with
152
+ # the least significant bits and applying the mask to isolate the target bits
153
+ unpacked_val = (l1a_dataset[row["var_name"]].data >> shift) & mask
154
+
155
+ telemetry_data[unpacked_name] = xr.DataArray(
156
+ name=unpacked_name,
157
+ data=unpacked_val,
158
+ dims=("epoch"),
159
+ attrs=idex_attrs.get_variable_attributes(unpacked_name),
160
+ )
161
+
162
+ return telemetry_data
163
+
164
+
165
+ def convert_waveforms(
166
+ l1a_dataset: xr.Dataset, idex_attrs: ImapCdfAttributes
167
+ ) -> dict[str, xr.DataArray]:
168
+ """
169
+ Apply transformation from raw DN to picocoulombs (pC) for each of the six waveforms.
170
+
171
+ Parameters
172
+ ----------
173
+ l1a_dataset : xarray.Dataset
174
+ IDEX L1a dataset containing the six waveform arrays.
175
+ idex_attrs : ImapCdfAttributes
176
+ CDF attribute manager object.
177
+
178
+ Returns
179
+ -------
180
+ waveforms_converted : dict
181
+ A dictionary where the keys are the waveform array names and the values are
182
+ xr.DataArrays representing the waveforms transformed into picocoulombs.
183
+ """
184
+ waveforms_pc = {}
185
+
186
+ for var in ConversionFactors:
187
+ waveforms_pc[var.name] = l1a_dataset[var.name] * var.value
188
+ waveforms_pc[var.name].attrs = idex_attrs.get_variable_attributes(
189
+ var.name.lower()
190
+ )
191
+
192
+ return waveforms_pc
@@ -0,0 +1,33 @@
1
+ index,mnemonic,var_name,starting_bit,nbits_padding_before,unsigned_nbits,unit,c0,c1,c2,c3,c4,c5,c6,c7,convertAs,packetName
2
+ 1,current_1v_pol,idx__txhdrprochkch01,4,4,12,mA,0,0.9029253,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
3
+ 2,current_1p9v_pol,idx__txhdrprochkch01,20,4,12,mA,0,0.9029253,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
4
+ 3,temperature_1,idx__txhdrprochkch23,4,4,12,C,162.8276,-0.2668687,0.000332379,-2.43E-07,9.03E-11,-1.33E-14,0.00E+00,0.00E+00,UNSEGMENTED_POLY,IDEX_SCI
5
+ 4,temperature_2,idx__txhdrprochkch23,20,4,12,C,162.8276,-0.2668687,0.000332379,-2.43E-07,9.03E-11,-1.33E-14,0.00E+00,0.00E+00,UNSEGMENTED_POLY,IDEX_SCI
6
+ 5,voltage_1v_bus,idx__txhdrprochkch45,4,4,12,V,0,0.000805861,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
7
+ 6,fpga_temperature,idx__txhdrprochkch45,20,4,12,C,358.31,-0.1187104,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
8
+ 7,voltage_1p9v_bus,idx__txhdrprochkch67,4,4,12,V,0,0.000805861,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
9
+ 8,voltage_3p3v_bus,idx__txhdrprochkch67,20,4,12,V,0,0.001611722,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
10
+ 9,detector_voltage,idx__txhdrhvpshkch01,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
11
+ 10,sensor_voltage,idx__txhdrhvpshkch01,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
12
+ 11,target_voltage,idx__txhdrhvpshkch23,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
13
+ 12,reflectron_voltage,idx__txhdrhvpshkch23,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
14
+ 13,rejection_voltage,idx__txhdrhvpshkch45,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
15
+ 14,current_hvps_sensor,idx__txhdrhvpshkch45,20,4,12,mA,0,7.33E-06,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
16
+ 15,positive_current_hvps,idx__txhdrhvpshkch67,4,4,12,mA,0,2.43E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
17
+ 16,negative_current_hvps,idx__txhdrhvpshkch67,20,4,12,mA,0,2.43E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
18
+ 17,voltage_3p3_ref,idx__txhdrlvhk0ch01,4,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
19
+ 18,voltage_3p3_op_ref,idx__txhdrlvhk0ch01,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
20
+ 19,voltage_neg6v_bus,idx__txhdrlvhk0ch23,4,4,12,V,-33,0.00886447,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
21
+ 20,voltage_pos6v_bus,idx__txhdrlvhk0ch23,20,4,12,V,0,0.00241758,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
22
+ 21,voltage_pos16v_bus,idx__txhdrlvhk0ch45,4,4,12,V,0,0.00482711,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
23
+ 22,voltage_pos3p3v_bus,idx__txhdrlvhk0ch45,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
24
+ 23,voltage_neg5v_bus,idx__txhdrlvhk0ch67,4,4,12,V,-33,0.00886447,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
25
+ 24,voltage_pos5v_bus,idx__txhdrlvhk0ch67,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
26
+ 25,current_3p3_bus,idx__txhdrlvhk1ch01,4,4,12,A,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
27
+ 26,current_16v_bus,idx__txhdrlvhk1ch01,20,4,12,A,0,0.000161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
28
+ 27,current_6v_bus,idx__txhdrlvhk1ch23,4,4,12,A,0,8.06E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
29
+ 28,current_neg6v_bus,idx__txhdrlvhk1ch23,20,4,12,A,0,-8.06E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
30
+ 29,current_5v_bus,idx__txhdrlvhk1ch45,4,4,12,A,0,4.03E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
31
+ 30,current_neg5v_bus,idx__txhdrlvhk1ch45,20,4,12,A,0,-2.01E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
32
+ 31,current_2p5v_bus,idx__txhdrlvhk1ch67,4,4,12,A,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
33
+ 32,current_neg2p5v_bus,idx__txhdrlvhk1ch67,20,4,12,A,0,-4.03E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI