imap-processing 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (99) hide show
  1. imap_processing/_version.py +2 -2
  2. imap_processing/ccsds/excel_to_xtce.py +2 -0
  3. imap_processing/cdf/config/imap_hi_variable_attrs.yaml +100 -1
  4. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +14 -0
  5. imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +63 -1
  6. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +7 -0
  7. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +574 -231
  8. imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +326 -0
  9. imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +33 -23
  10. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +7 -4
  11. imap_processing/cdf/utils.py +3 -5
  12. imap_processing/cli.py +13 -4
  13. imap_processing/codice/codice_l1a.py +5 -5
  14. imap_processing/codice/constants.py +9 -9
  15. imap_processing/codice/decompress.py +6 -2
  16. imap_processing/glows/l1a/glows_l1a.py +1 -2
  17. imap_processing/hi/l1a/hi_l1a.py +4 -4
  18. imap_processing/hi/l1a/histogram.py +106 -108
  19. imap_processing/hi/l1a/science_direct_event.py +91 -224
  20. imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +3994 -0
  21. imap_processing/hit/l0/constants.py +2 -2
  22. imap_processing/hit/l0/decom_hit.py +12 -101
  23. imap_processing/hit/l1a/hit_l1a.py +164 -23
  24. imap_processing/ialirt/l0/process_codicelo.py +153 -0
  25. imap_processing/ialirt/l0/process_hit.py +5 -5
  26. imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +281 -0
  27. imap_processing/ialirt/process_ephemeris.py +212 -0
  28. imap_processing/idex/idex_l1a.py +55 -75
  29. imap_processing/idex/idex_l1b.py +192 -0
  30. imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -0
  31. imap_processing/idex/packet_definitions/idex_packet_definition.xml +97 -595
  32. imap_processing/lo/l0/decompression_tables/decompression_tables.py +16 -0
  33. imap_processing/lo/l0/lo_science.py +44 -12
  34. imap_processing/lo/l1a/lo_l1a.py +76 -8
  35. imap_processing/lo/packet_definitions/lo_xtce.xml +9877 -87
  36. imap_processing/mag/l1a/mag_l1a.py +1 -2
  37. imap_processing/mag/l1a/mag_l1a_data.py +1 -2
  38. imap_processing/mag/l1b/mag_l1b.py +2 -1
  39. imap_processing/spice/geometry.py +37 -19
  40. imap_processing/spice/time.py +144 -2
  41. imap_processing/swapi/l1/swapi_l1.py +3 -3
  42. imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +1535 -446
  43. imap_processing/swe/l2/swe_l2.py +134 -17
  44. imap_processing/tests/ccsds/test_data/expected_output.xml +1 -1
  45. imap_processing/tests/codice/test_codice_l1a.py +8 -8
  46. imap_processing/tests/codice/test_decompress.py +4 -4
  47. imap_processing/tests/conftest.py +46 -43
  48. imap_processing/tests/hi/test_data/l0/H90_NHK_20241104.bin +0 -0
  49. imap_processing/tests/hi/test_data/l0/H90_sci_cnt_20241104.bin +0 -0
  50. imap_processing/tests/hi/test_data/l0/H90_sci_de_20241104.bin +0 -0
  51. imap_processing/tests/hi/test_hi_l1b.py +2 -2
  52. imap_processing/tests/hi/test_l1a.py +31 -58
  53. imap_processing/tests/hi/test_science_direct_event.py +58 -0
  54. imap_processing/tests/hit/test_data/sci_sample1.ccsds +0 -0
  55. imap_processing/tests/hit/test_decom_hit.py +60 -50
  56. imap_processing/tests/hit/test_hit_l1a.py +327 -12
  57. imap_processing/tests/hit/test_hit_l1b.py +76 -0
  58. imap_processing/tests/hit/validation_data/hskp_sample_eu.csv +89 -0
  59. imap_processing/tests/hit/validation_data/sci_sample_raw1.csv +29 -0
  60. imap_processing/tests/ialirt/test_data/l0/apid01152.tlm +0 -0
  61. imap_processing/tests/ialirt/test_data/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
  62. imap_processing/tests/ialirt/unit/test_process_codicelo.py +106 -0
  63. imap_processing/tests/ialirt/unit/test_process_ephemeris.py +109 -0
  64. imap_processing/tests/ialirt/unit/test_process_hit.py +9 -6
  65. imap_processing/tests/idex/conftest.py +1 -1
  66. imap_processing/tests/idex/test_idex_l0.py +1 -1
  67. imap_processing/tests/idex/test_idex_l1a.py +7 -1
  68. imap_processing/tests/idex/test_idex_l1b.py +126 -0
  69. imap_processing/tests/lo/test_lo_l1a.py +7 -16
  70. imap_processing/tests/lo/test_lo_science.py +67 -3
  71. imap_processing/tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts +0 -0
  72. imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv +1999 -0
  73. imap_processing/tests/mag/test_mag_l1b.py +39 -5
  74. imap_processing/tests/spice/test_geometry.py +32 -6
  75. imap_processing/tests/spice/test_time.py +135 -6
  76. imap_processing/tests/swapi/test_swapi_decom.py +75 -69
  77. imap_processing/tests/swapi/test_swapi_l1.py +4 -4
  78. imap_processing/tests/swe/test_swe_l2.py +64 -8
  79. imap_processing/tests/test_utils.py +1 -1
  80. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -3314
  81. imap_processing/tests/ultra/unit/test_de.py +8 -3
  82. imap_processing/tests/ultra/unit/test_spatial_utils.py +125 -0
  83. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +39 -29
  84. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +2 -25
  85. imap_processing/ultra/constants.py +4 -0
  86. imap_processing/ultra/l1b/de.py +8 -14
  87. imap_processing/ultra/l1b/ultra_l1b_extended.py +29 -70
  88. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +1 -36
  89. imap_processing/ultra/utils/spatial_utils.py +221 -0
  90. {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/METADATA +1 -1
  91. {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/RECORD +94 -76
  92. imap_processing/hi/l0/__init__.py +0 -0
  93. imap_processing/hi/l0/decom_hi.py +0 -24
  94. imap_processing/hi/packet_definitions/hi_packet_definition.xml +0 -482
  95. imap_processing/tests/hi/test_decom.py +0 -55
  96. imap_processing/tests/hi/test_l1a_sci_de.py +0 -72
  97. {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/LICENSE +0 -0
  98. {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/WHEEL +0 -0
  99. {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/entry_points.txt +0 -0
@@ -15,7 +15,6 @@ Examples
15
15
  """
16
16
 
17
17
  import logging
18
- from collections import namedtuple
19
18
  from enum import IntEnum
20
19
  from pathlib import Path
21
20
  from typing import Union
@@ -32,59 +31,6 @@ from imap_processing.utils import convert_to_binary_string
32
31
 
33
32
  logger = logging.getLogger(__name__)
34
33
 
35
- # TODO: Generate quicklook plots
36
-
37
- # Create a large dictionary of values from the FPGA header that need to be
38
- # captured into the CDF file. They are lumped together because they share
39
- # similar attributes.
40
- # Notes about the variables are set here, acting as comments and will also be
41
- # placed into the CDF in the VAR_NOTES attribute.
42
- TRIGGER_DESCRIPTION = namedtuple(
43
- "TRIGGER_DESCRIPTION",
44
- ["name", "packet_name"],
45
- )
46
- TRIGGER_DESCRIPTION_DICT = {
47
- trigger.name: trigger
48
- for trigger in [
49
- TRIGGER_DESCRIPTION("event_number", "IDX__TXHDREVTNUM"),
50
- TRIGGER_DESCRIPTION("tof_high_trigger_level", "IDX__TXHDRHGTRIGLVL"),
51
- TRIGGER_DESCRIPTION("tof_high_trigger_num_max_1_2", "IDX__TXHDRHGTRIGNMAX12"),
52
- TRIGGER_DESCRIPTION("tof_high_trigger_num_min_1_2", "IDX__TXHDRHGTRIGNMIN12"),
53
- TRIGGER_DESCRIPTION("tof_high_trigger_num_min_1", "IDX__TXHDRHGTRIGNMIN1"),
54
- TRIGGER_DESCRIPTION("tof_high_trigger_num_max_1", "IDX__TXHDRHGTRIGNMAX1"),
55
- TRIGGER_DESCRIPTION("tof_high_trigger_num_min_2", "IDX__TXHDRHGTRIGNMIN2"),
56
- TRIGGER_DESCRIPTION("tof_high_trigger_num_max_2", "IDX__TXHDRHGTRIGNMAX2"),
57
- TRIGGER_DESCRIPTION("tof_low_trigger_level", "IDX__TXHDRLGTRIGLVL"),
58
- TRIGGER_DESCRIPTION("tof_low_trigger_num_max_1_2", "IDX__TXHDRLGTRIGNMAX12"),
59
- TRIGGER_DESCRIPTION("tof_low_trigger_num_min_1_2", "IDX__TXHDRLGTRIGNMIN12"),
60
- TRIGGER_DESCRIPTION("tof_low_trigger_num_min_1", "IDX__TXHDRLGTRIGNMIN1"),
61
- TRIGGER_DESCRIPTION("tof_low_trigger_num_max_1", "IDX__TXHDRLGTRIGNMAX1"),
62
- TRIGGER_DESCRIPTION("tof_low_trigger_num_min_2", "IDX__TXHDRLGTRIGNMIN2"),
63
- TRIGGER_DESCRIPTION("tof_low_trigger_num_max_2", "IDX__TXHDRLGTRIGNMAX2"),
64
- TRIGGER_DESCRIPTION("tof_mid_trigger_level", "IDX__TXHDRMGTRIGLVL"),
65
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_1_2", "IDX__TXHDRMGTRIGNMAX12"),
66
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_1_2", "IDX__TXHDRMGTRIGNMIN12"),
67
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_1", "IDX__TXHDRMGTRIGNMIN1"),
68
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_1", "IDX__TXHDRMGTRIGNMAX1"),
69
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_2", "IDX__TXHDRMGTRIGNMIN2"),
70
- TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_2", "IDX__TXHDRMGTRIGNMAX2"),
71
- TRIGGER_DESCRIPTION("low_sample_coincidence_mode_blocks", "IDX__TXHDRLSTRIGCMBLOCKS"), # noqa
72
- TRIGGER_DESCRIPTION("low_sample_trigger_polarity", "IDX__TXHDRLSTRIGPOL"),
73
- TRIGGER_DESCRIPTION("low_sample_trigger_level", "IDX__TXHDRLSTRIGLVL"),
74
- TRIGGER_DESCRIPTION("low_sample_trigger_num_min", "IDX__TXHDRLSTRIGNMIN"),
75
- TRIGGER_DESCRIPTION("low_sample_trigger_mode", "IDX__TXHDRLSTRIGMODE"),
76
- TRIGGER_DESCRIPTION("tof_low_trigger_mode", "IDX__TXHDRLSTRIGMODE"),
77
- TRIGGER_DESCRIPTION("tof_mid_trigger_mode", "IDX__TXHDRMGTRIGMODE"),
78
- TRIGGER_DESCRIPTION("tof_high_trigger_mode", "IDX__TXHDRHGTRIGMODE"),
79
- TRIGGER_DESCRIPTION("detector_voltage", "IDX__TXHDRHVPSHKCH0"),
80
- TRIGGER_DESCRIPTION("sensor_voltage", "IDX__TXHDRHVPSHKCH1"),
81
- TRIGGER_DESCRIPTION("target_voltage", "IDX__TXHDRHVPSHKCH2"),
82
- TRIGGER_DESCRIPTION("reflectron_voltage", "IDX__TXHDRHVPSHKCH3"),
83
- TRIGGER_DESCRIPTION("rejection_voltage", "IDX__TXHDRHVPSHKCH4"),
84
- TRIGGER_DESCRIPTION("detector_current", "IDX__TXHDRHVPSHKCH5"),
85
- ]
86
- } # fmt: skip
87
-
88
34
 
89
35
  class Scitype(IntEnum):
90
36
  """Define parameters for IDEX Science Type."""
@@ -159,6 +105,23 @@ class PacketParser:
159
105
  idex_attrs = get_idex_attrs(data_version)
160
106
  self.data.attrs = idex_attrs.get_global_attributes("imap_idex_l1a_sci")
161
107
 
108
+ # NOTE: LABL_PTR_1 should be CDF_CHAR.
109
+ self.data["time_low_sr_label"] = xr.DataArray(
110
+ self.data.time_low_sr_dim.values.astype(str),
111
+ name="time_low_sr_label",
112
+ dims=["time_low_sr_label"],
113
+ attrs=idex_attrs.get_variable_attributes("time_low_sr_label"),
114
+ )
115
+
116
+ self.data["time_high_sr_label"] = xr.DataArray(
117
+ self.data.time_high_sr_dim.values.astype(str),
118
+ name="time_high_sr_label",
119
+ dims=["time_high_sr_label"],
120
+ attrs=idex_attrs.get_variable_attributes("time_high_sr_label"),
121
+ )
122
+
123
+ logger.info("IDEX L1A science data processing completed.")
124
+
162
125
 
163
126
  class RawDustEvent:
164
127
  """
@@ -247,13 +210,15 @@ class RawDustEvent:
247
210
  self.high_sample_trigger_time = 0
248
211
  self._set_sample_trigger_times(header_packet)
249
212
 
250
- # Iterate through the trigger description dictionary and pull out the values
251
- self.trigger_values = {
252
- trigger.name: header_packet[trigger.packet_name].raw_value
253
- for trigger in TRIGGER_DESCRIPTION_DICT.values()
213
+ # Iterate through every telemetry item not in the header and pull out the values
214
+ self.telemetry_items = {
215
+ key.lower(): val
216
+ for key, val in header_packet.items()
217
+ if key not in header_packet.header.keys()
254
218
  }
219
+
255
220
  logger.debug(
256
- f"trigger_values:\n{self.trigger_values}"
221
+ f"telemetry_items:\n{self.telemetry_items}"
257
222
  ) # Log values here in case of error
258
223
 
259
224
  # Initialize the binary data received from future packets
@@ -354,12 +319,28 @@ class RawDustEvent:
354
319
  The header has information about the number of blocks before triggering,
355
320
  rather than the number of samples before triggering.
356
321
  """
357
- # Retrieve the number of samples of high gain delay
358
- high_gain_delay = packet["IDX__TXHDRADC0IDELAY"]
322
+ # Retrieve the number of samples for high gain delay
323
+
324
+ # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits
325
+ # representing the high gain sample delay and the first 2 bits used for padding.
326
+ # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits
327
+ # 20 positions to the right, and the mask (0b1111111111) keeps only the least
328
+ # significant 10 bits.
329
+ high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111
330
+ n_blocks = packet["IDX__TXHDRBLOCKS"]
359
331
 
360
332
  # Retrieve number of low/high sample pre-trigger blocks
361
- num_low_sample_pretrigger_blocks = packet["IDX__TXHDRLSPREBLOCKS"]
362
- num_high_sample_pretrigger_blocks = packet["IDX__TXHDRHSPREBLOCKS"]
333
+
334
+ # packet['IDX__TXHDRBLOCKS'] is a 32-bit value:
335
+ # Bits 21-26 represent the number of low sampling pre-trigger blocks.
336
+ # We can extract this by shifting right by 6 bits and applying a mask to keep
337
+ # the last 6 bits.
338
+ # Bits 13-16 represent the number of high sampling pre-trigger blocks.
339
+ # We can extract this by shifting right by 16 bits and applying a mask to keep
340
+ # the last 4 bits.
341
+
342
+ num_low_sample_pretrigger_blocks = (n_blocks >> 6) & 0b111111
343
+ num_high_sample_pretrigger_blocks = (n_blocks >> 16) & 0b1111
363
344
 
364
345
  # Calculate the low and high sample trigger times based on the high gain delay
365
346
  # and the number of high sample/low sample pretrigger blocks
@@ -412,7 +393,7 @@ class RawDustEvent:
412
393
  Will process the low sample waveform.
413
394
 
414
395
  Parse a binary string representing a low sample waveform
415
- Data arrives in 32 bit chunks, divided up into:
396
+ Data arrives in 32-bit chunks, divided up into:
416
397
  * 8 bits of padding
417
398
  * 2x12 bits of integer data.
418
399
 
@@ -515,55 +496,54 @@ class RawDustEvent:
515
496
  dataset : xarray.Dataset
516
497
  A Dataset object containing the data from a single impact.
517
498
  """
518
- # Create object for CDF attrs
499
+ # Create an object for CDF attrs
519
500
  idex_attrs = self.cdf_attrs
520
501
 
521
- # Gather the huge number of trigger info metadata
502
+ # Gather the huge amount of metadata info
522
503
  trigger_vars = {}
523
- for var, value in self.trigger_values.items():
524
- trigger_desc = TRIGGER_DESCRIPTION_DICT[var]
504
+ for var, value in self.telemetry_items.items():
525
505
  trigger_vars[var] = xr.DataArray(
526
506
  name=var,
527
507
  data=[value],
528
508
  dims=("epoch"),
529
- attrs=idex_attrs.get_variable_attributes(trigger_desc.name),
509
+ attrs=idex_attrs.get_variable_attributes(var),
530
510
  )
531
511
 
532
512
  # Process the 6 primary data variables
533
513
  tof_high_xr = xr.DataArray(
534
514
  name="TOF_High",
535
515
  data=[self._parse_high_sample_waveform(self.TOF_High_bits)],
536
- dims=("epoch", "time_high_ssr_dim"),
516
+ dims=("epoch", "time_high_sr_dim"),
537
517
  attrs=idex_attrs.get_variable_attributes("tof_high_attrs"),
538
518
  )
539
519
  tof_low_xr = xr.DataArray(
540
520
  name="TOF_Low",
541
521
  data=[self._parse_high_sample_waveform(self.TOF_Low_bits)],
542
- dims=("epoch", "time_high_sr"),
522
+ dims=("epoch", "time_high_sr_dim"),
543
523
  attrs=idex_attrs.get_variable_attributes("tof_low_attrs"),
544
524
  )
545
525
  tof_mid_xr = xr.DataArray(
546
526
  name="TOF_Mid",
547
527
  data=[self._parse_high_sample_waveform(self.TOF_Mid_bits)],
548
- dims=("epoch", "time_high_sr"),
528
+ dims=("epoch", "time_high_sr_dim"),
549
529
  attrs=idex_attrs.get_variable_attributes("tof_mid_attrs"),
550
530
  )
551
531
  target_high_xr = xr.DataArray(
552
532
  name="Target_High",
553
533
  data=[self._parse_low_sample_waveform(self.Target_High_bits)],
554
- dims=("epoch", "time_low_sr"),
534
+ dims=("epoch", "time_low_sr_dim"),
555
535
  attrs=idex_attrs.get_variable_attributes("target_high_attrs"),
556
536
  )
557
537
  target_low_xr = xr.DataArray(
558
538
  name="Target_Low",
559
539
  data=[self._parse_low_sample_waveform(self.Target_Low_bits)],
560
- dims=("epoch", "time_low_sr"),
540
+ dims=("epoch", "time_low_sr_dim"),
561
541
  attrs=idex_attrs.get_variable_attributes("target_low_attrs"),
562
542
  )
563
543
  ion_grid_xr = xr.DataArray(
564
544
  name="Ion_Grid",
565
545
  data=[self._parse_low_sample_waveform(self.Ion_Grid_bits)],
566
- dims=("epoch", "time_low_sr"),
546
+ dims=("epoch", "time_low_sr_dim"),
567
547
  attrs=idex_attrs.get_variable_attributes("ion_grid_attrs"),
568
548
  )
569
549
 
@@ -0,0 +1,192 @@
1
+ """
2
+ Perform IDEX L1b Processing.
3
+
4
+ Examples
5
+ --------
6
+ .. code-block:: python
7
+
8
+ from imap_processing.idex.idex_l1a import PacketParser
9
+ from imap_processing.idex.idex_l1b import idex_l1b
10
+
11
+ l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
12
+ l1a_data = PacketParser(l0_file, data_version)
13
+ l1b_data = idex_l1b(l1a_data, data_version)
14
+ write_cdf(l1b_data)
15
+ """
16
+
17
+ import logging
18
+ from enum import Enum
19
+
20
+ import pandas as pd
21
+ import xarray as xr
22
+
23
+ from imap_processing import imap_module_directory
24
+ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
25
+ from imap_processing.utils import convert_raw_to_eu
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+
30
+ class ConversionFactors(float, Enum):
31
+ """Enum class for conversion factor values."""
32
+
33
+ TOF_High = 2.89e-4
34
+ TOF_Low = 5.14e-4
35
+ TOF_Mid = 1.13e-2
36
+ Target_Low = 1.58e1
37
+ Target_High = 1.63e-1
38
+ Ion_Grid = 7.46e-4
39
+
40
+
41
+ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
42
+ """
43
+ Will process IDEX l1a data to create l1b data products.
44
+
45
+ Parameters
46
+ ----------
47
+ l1a_dataset : xarray.Dataset
48
+ IDEX L1a dataset to process.
49
+ data_version : str
50
+ Version of the data product being created.
51
+
52
+ Returns
53
+ -------
54
+ l1b_dataset : xarray.Dataset
55
+ The``xarray`` dataset containing the science data and supporting metadata.
56
+ """
57
+ logger.info(
58
+ f"Running IDEX L1B processing on dataset: {l1a_dataset.attrs['Logical_source']}"
59
+ )
60
+
61
+ # create the attribute manager for this data level
62
+ idex_attrs = ImapCdfAttributes()
63
+ idex_attrs.add_instrument_global_attrs(instrument="idex")
64
+ idex_attrs.add_instrument_variable_attrs(instrument="idex", level="l1b")
65
+ idex_attrs.add_global_attribute("Data_version", data_version)
66
+
67
+ var_information_path = (
68
+ f"{imap_module_directory}/idex/idex_variable_unpacking_and_eu_conversion.csv"
69
+ )
70
+ # Read in csv that contains instrument variable setting information
71
+ var_information_df = pd.read_csv(var_information_path)
72
+
73
+ processed_vars = unpack_instrument_settings(
74
+ l1a_dataset, var_information_df, idex_attrs
75
+ )
76
+
77
+ waveforms_converted = convert_waveforms(l1a_dataset, idex_attrs)
78
+
79
+ epoch_da = xr.DataArray(
80
+ l1a_dataset["epoch"],
81
+ name="epoch",
82
+ dims=["epoch"],
83
+ attrs=idex_attrs.get_variable_attributes("epoch"),
84
+ )
85
+ # Create l1b Dataset
86
+ l1b_dataset = xr.Dataset(
87
+ coords={"epoch": epoch_da},
88
+ data_vars=processed_vars | waveforms_converted,
89
+ attrs=idex_attrs.get_global_attributes("imap_idex_l1b_sci"),
90
+ )
91
+ # Convert variables
92
+ l1b_dataset = convert_raw_to_eu(
93
+ l1b_dataset,
94
+ conversion_table_path=var_information_path,
95
+ packet_name="IDEX_SCI",
96
+ )
97
+ vars_to_copy = [
98
+ "shcoarse",
99
+ "shfine",
100
+ "time_high_sr",
101
+ "time_low_sr",
102
+ "time_high_sr_label",
103
+ "time_low_sr_label",
104
+ ]
105
+ # Copy arrays from the l1a_dataset that do not need l1b processing
106
+ for var in vars_to_copy:
107
+ l1b_dataset[var] = l1a_dataset[var].copy()
108
+
109
+ # TODO: Add TriggerMode and TriggerLevel attr
110
+ # TODO: Spice data?
111
+
112
+ logger.info("IDEX L1B science data processing completed.")
113
+
114
+ return l1b_dataset
115
+
116
+
117
+ def unpack_instrument_settings(
118
+ l1a_dataset: xr.Dataset,
119
+ var_information_df: pd.DataFrame,
120
+ idex_attrs: ImapCdfAttributes,
121
+ ) -> dict[str, xr.DataArray]:
122
+ """
123
+ Unpack raw telemetry data from the l1a dataset into individual variables.
124
+
125
+ Parameters
126
+ ----------
127
+ l1a_dataset : xarray.Dataset
128
+ IDEX L1a dataset containing the 6 waveform arrays.
129
+ var_information_df : pd.DataFrame
130
+ Pandas data frame that contains information about each variable
131
+ (e.g., bit-size, starting bit, and padding). This is used to unpack raw
132
+ telemetry data from the input dataset (`l1a_dataset`).
133
+ idex_attrs : ImapCdfAttributes
134
+ CDF attribute manager object.
135
+
136
+ Returns
137
+ -------
138
+ telemetry_data : dict
139
+ A dictionary where the keys are the instrument setting array names and the
140
+ values are the unpacked xr.DataArrays.
141
+ """
142
+ telemetry_data = {}
143
+
144
+ for _, row in var_information_df.iterrows():
145
+ unpacked_name = row["mnemonic"]
146
+
147
+ # Create binary mask of the size of the variable in bits
148
+ mask = (1 << row["unsigned_nbits"]) - 1
149
+ # Determine the number of bits to shift
150
+ shift = row["starting_bit"] - row["nbits_padding_before"]
151
+ # Get the unpacked value by shifting the data to align the desired bits with
152
+ # the least significant bits and applying the mask to isolate the target bits
153
+ unpacked_val = (l1a_dataset[row["var_name"]].data >> shift) & mask
154
+
155
+ telemetry_data[unpacked_name] = xr.DataArray(
156
+ name=unpacked_name,
157
+ data=unpacked_val,
158
+ dims=("epoch"),
159
+ attrs=idex_attrs.get_variable_attributes(unpacked_name),
160
+ )
161
+
162
+ return telemetry_data
163
+
164
+
165
+ def convert_waveforms(
166
+ l1a_dataset: xr.Dataset, idex_attrs: ImapCdfAttributes
167
+ ) -> dict[str, xr.DataArray]:
168
+ """
169
+ Apply transformation from raw DN to picocoulombs (pC) for each of the six waveforms.
170
+
171
+ Parameters
172
+ ----------
173
+ l1a_dataset : xarray.Dataset
174
+ IDEX L1a dataset containing the six waveform arrays.
175
+ idex_attrs : ImapCdfAttributes
176
+ CDF attribute manager object.
177
+
178
+ Returns
179
+ -------
180
+ waveforms_converted : dict
181
+ A dictionary where the keys are the waveform array names and the values are
182
+ xr.DataArrays representing the waveforms transformed into picocoulombs.
183
+ """
184
+ waveforms_pc = {}
185
+
186
+ for var in ConversionFactors:
187
+ waveforms_pc[var.name] = l1a_dataset[var.name] * var.value
188
+ waveforms_pc[var.name].attrs = idex_attrs.get_variable_attributes(
189
+ var.name.lower()
190
+ )
191
+
192
+ return waveforms_pc
@@ -0,0 +1,33 @@
1
+ index,mnemonic,var_name,starting_bit,nbits_padding_before,unsigned_nbits,unit,c0,c1,c2,c3,c4,c5,c6,c7,convertAs,packetName
2
+ 1,current_1v_pol,idx__txhdrprochkch01,4,4,12,mA,0,0.9029253,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
3
+ 2,current_1p9v_pol,idx__txhdrprochkch01,20,4,12,mA,0,0.9029253,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
4
+ 3,temperature_1,idx__txhdrprochkch23,4,4,12,C,162.8276,-0.2668687,0.000332379,-2.43E-07,9.03E-11,-1.33E-14,0.00E+00,0.00E+00,UNSEGMENTED_POLY,IDEX_SCI
5
+ 4,temperature_2,idx__txhdrprochkch23,20,4,12,C,162.8276,-0.2668687,0.000332379,-2.43E-07,9.03E-11,-1.33E-14,0.00E+00,0.00E+00,UNSEGMENTED_POLY,IDEX_SCI
6
+ 5,voltage_1v_bus,idx__txhdrprochkch45,4,4,12,V,0,0.000805861,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
7
+ 6,fpga_temperature,idx__txhdrprochkch45,20,4,12,C,358.31,-0.1187104,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
8
+ 7,voltage_1p9v_bus,idx__txhdrprochkch67,4,4,12,V,0,0.000805861,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
9
+ 8,voltage_3p3v_bus,idx__txhdrprochkch67,20,4,12,V,0,0.001611722,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
10
+ 9,detector_voltage,idx__txhdrhvpshkch01,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
11
+ 10,sensor_voltage,idx__txhdrhvpshkch01,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
12
+ 11,target_voltage,idx__txhdrhvpshkch23,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
13
+ 12,reflectron_voltage,idx__txhdrhvpshkch23,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
14
+ 13,rejection_voltage,idx__txhdrhvpshkch45,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
15
+ 14,current_hvps_sensor,idx__txhdrhvpshkch45,20,4,12,mA,0,7.33E-06,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
16
+ 15,positive_current_hvps,idx__txhdrhvpshkch67,4,4,12,mA,0,2.43E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
17
+ 16,negative_current_hvps,idx__txhdrhvpshkch67,20,4,12,mA,0,2.43E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
18
+ 17,voltage_3p3_ref,idx__txhdrlvhk0ch01,4,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
19
+ 18,voltage_3p3_op_ref,idx__txhdrlvhk0ch01,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
20
+ 19,voltage_neg6v_bus,idx__txhdrlvhk0ch23,4,4,12,V,-33,0.00886447,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
21
+ 20,voltage_pos6v_bus,idx__txhdrlvhk0ch23,20,4,12,V,0,0.00241758,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
22
+ 21,voltage_pos16v_bus,idx__txhdrlvhk0ch45,4,4,12,V,0,0.00482711,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
23
+ 22,voltage_pos3p3v_bus,idx__txhdrlvhk0ch45,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
24
+ 23,voltage_neg5v_bus,idx__txhdrlvhk0ch67,4,4,12,V,-33,0.00886447,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
25
+ 24,voltage_pos5v_bus,idx__txhdrlvhk0ch67,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
26
+ 25,current_3p3_bus,idx__txhdrlvhk1ch01,4,4,12,A,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
27
+ 26,current_16v_bus,idx__txhdrlvhk1ch01,20,4,12,A,0,0.000161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
28
+ 27,current_6v_bus,idx__txhdrlvhk1ch23,4,4,12,A,0,8.06E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
29
+ 28,current_neg6v_bus,idx__txhdrlvhk1ch23,20,4,12,A,0,-8.06E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
30
+ 29,current_5v_bus,idx__txhdrlvhk1ch45,4,4,12,A,0,4.03E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
31
+ 30,current_neg5v_bus,idx__txhdrlvhk1ch45,20,4,12,A,0,-2.01E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
32
+ 31,current_2p5v_bus,idx__txhdrlvhk1ch67,4,4,12,A,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
33
+ 32,current_neg2p5v_bus,idx__txhdrlvhk1ch67,20,4,12,A,0,-4.03E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI