imap-processing 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imap-processing might be problematic. Click here for more details.
- imap_processing/_version.py +2 -2
- imap_processing/ccsds/excel_to_xtce.py +2 -0
- imap_processing/cdf/config/imap_hi_variable_attrs.yaml +100 -1
- imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +14 -0
- imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +63 -1
- imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +7 -0
- imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +574 -231
- imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +326 -0
- imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +33 -23
- imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +7 -4
- imap_processing/cdf/utils.py +3 -5
- imap_processing/cli.py +13 -4
- imap_processing/codice/codice_l1a.py +5 -5
- imap_processing/codice/constants.py +9 -9
- imap_processing/codice/decompress.py +6 -2
- imap_processing/glows/l1a/glows_l1a.py +1 -2
- imap_processing/hi/l1a/hi_l1a.py +4 -4
- imap_processing/hi/l1a/histogram.py +106 -108
- imap_processing/hi/l1a/science_direct_event.py +91 -224
- imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +3994 -0
- imap_processing/hit/l0/constants.py +2 -2
- imap_processing/hit/l0/decom_hit.py +12 -101
- imap_processing/hit/l1a/hit_l1a.py +164 -23
- imap_processing/ialirt/l0/process_codicelo.py +153 -0
- imap_processing/ialirt/l0/process_hit.py +5 -5
- imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +281 -0
- imap_processing/ialirt/process_ephemeris.py +212 -0
- imap_processing/idex/idex_l1a.py +55 -75
- imap_processing/idex/idex_l1b.py +192 -0
- imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -0
- imap_processing/idex/packet_definitions/idex_packet_definition.xml +97 -595
- imap_processing/lo/l0/decompression_tables/decompression_tables.py +16 -0
- imap_processing/lo/l0/lo_science.py +44 -12
- imap_processing/lo/l1a/lo_l1a.py +76 -8
- imap_processing/lo/packet_definitions/lo_xtce.xml +9877 -87
- imap_processing/mag/l1a/mag_l1a.py +1 -2
- imap_processing/mag/l1a/mag_l1a_data.py +1 -2
- imap_processing/mag/l1b/mag_l1b.py +2 -1
- imap_processing/spice/geometry.py +37 -19
- imap_processing/spice/time.py +144 -2
- imap_processing/swapi/l1/swapi_l1.py +3 -3
- imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +1535 -446
- imap_processing/swe/l2/swe_l2.py +134 -17
- imap_processing/tests/ccsds/test_data/expected_output.xml +1 -1
- imap_processing/tests/codice/test_codice_l1a.py +8 -8
- imap_processing/tests/codice/test_decompress.py +4 -4
- imap_processing/tests/conftest.py +46 -43
- imap_processing/tests/hi/test_data/l0/H90_NHK_20241104.bin +0 -0
- imap_processing/tests/hi/test_data/l0/H90_sci_cnt_20241104.bin +0 -0
- imap_processing/tests/hi/test_data/l0/H90_sci_de_20241104.bin +0 -0
- imap_processing/tests/hi/test_hi_l1b.py +2 -2
- imap_processing/tests/hi/test_l1a.py +31 -58
- imap_processing/tests/hi/test_science_direct_event.py +58 -0
- imap_processing/tests/hit/test_data/sci_sample1.ccsds +0 -0
- imap_processing/tests/hit/test_decom_hit.py +60 -50
- imap_processing/tests/hit/test_hit_l1a.py +327 -12
- imap_processing/tests/hit/test_hit_l1b.py +76 -0
- imap_processing/tests/hit/validation_data/hskp_sample_eu.csv +89 -0
- imap_processing/tests/hit/validation_data/sci_sample_raw1.csv +29 -0
- imap_processing/tests/ialirt/test_data/l0/apid01152.tlm +0 -0
- imap_processing/tests/ialirt/test_data/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
- imap_processing/tests/ialirt/unit/test_process_codicelo.py +106 -0
- imap_processing/tests/ialirt/unit/test_process_ephemeris.py +109 -0
- imap_processing/tests/ialirt/unit/test_process_hit.py +9 -6
- imap_processing/tests/idex/conftest.py +1 -1
- imap_processing/tests/idex/test_idex_l0.py +1 -1
- imap_processing/tests/idex/test_idex_l1a.py +7 -1
- imap_processing/tests/idex/test_idex_l1b.py +126 -0
- imap_processing/tests/lo/test_lo_l1a.py +7 -16
- imap_processing/tests/lo/test_lo_science.py +67 -3
- imap_processing/tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts +0 -0
- imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv +1999 -0
- imap_processing/tests/mag/test_mag_l1b.py +39 -5
- imap_processing/tests/spice/test_geometry.py +32 -6
- imap_processing/tests/spice/test_time.py +135 -6
- imap_processing/tests/swapi/test_swapi_decom.py +75 -69
- imap_processing/tests/swapi/test_swapi_l1.py +4 -4
- imap_processing/tests/swe/test_swe_l2.py +64 -8
- imap_processing/tests/test_utils.py +1 -1
- imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -3314
- imap_processing/tests/ultra/unit/test_de.py +8 -3
- imap_processing/tests/ultra/unit/test_spatial_utils.py +125 -0
- imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +39 -29
- imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +2 -25
- imap_processing/ultra/constants.py +4 -0
- imap_processing/ultra/l1b/de.py +8 -14
- imap_processing/ultra/l1b/ultra_l1b_extended.py +29 -70
- imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +1 -36
- imap_processing/ultra/utils/spatial_utils.py +221 -0
- {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/METADATA +1 -1
- {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/RECORD +94 -76
- imap_processing/hi/l0/__init__.py +0 -0
- imap_processing/hi/l0/decom_hi.py +0 -24
- imap_processing/hi/packet_definitions/hi_packet_definition.xml +0 -482
- imap_processing/tests/hi/test_decom.py +0 -55
- imap_processing/tests/hi/test_l1a_sci_de.py +0 -72
- {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/LICENSE +0 -0
- {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/WHEEL +0 -0
- {imap_processing-0.8.0.dist-info → imap_processing-0.9.0.dist-info}/entry_points.txt +0 -0
imap_processing/idex/idex_l1a.py
CHANGED
|
@@ -15,7 +15,6 @@ Examples
|
|
|
15
15
|
"""
|
|
16
16
|
|
|
17
17
|
import logging
|
|
18
|
-
from collections import namedtuple
|
|
19
18
|
from enum import IntEnum
|
|
20
19
|
from pathlib import Path
|
|
21
20
|
from typing import Union
|
|
@@ -32,59 +31,6 @@ from imap_processing.utils import convert_to_binary_string
|
|
|
32
31
|
|
|
33
32
|
logger = logging.getLogger(__name__)
|
|
34
33
|
|
|
35
|
-
# TODO: Generate quicklook plots
|
|
36
|
-
|
|
37
|
-
# Create a large dictionary of values from the FPGA header that need to be
|
|
38
|
-
# captured into the CDF file. They are lumped together because they share
|
|
39
|
-
# similar attributes.
|
|
40
|
-
# Notes about the variables are set here, acting as comments and will also be
|
|
41
|
-
# placed into the CDF in the VAR_NOTES attribute.
|
|
42
|
-
TRIGGER_DESCRIPTION = namedtuple(
|
|
43
|
-
"TRIGGER_DESCRIPTION",
|
|
44
|
-
["name", "packet_name"],
|
|
45
|
-
)
|
|
46
|
-
TRIGGER_DESCRIPTION_DICT = {
|
|
47
|
-
trigger.name: trigger
|
|
48
|
-
for trigger in [
|
|
49
|
-
TRIGGER_DESCRIPTION("event_number", "IDX__TXHDREVTNUM"),
|
|
50
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_level", "IDX__TXHDRHGTRIGLVL"),
|
|
51
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_num_max_1_2", "IDX__TXHDRHGTRIGNMAX12"),
|
|
52
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_num_min_1_2", "IDX__TXHDRHGTRIGNMIN12"),
|
|
53
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_num_min_1", "IDX__TXHDRHGTRIGNMIN1"),
|
|
54
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_num_max_1", "IDX__TXHDRHGTRIGNMAX1"),
|
|
55
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_num_min_2", "IDX__TXHDRHGTRIGNMIN2"),
|
|
56
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_num_max_2", "IDX__TXHDRHGTRIGNMAX2"),
|
|
57
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_level", "IDX__TXHDRLGTRIGLVL"),
|
|
58
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_num_max_1_2", "IDX__TXHDRLGTRIGNMAX12"),
|
|
59
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_num_min_1_2", "IDX__TXHDRLGTRIGNMIN12"),
|
|
60
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_num_min_1", "IDX__TXHDRLGTRIGNMIN1"),
|
|
61
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_num_max_1", "IDX__TXHDRLGTRIGNMAX1"),
|
|
62
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_num_min_2", "IDX__TXHDRLGTRIGNMIN2"),
|
|
63
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_num_max_2", "IDX__TXHDRLGTRIGNMAX2"),
|
|
64
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_level", "IDX__TXHDRMGTRIGLVL"),
|
|
65
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_1_2", "IDX__TXHDRMGTRIGNMAX12"),
|
|
66
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_1_2", "IDX__TXHDRMGTRIGNMIN12"),
|
|
67
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_1", "IDX__TXHDRMGTRIGNMIN1"),
|
|
68
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_1", "IDX__TXHDRMGTRIGNMAX1"),
|
|
69
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_num_min_2", "IDX__TXHDRMGTRIGNMIN2"),
|
|
70
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_num_max_2", "IDX__TXHDRMGTRIGNMAX2"),
|
|
71
|
-
TRIGGER_DESCRIPTION("low_sample_coincidence_mode_blocks", "IDX__TXHDRLSTRIGCMBLOCKS"), # noqa
|
|
72
|
-
TRIGGER_DESCRIPTION("low_sample_trigger_polarity", "IDX__TXHDRLSTRIGPOL"),
|
|
73
|
-
TRIGGER_DESCRIPTION("low_sample_trigger_level", "IDX__TXHDRLSTRIGLVL"),
|
|
74
|
-
TRIGGER_DESCRIPTION("low_sample_trigger_num_min", "IDX__TXHDRLSTRIGNMIN"),
|
|
75
|
-
TRIGGER_DESCRIPTION("low_sample_trigger_mode", "IDX__TXHDRLSTRIGMODE"),
|
|
76
|
-
TRIGGER_DESCRIPTION("tof_low_trigger_mode", "IDX__TXHDRLSTRIGMODE"),
|
|
77
|
-
TRIGGER_DESCRIPTION("tof_mid_trigger_mode", "IDX__TXHDRMGTRIGMODE"),
|
|
78
|
-
TRIGGER_DESCRIPTION("tof_high_trigger_mode", "IDX__TXHDRHGTRIGMODE"),
|
|
79
|
-
TRIGGER_DESCRIPTION("detector_voltage", "IDX__TXHDRHVPSHKCH0"),
|
|
80
|
-
TRIGGER_DESCRIPTION("sensor_voltage", "IDX__TXHDRHVPSHKCH1"),
|
|
81
|
-
TRIGGER_DESCRIPTION("target_voltage", "IDX__TXHDRHVPSHKCH2"),
|
|
82
|
-
TRIGGER_DESCRIPTION("reflectron_voltage", "IDX__TXHDRHVPSHKCH3"),
|
|
83
|
-
TRIGGER_DESCRIPTION("rejection_voltage", "IDX__TXHDRHVPSHKCH4"),
|
|
84
|
-
TRIGGER_DESCRIPTION("detector_current", "IDX__TXHDRHVPSHKCH5"),
|
|
85
|
-
]
|
|
86
|
-
} # fmt: skip
|
|
87
|
-
|
|
88
34
|
|
|
89
35
|
class Scitype(IntEnum):
|
|
90
36
|
"""Define parameters for IDEX Science Type."""
|
|
@@ -159,6 +105,23 @@ class PacketParser:
|
|
|
159
105
|
idex_attrs = get_idex_attrs(data_version)
|
|
160
106
|
self.data.attrs = idex_attrs.get_global_attributes("imap_idex_l1a_sci")
|
|
161
107
|
|
|
108
|
+
# NOTE: LABL_PTR_1 should be CDF_CHAR.
|
|
109
|
+
self.data["time_low_sr_label"] = xr.DataArray(
|
|
110
|
+
self.data.time_low_sr_dim.values.astype(str),
|
|
111
|
+
name="time_low_sr_label",
|
|
112
|
+
dims=["time_low_sr_label"],
|
|
113
|
+
attrs=idex_attrs.get_variable_attributes("time_low_sr_label"),
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
self.data["time_high_sr_label"] = xr.DataArray(
|
|
117
|
+
self.data.time_high_sr_dim.values.astype(str),
|
|
118
|
+
name="time_high_sr_label",
|
|
119
|
+
dims=["time_high_sr_label"],
|
|
120
|
+
attrs=idex_attrs.get_variable_attributes("time_high_sr_label"),
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
logger.info("IDEX L1A science data processing completed.")
|
|
124
|
+
|
|
162
125
|
|
|
163
126
|
class RawDustEvent:
|
|
164
127
|
"""
|
|
@@ -247,13 +210,15 @@ class RawDustEvent:
|
|
|
247
210
|
self.high_sample_trigger_time = 0
|
|
248
211
|
self._set_sample_trigger_times(header_packet)
|
|
249
212
|
|
|
250
|
-
# Iterate through
|
|
251
|
-
self.
|
|
252
|
-
|
|
253
|
-
for
|
|
213
|
+
# Iterate through every telemetry item not in the header and pull out the values
|
|
214
|
+
self.telemetry_items = {
|
|
215
|
+
key.lower(): val
|
|
216
|
+
for key, val in header_packet.items()
|
|
217
|
+
if key not in header_packet.header.keys()
|
|
254
218
|
}
|
|
219
|
+
|
|
255
220
|
logger.debug(
|
|
256
|
-
f"
|
|
221
|
+
f"telemetry_items:\n{self.telemetry_items}"
|
|
257
222
|
) # Log values here in case of error
|
|
258
223
|
|
|
259
224
|
# Initialize the binary data received from future packets
|
|
@@ -354,12 +319,28 @@ class RawDustEvent:
|
|
|
354
319
|
The header has information about the number of blocks before triggering,
|
|
355
320
|
rather than the number of samples before triggering.
|
|
356
321
|
"""
|
|
357
|
-
# Retrieve the number of samples
|
|
358
|
-
|
|
322
|
+
# Retrieve the number of samples for high gain delay
|
|
323
|
+
|
|
324
|
+
# packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits
|
|
325
|
+
# representing the high gain sample delay and the first 2 bits used for padding.
|
|
326
|
+
# To extract the high gain bits, the bitwise right shift (>> 20) moves the bits
|
|
327
|
+
# 20 positions to the right, and the mask (0b1111111111) keeps only the least
|
|
328
|
+
# significant 10 bits.
|
|
329
|
+
high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111
|
|
330
|
+
n_blocks = packet["IDX__TXHDRBLOCKS"]
|
|
359
331
|
|
|
360
332
|
# Retrieve number of low/high sample pre-trigger blocks
|
|
361
|
-
|
|
362
|
-
|
|
333
|
+
|
|
334
|
+
# packet['IDX__TXHDRBLOCKS'] is a 32-bit value:
|
|
335
|
+
# Bits 21-26 represent the number of low sampling pre-trigger blocks.
|
|
336
|
+
# We can extract this by shifting right by 6 bits and applying a mask to keep
|
|
337
|
+
# the last 6 bits.
|
|
338
|
+
# Bits 13-16 represent the number of high sampling pre-trigger blocks.
|
|
339
|
+
# We can extract this by shifting right by 16 bits and applying a mask to keep
|
|
340
|
+
# the last 4 bits.
|
|
341
|
+
|
|
342
|
+
num_low_sample_pretrigger_blocks = (n_blocks >> 6) & 0b111111
|
|
343
|
+
num_high_sample_pretrigger_blocks = (n_blocks >> 16) & 0b1111
|
|
363
344
|
|
|
364
345
|
# Calculate the low and high sample trigger times based on the high gain delay
|
|
365
346
|
# and the number of high sample/low sample pretrigger blocks
|
|
@@ -412,7 +393,7 @@ class RawDustEvent:
|
|
|
412
393
|
Will process the low sample waveform.
|
|
413
394
|
|
|
414
395
|
Parse a binary string representing a low sample waveform
|
|
415
|
-
Data arrives in 32
|
|
396
|
+
Data arrives in 32-bit chunks, divided up into:
|
|
416
397
|
* 8 bits of padding
|
|
417
398
|
* 2x12 bits of integer data.
|
|
418
399
|
|
|
@@ -515,55 +496,54 @@ class RawDustEvent:
|
|
|
515
496
|
dataset : xarray.Dataset
|
|
516
497
|
A Dataset object containing the data from a single impact.
|
|
517
498
|
"""
|
|
518
|
-
# Create object for CDF attrs
|
|
499
|
+
# Create an object for CDF attrs
|
|
519
500
|
idex_attrs = self.cdf_attrs
|
|
520
501
|
|
|
521
|
-
# Gather the huge
|
|
502
|
+
# Gather the huge amount of metadata info
|
|
522
503
|
trigger_vars = {}
|
|
523
|
-
for var, value in self.
|
|
524
|
-
trigger_desc = TRIGGER_DESCRIPTION_DICT[var]
|
|
504
|
+
for var, value in self.telemetry_items.items():
|
|
525
505
|
trigger_vars[var] = xr.DataArray(
|
|
526
506
|
name=var,
|
|
527
507
|
data=[value],
|
|
528
508
|
dims=("epoch"),
|
|
529
|
-
attrs=idex_attrs.get_variable_attributes(
|
|
509
|
+
attrs=idex_attrs.get_variable_attributes(var),
|
|
530
510
|
)
|
|
531
511
|
|
|
532
512
|
# Process the 6 primary data variables
|
|
533
513
|
tof_high_xr = xr.DataArray(
|
|
534
514
|
name="TOF_High",
|
|
535
515
|
data=[self._parse_high_sample_waveform(self.TOF_High_bits)],
|
|
536
|
-
dims=("epoch", "
|
|
516
|
+
dims=("epoch", "time_high_sr_dim"),
|
|
537
517
|
attrs=idex_attrs.get_variable_attributes("tof_high_attrs"),
|
|
538
518
|
)
|
|
539
519
|
tof_low_xr = xr.DataArray(
|
|
540
520
|
name="TOF_Low",
|
|
541
521
|
data=[self._parse_high_sample_waveform(self.TOF_Low_bits)],
|
|
542
|
-
dims=("epoch", "
|
|
522
|
+
dims=("epoch", "time_high_sr_dim"),
|
|
543
523
|
attrs=idex_attrs.get_variable_attributes("tof_low_attrs"),
|
|
544
524
|
)
|
|
545
525
|
tof_mid_xr = xr.DataArray(
|
|
546
526
|
name="TOF_Mid",
|
|
547
527
|
data=[self._parse_high_sample_waveform(self.TOF_Mid_bits)],
|
|
548
|
-
dims=("epoch", "
|
|
528
|
+
dims=("epoch", "time_high_sr_dim"),
|
|
549
529
|
attrs=idex_attrs.get_variable_attributes("tof_mid_attrs"),
|
|
550
530
|
)
|
|
551
531
|
target_high_xr = xr.DataArray(
|
|
552
532
|
name="Target_High",
|
|
553
533
|
data=[self._parse_low_sample_waveform(self.Target_High_bits)],
|
|
554
|
-
dims=("epoch", "
|
|
534
|
+
dims=("epoch", "time_low_sr_dim"),
|
|
555
535
|
attrs=idex_attrs.get_variable_attributes("target_high_attrs"),
|
|
556
536
|
)
|
|
557
537
|
target_low_xr = xr.DataArray(
|
|
558
538
|
name="Target_Low",
|
|
559
539
|
data=[self._parse_low_sample_waveform(self.Target_Low_bits)],
|
|
560
|
-
dims=("epoch", "
|
|
540
|
+
dims=("epoch", "time_low_sr_dim"),
|
|
561
541
|
attrs=idex_attrs.get_variable_attributes("target_low_attrs"),
|
|
562
542
|
)
|
|
563
543
|
ion_grid_xr = xr.DataArray(
|
|
564
544
|
name="Ion_Grid",
|
|
565
545
|
data=[self._parse_low_sample_waveform(self.Ion_Grid_bits)],
|
|
566
|
-
dims=("epoch", "
|
|
546
|
+
dims=("epoch", "time_low_sr_dim"),
|
|
567
547
|
attrs=idex_attrs.get_variable_attributes("ion_grid_attrs"),
|
|
568
548
|
)
|
|
569
549
|
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Perform IDEX L1b Processing.
|
|
3
|
+
|
|
4
|
+
Examples
|
|
5
|
+
--------
|
|
6
|
+
.. code-block:: python
|
|
7
|
+
|
|
8
|
+
from imap_processing.idex.idex_l1a import PacketParser
|
|
9
|
+
from imap_processing.idex.idex_l1b import idex_l1b
|
|
10
|
+
|
|
11
|
+
l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
|
|
12
|
+
l1a_data = PacketParser(l0_file, data_version)
|
|
13
|
+
l1b_data = idex_l1b(l1a_data, data_version)
|
|
14
|
+
write_cdf(l1b_data)
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
from enum import Enum
|
|
19
|
+
|
|
20
|
+
import pandas as pd
|
|
21
|
+
import xarray as xr
|
|
22
|
+
|
|
23
|
+
from imap_processing import imap_module_directory
|
|
24
|
+
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
|
|
25
|
+
from imap_processing.utils import convert_raw_to_eu
|
|
26
|
+
|
|
27
|
+
logger = logging.getLogger(__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ConversionFactors(float, Enum):
|
|
31
|
+
"""Enum class for conversion factor values."""
|
|
32
|
+
|
|
33
|
+
TOF_High = 2.89e-4
|
|
34
|
+
TOF_Low = 5.14e-4
|
|
35
|
+
TOF_Mid = 1.13e-2
|
|
36
|
+
Target_Low = 1.58e1
|
|
37
|
+
Target_High = 1.63e-1
|
|
38
|
+
Ion_Grid = 7.46e-4
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
42
|
+
"""
|
|
43
|
+
Will process IDEX l1a data to create l1b data products.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
l1a_dataset : xarray.Dataset
|
|
48
|
+
IDEX L1a dataset to process.
|
|
49
|
+
data_version : str
|
|
50
|
+
Version of the data product being created.
|
|
51
|
+
|
|
52
|
+
Returns
|
|
53
|
+
-------
|
|
54
|
+
l1b_dataset : xarray.Dataset
|
|
55
|
+
The``xarray`` dataset containing the science data and supporting metadata.
|
|
56
|
+
"""
|
|
57
|
+
logger.info(
|
|
58
|
+
f"Running IDEX L1B processing on dataset: {l1a_dataset.attrs['Logical_source']}"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# create the attribute manager for this data level
|
|
62
|
+
idex_attrs = ImapCdfAttributes()
|
|
63
|
+
idex_attrs.add_instrument_global_attrs(instrument="idex")
|
|
64
|
+
idex_attrs.add_instrument_variable_attrs(instrument="idex", level="l1b")
|
|
65
|
+
idex_attrs.add_global_attribute("Data_version", data_version)
|
|
66
|
+
|
|
67
|
+
var_information_path = (
|
|
68
|
+
f"{imap_module_directory}/idex/idex_variable_unpacking_and_eu_conversion.csv"
|
|
69
|
+
)
|
|
70
|
+
# Read in csv that contains instrument variable setting information
|
|
71
|
+
var_information_df = pd.read_csv(var_information_path)
|
|
72
|
+
|
|
73
|
+
processed_vars = unpack_instrument_settings(
|
|
74
|
+
l1a_dataset, var_information_df, idex_attrs
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
waveforms_converted = convert_waveforms(l1a_dataset, idex_attrs)
|
|
78
|
+
|
|
79
|
+
epoch_da = xr.DataArray(
|
|
80
|
+
l1a_dataset["epoch"],
|
|
81
|
+
name="epoch",
|
|
82
|
+
dims=["epoch"],
|
|
83
|
+
attrs=idex_attrs.get_variable_attributes("epoch"),
|
|
84
|
+
)
|
|
85
|
+
# Create l1b Dataset
|
|
86
|
+
l1b_dataset = xr.Dataset(
|
|
87
|
+
coords={"epoch": epoch_da},
|
|
88
|
+
data_vars=processed_vars | waveforms_converted,
|
|
89
|
+
attrs=idex_attrs.get_global_attributes("imap_idex_l1b_sci"),
|
|
90
|
+
)
|
|
91
|
+
# Convert variables
|
|
92
|
+
l1b_dataset = convert_raw_to_eu(
|
|
93
|
+
l1b_dataset,
|
|
94
|
+
conversion_table_path=var_information_path,
|
|
95
|
+
packet_name="IDEX_SCI",
|
|
96
|
+
)
|
|
97
|
+
vars_to_copy = [
|
|
98
|
+
"shcoarse",
|
|
99
|
+
"shfine",
|
|
100
|
+
"time_high_sr",
|
|
101
|
+
"time_low_sr",
|
|
102
|
+
"time_high_sr_label",
|
|
103
|
+
"time_low_sr_label",
|
|
104
|
+
]
|
|
105
|
+
# Copy arrays from the l1a_dataset that do not need l1b processing
|
|
106
|
+
for var in vars_to_copy:
|
|
107
|
+
l1b_dataset[var] = l1a_dataset[var].copy()
|
|
108
|
+
|
|
109
|
+
# TODO: Add TriggerMode and TriggerLevel attr
|
|
110
|
+
# TODO: Spice data?
|
|
111
|
+
|
|
112
|
+
logger.info("IDEX L1B science data processing completed.")
|
|
113
|
+
|
|
114
|
+
return l1b_dataset
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def unpack_instrument_settings(
|
|
118
|
+
l1a_dataset: xr.Dataset,
|
|
119
|
+
var_information_df: pd.DataFrame,
|
|
120
|
+
idex_attrs: ImapCdfAttributes,
|
|
121
|
+
) -> dict[str, xr.DataArray]:
|
|
122
|
+
"""
|
|
123
|
+
Unpack raw telemetry data from the l1a dataset into individual variables.
|
|
124
|
+
|
|
125
|
+
Parameters
|
|
126
|
+
----------
|
|
127
|
+
l1a_dataset : xarray.Dataset
|
|
128
|
+
IDEX L1a dataset containing the 6 waveform arrays.
|
|
129
|
+
var_information_df : pd.DataFrame
|
|
130
|
+
Pandas data frame that contains information about each variable
|
|
131
|
+
(e.g., bit-size, starting bit, and padding). This is used to unpack raw
|
|
132
|
+
telemetry data from the input dataset (`l1a_dataset`).
|
|
133
|
+
idex_attrs : ImapCdfAttributes
|
|
134
|
+
CDF attribute manager object.
|
|
135
|
+
|
|
136
|
+
Returns
|
|
137
|
+
-------
|
|
138
|
+
telemetry_data : dict
|
|
139
|
+
A dictionary where the keys are the instrument setting array names and the
|
|
140
|
+
values are the unpacked xr.DataArrays.
|
|
141
|
+
"""
|
|
142
|
+
telemetry_data = {}
|
|
143
|
+
|
|
144
|
+
for _, row in var_information_df.iterrows():
|
|
145
|
+
unpacked_name = row["mnemonic"]
|
|
146
|
+
|
|
147
|
+
# Create binary mask of the size of the variable in bits
|
|
148
|
+
mask = (1 << row["unsigned_nbits"]) - 1
|
|
149
|
+
# Determine the number of bits to shift
|
|
150
|
+
shift = row["starting_bit"] - row["nbits_padding_before"]
|
|
151
|
+
# Get the unpacked value by shifting the data to align the desired bits with
|
|
152
|
+
# the least significant bits and applying the mask to isolate the target bits
|
|
153
|
+
unpacked_val = (l1a_dataset[row["var_name"]].data >> shift) & mask
|
|
154
|
+
|
|
155
|
+
telemetry_data[unpacked_name] = xr.DataArray(
|
|
156
|
+
name=unpacked_name,
|
|
157
|
+
data=unpacked_val,
|
|
158
|
+
dims=("epoch"),
|
|
159
|
+
attrs=idex_attrs.get_variable_attributes(unpacked_name),
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
return telemetry_data
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def convert_waveforms(
|
|
166
|
+
l1a_dataset: xr.Dataset, idex_attrs: ImapCdfAttributes
|
|
167
|
+
) -> dict[str, xr.DataArray]:
|
|
168
|
+
"""
|
|
169
|
+
Apply transformation from raw DN to picocoulombs (pC) for each of the six waveforms.
|
|
170
|
+
|
|
171
|
+
Parameters
|
|
172
|
+
----------
|
|
173
|
+
l1a_dataset : xarray.Dataset
|
|
174
|
+
IDEX L1a dataset containing the six waveform arrays.
|
|
175
|
+
idex_attrs : ImapCdfAttributes
|
|
176
|
+
CDF attribute manager object.
|
|
177
|
+
|
|
178
|
+
Returns
|
|
179
|
+
-------
|
|
180
|
+
waveforms_converted : dict
|
|
181
|
+
A dictionary where the keys are the waveform array names and the values are
|
|
182
|
+
xr.DataArrays representing the waveforms transformed into picocoulombs.
|
|
183
|
+
"""
|
|
184
|
+
waveforms_pc = {}
|
|
185
|
+
|
|
186
|
+
for var in ConversionFactors:
|
|
187
|
+
waveforms_pc[var.name] = l1a_dataset[var.name] * var.value
|
|
188
|
+
waveforms_pc[var.name].attrs = idex_attrs.get_variable_attributes(
|
|
189
|
+
var.name.lower()
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
return waveforms_pc
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
index,mnemonic,var_name,starting_bit,nbits_padding_before,unsigned_nbits,unit,c0,c1,c2,c3,c4,c5,c6,c7,convertAs,packetName
|
|
2
|
+
1,current_1v_pol,idx__txhdrprochkch01,4,4,12,mA,0,0.9029253,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
3
|
+
2,current_1p9v_pol,idx__txhdrprochkch01,20,4,12,mA,0,0.9029253,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
4
|
+
3,temperature_1,idx__txhdrprochkch23,4,4,12,C,162.8276,-0.2668687,0.000332379,-2.43E-07,9.03E-11,-1.33E-14,0.00E+00,0.00E+00,UNSEGMENTED_POLY,IDEX_SCI
|
|
5
|
+
4,temperature_2,idx__txhdrprochkch23,20,4,12,C,162.8276,-0.2668687,0.000332379,-2.43E-07,9.03E-11,-1.33E-14,0.00E+00,0.00E+00,UNSEGMENTED_POLY,IDEX_SCI
|
|
6
|
+
5,voltage_1v_bus,idx__txhdrprochkch45,4,4,12,V,0,0.000805861,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
7
|
+
6,fpga_temperature,idx__txhdrprochkch45,20,4,12,C,358.31,-0.1187104,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
8
|
+
7,voltage_1p9v_bus,idx__txhdrprochkch67,4,4,12,V,0,0.000805861,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
9
|
+
8,voltage_3p3v_bus,idx__txhdrprochkch67,20,4,12,V,0,0.001611722,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
10
|
+
9,detector_voltage,idx__txhdrhvpshkch01,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
11
|
+
10,sensor_voltage,idx__txhdrhvpshkch01,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
12
|
+
11,target_voltage,idx__txhdrhvpshkch23,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
13
|
+
12,reflectron_voltage,idx__txhdrhvpshkch23,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
14
|
+
13,rejection_voltage,idx__txhdrhvpshkch45,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
15
|
+
14,current_hvps_sensor,idx__txhdrhvpshkch45,20,4,12,mA,0,7.33E-06,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
16
|
+
15,positive_current_hvps,idx__txhdrhvpshkch67,4,4,12,mA,0,2.43E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
17
|
+
16,negative_current_hvps,idx__txhdrhvpshkch67,20,4,12,mA,0,2.43E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
18
|
+
17,voltage_3p3_ref,idx__txhdrlvhk0ch01,4,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
19
|
+
18,voltage_3p3_op_ref,idx__txhdrlvhk0ch01,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
20
|
+
19,voltage_neg6v_bus,idx__txhdrlvhk0ch23,4,4,12,V,-33,0.00886447,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
21
|
+
20,voltage_pos6v_bus,idx__txhdrlvhk0ch23,20,4,12,V,0,0.00241758,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
22
|
+
21,voltage_pos16v_bus,idx__txhdrlvhk0ch45,4,4,12,V,0,0.00482711,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
23
|
+
22,voltage_pos3p3v_bus,idx__txhdrlvhk0ch45,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
24
|
+
23,voltage_neg5v_bus,idx__txhdrlvhk0ch67,4,4,12,V,-33,0.00886447,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
25
|
+
24,voltage_pos5v_bus,idx__txhdrlvhk0ch67,20,4,12,V,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
26
|
+
25,current_3p3_bus,idx__txhdrlvhk1ch01,4,4,12,A,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
27
|
+
26,current_16v_bus,idx__txhdrlvhk1ch01,20,4,12,A,0,0.000161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
28
|
+
27,current_6v_bus,idx__txhdrlvhk1ch23,4,4,12,A,0,8.06E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
29
|
+
28,current_neg6v_bus,idx__txhdrlvhk1ch23,20,4,12,A,0,-8.06E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
30
|
+
29,current_5v_bus,idx__txhdrlvhk1ch45,4,4,12,A,0,4.03E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
31
|
+
30,current_neg5v_bus,idx__txhdrlvhk1ch45,20,4,12,A,0,-2.01E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
32
|
+
31,current_2p5v_bus,idx__txhdrlvhk1ch67,4,4,12,A,0,0.00161172,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|
|
33
|
+
32,current_neg2p5v_bus,idx__txhdrlvhk1ch67,20,4,12,A,0,-4.03E-05,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI
|