imap-processing 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imap-processing might be problematic. Click here for more details.
- imap_processing/__init__.py +1 -1
- imap_processing/_version.py +2 -2
- imap_processing/ccsds/excel_to_xtce.py +34 -2
- imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
- imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
- imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
- imap_processing/cdf/config/imap_hi_variable_attrs.yaml +36 -8
- imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
- imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +7 -7
- imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +32 -33
- imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
- imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
- imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +133 -78
- imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
- imap_processing/cdf/imap_cdf_manager.py +31 -27
- imap_processing/cli.py +12 -10
- imap_processing/codice/codice_l1a.py +151 -61
- imap_processing/codice/constants.py +1 -1
- imap_processing/codice/decompress.py +4 -9
- imap_processing/codice/utils.py +1 -0
- imap_processing/glows/l1b/glows_l1b.py +3 -3
- imap_processing/glows/l1b/glows_l1b_data.py +59 -37
- imap_processing/glows/l2/glows_l2_data.py +123 -0
- imap_processing/hi/l1a/histogram.py +1 -1
- imap_processing/hi/l1a/science_direct_event.py +1 -1
- imap_processing/hi/l1b/hi_l1b.py +85 -11
- imap_processing/hi/l1c/hi_l1c.py +23 -1
- imap_processing/hi/utils.py +1 -1
- imap_processing/hit/hit_utils.py +221 -0
- imap_processing/hit/l0/constants.py +118 -0
- imap_processing/hit/l0/decom_hit.py +186 -153
- imap_processing/hit/l1a/hit_l1a.py +20 -175
- imap_processing/hit/l1b/hit_l1b.py +33 -153
- imap_processing/idex/idex_l1a.py +10 -9
- imap_processing/lo/l0/decompression_tables/decompression_tables.py +1 -1
- imap_processing/lo/l0/lo_science.py +1 -1
- imap_processing/lo/packet_definitions/lo_xtce.xml +1 -3296
- imap_processing/mag/l0/decom_mag.py +4 -3
- imap_processing/mag/l1a/mag_l1a.py +11 -11
- imap_processing/mag/l1b/mag_l1b.py +89 -7
- imap_processing/spice/geometry.py +126 -4
- imap_processing/swapi/l1/swapi_l1.py +1 -1
- imap_processing/swapi/l2/swapi_l2.py +1 -1
- imap_processing/swe/l1b/swe_l1b_science.py +8 -8
- imap_processing/tests/ccsds/test_data/expected_output.xml +1 -0
- imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
- imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
- imap_processing/tests/codice/conftest.py +1 -17
- imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
- imap_processing/tests/codice/test_codice_l0.py +8 -2
- imap_processing/tests/codice/test_codice_l1a.py +127 -107
- imap_processing/tests/codice/test_codice_l1b.py +1 -0
- imap_processing/tests/codice/test_decompress.py +7 -7
- imap_processing/tests/conftest.py +54 -15
- imap_processing/tests/glows/conftest.py +6 -0
- imap_processing/tests/glows/test_glows_l1b.py +9 -9
- imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
- imap_processing/tests/glows/test_glows_l2_data.py +0 -0
- imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
- imap_processing/tests/hi/test_hi_l1b.py +71 -1
- imap_processing/tests/hi/test_hi_l1c.py +10 -2
- imap_processing/tests/hi/test_utils.py +4 -3
- imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +84 -35
- imap_processing/tests/hit/test_hit_l1a.py +2 -197
- imap_processing/tests/hit/test_hit_l1b.py +156 -25
- imap_processing/tests/hit/test_hit_utils.py +218 -0
- imap_processing/tests/idex/conftest.py +1 -1
- imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
- imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
- imap_processing/tests/idex/test_idex_l0.py +3 -3
- imap_processing/tests/idex/test_idex_l1a.py +1 -1
- imap_processing/tests/lo/test_lo_science.py +2 -2
- imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
- imap_processing/tests/mag/test_mag_l1b.py +59 -3
- imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
- imap_processing/tests/spice/test_geometry.py +84 -4
- imap_processing/tests/swe/conftest.py +33 -0
- imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
- imap_processing/tests/swe/test_swe_l1b.py +29 -8
- imap_processing/tests/test_utils.py +1 -1
- imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
- imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
- imap_processing/tests/ultra/unit/test_de.py +108 -0
- imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
- imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
- imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +21 -11
- imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +9 -44
- imap_processing/ultra/constants.py +8 -3
- imap_processing/ultra/l1b/de.py +174 -30
- imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
- imap_processing/ultra/l1b/ultra_l1b_extended.py +21 -14
- imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +70 -119
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/METADATA +15 -14
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/RECORD +98 -113
- imap_processing/cdf/cdf_attribute_manager.py +0 -322
- imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
- imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
- imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
- imap_processing/hit/l0/data_classes/science_packet.py +0 -259
- imap_processing/hit/l0/utils/hit_base.py +0 -57
- imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
- imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
- imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
- imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
- imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
- imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
- /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/LICENSE +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/WHEEL +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/entry_points.txt +0 -0
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import numpy as np
|
|
4
4
|
import pytest
|
|
5
5
|
|
|
6
|
+
from imap_processing.cdf.utils import write_cdf
|
|
6
7
|
from imap_processing.hi.l1a.hi_l1a import hi_l1a
|
|
7
8
|
from imap_processing.hi.l1b.hi_l1b import hi_l1b
|
|
8
9
|
from imap_processing.hi.l1c import hi_l1c
|
|
@@ -30,19 +31,26 @@ def test_generate_pset_dataset(create_de_data):
|
|
|
30
31
|
def test_allocate_pset_dataset():
|
|
31
32
|
"""Test coverage for allocate_pset_dataset function"""
|
|
32
33
|
n_esa_steps = 10
|
|
34
|
+
n_calibration_prods = 5
|
|
33
35
|
sensor_str = HIAPID.H90_SCI_DE.sensor
|
|
34
36
|
dataset = hi_l1c.allocate_pset_dataset(n_esa_steps, sensor_str)
|
|
35
37
|
|
|
36
38
|
assert dataset.epoch.size == 1
|
|
37
39
|
assert dataset.spin_angle_bin.size == 3600
|
|
40
|
+
assert dataset.esa_energy_step.size == n_esa_steps
|
|
41
|
+
assert dataset.calibration_prod.size == n_calibration_prods
|
|
38
42
|
np.testing.assert_array_equal(dataset.despun_z.data.shape, (1, 3))
|
|
39
43
|
np.testing.assert_array_equal(dataset.hae_latitude.data.shape, (1, 3600))
|
|
40
44
|
np.testing.assert_array_equal(dataset.hae_longitude.data.shape, (1, 3600))
|
|
41
|
-
n_esa_step = dataset.esa_energy_step.data.size
|
|
42
45
|
for var in [
|
|
43
46
|
"counts",
|
|
44
47
|
"exposure_times",
|
|
45
48
|
"background_rates",
|
|
46
49
|
"background_rates_uncertainty",
|
|
47
50
|
]:
|
|
48
|
-
np.testing.assert_array_equal(
|
|
51
|
+
np.testing.assert_array_equal(
|
|
52
|
+
dataset[var].data.shape, (1, n_esa_steps, n_calibration_prods, 3600)
|
|
53
|
+
)
|
|
54
|
+
# Verify resulting CDF is ISTP compliant by writing to disk
|
|
55
|
+
dataset.attrs["Data_version"] = 1
|
|
56
|
+
write_cdf(dataset)
|
|
@@ -48,7 +48,7 @@ def test_parse_sensor_number(test_str, expected):
|
|
|
48
48
|
[
|
|
49
49
|
("despun_z", (1, 3), (1, 3)),
|
|
50
50
|
("hae_latitude", None, (1, 360)),
|
|
51
|
-
("counts", None, (1, 10, 360)),
|
|
51
|
+
("counts", None, (1, 10, 5, 360)),
|
|
52
52
|
],
|
|
53
53
|
)
|
|
54
54
|
def test_full_dataarray(name, shape, expected_shape):
|
|
@@ -56,10 +56,11 @@ def test_full_dataarray(name, shape, expected_shape):
|
|
|
56
56
|
coords = {
|
|
57
57
|
"epoch": xr.DataArray(np.array([0])),
|
|
58
58
|
"esa_energy_step": xr.DataArray(np.arange(10)),
|
|
59
|
+
"calibration_prod": xr.DataArray(np.arange(5)),
|
|
59
60
|
"spin_angle_bin": xr.DataArray(np.arange(360)),
|
|
60
61
|
}
|
|
61
62
|
cdf_manager = ImapCdfAttributes()
|
|
62
|
-
cdf_manager.
|
|
63
|
+
cdf_manager.add_instrument_variable_attrs(instrument="hi", level=None)
|
|
63
64
|
|
|
64
65
|
dataarray = full_dataarray(
|
|
65
66
|
name, cdf_manager.get_variable_attributes(f"hi_pset_{name}"), coords, shape
|
|
@@ -83,7 +84,7 @@ def test_create_dataset_variables(var_names, shape, lookup_str):
|
|
|
83
84
|
assert len(l1b_de_vars) == len(var_names)
|
|
84
85
|
attr_mgr = ImapCdfAttributes()
|
|
85
86
|
attr_mgr.add_instrument_global_attrs("hi")
|
|
86
|
-
attr_mgr.
|
|
87
|
+
attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
|
|
87
88
|
|
|
88
89
|
for var_name, data_array in l1b_de_vars.items():
|
|
89
90
|
attrs = attr_mgr.get_variable_attributes(
|
|
@@ -4,14 +4,18 @@ import numpy as np
|
|
|
4
4
|
import pytest
|
|
5
5
|
|
|
6
6
|
from imap_processing import imap_module_directory
|
|
7
|
+
from imap_processing.hit.hit_utils import (
|
|
8
|
+
HitAPID,
|
|
9
|
+
)
|
|
7
10
|
from imap_processing.hit.l0.decom_hit import (
|
|
8
11
|
assemble_science_frames,
|
|
9
12
|
decom_hit,
|
|
10
|
-
|
|
11
|
-
|
|
13
|
+
decompress_rates_16_to_32,
|
|
14
|
+
get_valid_starting_indices,
|
|
12
15
|
is_sequential,
|
|
13
16
|
parse_count_rates,
|
|
14
17
|
parse_data,
|
|
18
|
+
subcom_sectorates,
|
|
15
19
|
update_ccsds_header_dims,
|
|
16
20
|
)
|
|
17
21
|
from imap_processing.utils import packet_file_to_datasets
|
|
@@ -30,9 +34,10 @@ def sci_dataset():
|
|
|
30
34
|
datasets_by_apid = packet_file_to_datasets(
|
|
31
35
|
packet_file=packet_file,
|
|
32
36
|
xtce_packet_definition=packet_definition,
|
|
37
|
+
use_derived_value=False,
|
|
33
38
|
)
|
|
34
39
|
|
|
35
|
-
science_dataset = datasets_by_apid[
|
|
40
|
+
science_dataset = datasets_by_apid[HitAPID.HIT_SCIENCE]
|
|
36
41
|
return science_dataset
|
|
37
42
|
|
|
38
43
|
|
|
@@ -58,8 +63,6 @@ def test_parse_data():
|
|
|
58
63
|
def test_parse_count_rates(sci_dataset):
|
|
59
64
|
"""Test the parse_count_rates function."""
|
|
60
65
|
|
|
61
|
-
# TODO: complete this test once the function is complete
|
|
62
|
-
|
|
63
66
|
# Update ccsds header fields to use sc_tick as dimension
|
|
64
67
|
sci_dataset = update_ccsds_header_dims(sci_dataset)
|
|
65
68
|
|
|
@@ -71,7 +74,10 @@ def test_parse_count_rates(sci_dataset):
|
|
|
71
74
|
count_rate_vars = [
|
|
72
75
|
"hdr_unit_num",
|
|
73
76
|
"hdr_frame_version",
|
|
74
|
-
"
|
|
77
|
+
"hdr_dynamic_threshold_state",
|
|
78
|
+
"hdr_leak_conv",
|
|
79
|
+
"hdr_heater_duty_cycle",
|
|
80
|
+
"hdr_code_ok",
|
|
75
81
|
"hdr_minute_cnt",
|
|
76
82
|
"spare",
|
|
77
83
|
"livetime",
|
|
@@ -128,7 +134,7 @@ def test_is_sequential():
|
|
|
128
134
|
assert True
|
|
129
135
|
|
|
130
136
|
|
|
131
|
-
def
|
|
137
|
+
def test_get_valid_starting_indices():
|
|
132
138
|
"""Test the find_valid_starting_indices function."""
|
|
133
139
|
flags = np.array(
|
|
134
140
|
[
|
|
@@ -170,35 +176,13 @@ def test_find_valid_starting_indices():
|
|
|
170
176
|
]
|
|
171
177
|
)
|
|
172
178
|
counters = np.arange(35)
|
|
173
|
-
result =
|
|
179
|
+
result = get_valid_starting_indices(flags, counters)
|
|
174
180
|
# The only valid starting index for a science frame
|
|
175
181
|
# in the flags array is 15.
|
|
176
182
|
assert len(result) == 1
|
|
177
183
|
assert result[0] == 15
|
|
178
184
|
|
|
179
185
|
|
|
180
|
-
def test_get_valid_indices():
|
|
181
|
-
"""Test the get_valid_indices function."""
|
|
182
|
-
# Array of starting indices for science frames
|
|
183
|
-
# in the science data
|
|
184
|
-
indices = np.array([0, 20, 40])
|
|
185
|
-
# Array of counters
|
|
186
|
-
counters = np.arange(60)
|
|
187
|
-
# Array of valid indices where the packets in the science
|
|
188
|
-
# frame have corresponding counters in sequential order
|
|
189
|
-
result = get_valid_indices(indices, counters, 20)
|
|
190
|
-
# All indices are valid with sequential counters
|
|
191
|
-
assert len(result) == 3
|
|
192
|
-
|
|
193
|
-
# Test array with invalid indices (use smaller sample size)
|
|
194
|
-
indices = np.array([0, 5, 10])
|
|
195
|
-
# Array of counters (missing counters 6-8)
|
|
196
|
-
counters = np.array([0, 1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 15, 16, 17])
|
|
197
|
-
result = get_valid_indices(indices, counters, 5)
|
|
198
|
-
# Only indices 0 and 10 are valid with sequential counters
|
|
199
|
-
assert len(result) == 2
|
|
200
|
-
|
|
201
|
-
|
|
202
186
|
def test_update_ccsds_header_dims(sci_dataset):
|
|
203
187
|
"""Test the update_ccsds_header_data function.
|
|
204
188
|
|
|
@@ -213,8 +197,67 @@ def test_assemble_science_frames(sci_dataset):
|
|
|
213
197
|
"""Test the assemble_science_frames function."""
|
|
214
198
|
updated_dataset = update_ccsds_header_dims(sci_dataset)
|
|
215
199
|
updated_dataset = assemble_science_frames(updated_dataset)
|
|
216
|
-
assert "
|
|
217
|
-
assert "
|
|
200
|
+
assert "count_rates_raw" in updated_dataset
|
|
201
|
+
assert "pha_raw" in updated_dataset
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def test_subcom_sectorates(sci_dataset):
|
|
205
|
+
"""Test the subcom_sectorates function.
|
|
206
|
+
|
|
207
|
+
This function organizes the sector rates data
|
|
208
|
+
into new variables for each species and adds
|
|
209
|
+
them to the dataset.
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
# Prepare the input needed for the function to be called
|
|
213
|
+
sci_dataset = update_ccsds_header_dims(sci_dataset)
|
|
214
|
+
sci_dataset = assemble_science_frames(sci_dataset)
|
|
215
|
+
parse_count_rates(sci_dataset)
|
|
216
|
+
|
|
217
|
+
# Call the function to be tested
|
|
218
|
+
subcom_sectorates(sci_dataset)
|
|
219
|
+
|
|
220
|
+
# Check if the dataset has the expected new variables
|
|
221
|
+
for species in ["H", "4He", "CNO", "NeMgSi", "Fe"]:
|
|
222
|
+
assert species in sci_dataset
|
|
223
|
+
assert f"{species}_energy_min" in sci_dataset
|
|
224
|
+
assert f"{species}_energy_max" in sci_dataset
|
|
225
|
+
|
|
226
|
+
# Check the shape of the new variables
|
|
227
|
+
for species in ["H", "4He", "CNO", "NeMgSi", "Fe"]:
|
|
228
|
+
if species == "H":
|
|
229
|
+
assert sci_dataset[species].shape == (86, 3, 8, 15)
|
|
230
|
+
assert sci_dataset[f"{species}_energy_min"].shape == (3,)
|
|
231
|
+
elif species in ("4He", "CNO", "NeMgSi"):
|
|
232
|
+
assert sci_dataset[species].shape == (86, 2, 8, 15)
|
|
233
|
+
assert sci_dataset[f"{species}_energy_min"].shape == (2,)
|
|
234
|
+
elif species == "Fe":
|
|
235
|
+
assert sci_dataset[species].shape == (86, 1, 8, 15)
|
|
236
|
+
assert sci_dataset[f"{species}_energy_min"].shape == (1,)
|
|
237
|
+
assert (
|
|
238
|
+
sci_dataset[f"{species}_energy_max"].shape
|
|
239
|
+
== sci_dataset[f"{species}_energy_min"].shape
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@pytest.mark.parametrize(
|
|
244
|
+
"packed, expected",
|
|
245
|
+
[
|
|
246
|
+
(0, 0), # Test with zero
|
|
247
|
+
(15, 15), # Test with packed integer with no scaling
|
|
248
|
+
(4096, 4096), # Test with packed integer with power = 1
|
|
249
|
+
(64188, 112132096), # Test with packed integer requiring scaling
|
|
250
|
+
(65535, 134201344), # Test with maximum 16-bit value
|
|
251
|
+
(62218, 79855616), # Test with arbitrary packed integer
|
|
252
|
+
],
|
|
253
|
+
)
|
|
254
|
+
def test_decompress_rates_16_to_32(packed, expected):
|
|
255
|
+
"""Test the decompress_rates_16_to_32 function.
|
|
256
|
+
|
|
257
|
+
This function decompresses a 16-bit packed integer
|
|
258
|
+
to a 32-bit integer. Used to decompress rates data.
|
|
259
|
+
"""
|
|
260
|
+
assert decompress_rates_16_to_32(packed) == expected
|
|
218
261
|
|
|
219
262
|
|
|
220
263
|
def test_decom_hit(sci_dataset):
|
|
@@ -225,6 +268,12 @@ def test_decom_hit(sci_dataset):
|
|
|
225
268
|
"""
|
|
226
269
|
# TODO: complete this test once the function is complete
|
|
227
270
|
updated_dataset = decom_hit(sci_dataset)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
assert "
|
|
271
|
+
# Check if the dataset has the expected new variables
|
|
272
|
+
# Check that binary science data exists
|
|
273
|
+
assert "count_rates_raw" in updated_dataset
|
|
274
|
+
assert "pha_raw" in updated_dataset
|
|
275
|
+
# Check that sector rates data has been organized
|
|
276
|
+
for species in ["H", "4He", "CNO", "NeMgSi", "Fe"]:
|
|
277
|
+
assert species in updated_dataset
|
|
278
|
+
assert f"{species}_energy_min" in updated_dataset
|
|
279
|
+
assert f"{species}_energy_max" in updated_dataset
|
|
@@ -1,55 +1,18 @@
|
|
|
1
|
-
import numpy as np
|
|
2
1
|
import pytest
|
|
3
2
|
import xarray as xr
|
|
4
3
|
|
|
5
4
|
from imap_processing import imap_module_directory
|
|
6
|
-
from imap_processing.
|
|
7
|
-
from imap_processing.hit.l1a.hit_l1a import (
|
|
8
|
-
HitAPID,
|
|
9
|
-
concatenate_leak_variables,
|
|
10
|
-
hit_l1a,
|
|
11
|
-
process_housekeeping,
|
|
12
|
-
)
|
|
13
|
-
from imap_processing.utils import packet_file_to_datasets
|
|
5
|
+
from imap_processing.hit.l1a.hit_l1a import hit_l1a
|
|
14
6
|
|
|
15
7
|
|
|
16
8
|
@pytest.fixture(scope="module")
|
|
17
9
|
def packet_filepath():
|
|
18
10
|
"""Set path to test data file"""
|
|
19
11
|
return (
|
|
20
|
-
imap_module_directory / "tests/hit/test_data/
|
|
12
|
+
imap_module_directory / "tests/hit/test_data/imap_hit_l0_raw_20100105_v001.pkts"
|
|
21
13
|
)
|
|
22
14
|
|
|
23
15
|
|
|
24
|
-
@pytest.fixture(scope="module")
|
|
25
|
-
def datasets(packet_filepath):
|
|
26
|
-
"""Create datasets from packet file"""
|
|
27
|
-
packet_definition = (
|
|
28
|
-
imap_module_directory / "hit/packet_definitions/" "hit_packet_definitions.xml"
|
|
29
|
-
)
|
|
30
|
-
datasets_by_apid = packet_file_to_datasets(
|
|
31
|
-
packet_file=packet_filepath,
|
|
32
|
-
xtce_packet_definition=packet_definition,
|
|
33
|
-
)
|
|
34
|
-
return datasets_by_apid
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
@pytest.fixture(scope="module")
|
|
38
|
-
def attribute_manager():
|
|
39
|
-
"""Create the attribute manager"""
|
|
40
|
-
attr_mgr = ImapCdfAttributes()
|
|
41
|
-
attr_mgr.add_instrument_global_attrs(instrument="hit")
|
|
42
|
-
attr_mgr.add_instrument_variable_attrs(instrument="hit", level="l1a")
|
|
43
|
-
attr_mgr.add_global_attribute("Data_version", "001")
|
|
44
|
-
return attr_mgr
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
@pytest.fixture(scope="module")
|
|
48
|
-
def housekeeping_dataset(datasets):
|
|
49
|
-
"""Get the housekeeping dataset"""
|
|
50
|
-
return datasets[HitAPID.HIT_HSKP]
|
|
51
|
-
|
|
52
|
-
|
|
53
16
|
def test_hit_l1a(packet_filepath):
|
|
54
17
|
"""Create L1A datasets from a packet file.
|
|
55
18
|
|
|
@@ -64,161 +27,3 @@ def test_hit_l1a(packet_filepath):
|
|
|
64
27
|
assert len(processed_datasets) == 1
|
|
65
28
|
assert isinstance(processed_datasets[0], xr.Dataset)
|
|
66
29
|
assert processed_datasets[0].attrs["Logical_source"] == "imap_hit_l1a_hk"
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def test_concatenate_leak_variables(housekeeping_dataset):
|
|
70
|
-
"""Test concatenation of leak_i variables"""
|
|
71
|
-
|
|
72
|
-
# Create data array for leak_i dependency
|
|
73
|
-
adc_channels = xr.DataArray(
|
|
74
|
-
np.arange(64, dtype=np.uint8),
|
|
75
|
-
name="adc_channels",
|
|
76
|
-
dims=["adc_channels"],
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
updated_dataset = concatenate_leak_variables(housekeeping_dataset, adc_channels)
|
|
80
|
-
|
|
81
|
-
# Assertions
|
|
82
|
-
# ----------------
|
|
83
|
-
assert "leak_i" in updated_dataset
|
|
84
|
-
assert updated_dataset["leak_i"].shape == (88, 64)
|
|
85
|
-
for i in range(64):
|
|
86
|
-
# Check if the values in the `leak_i` variable match the values in
|
|
87
|
-
# the original `leak_i_XX` variable.
|
|
88
|
-
# - First access the `leak_i` variable in the `updated_dataset`.
|
|
89
|
-
# The [:, i] selects all rows (`:`) and the `i`-th column of the `leak_i`
|
|
90
|
-
# variable.
|
|
91
|
-
# - Then access the `leak_i_XX` variable in the `housekeeping_dataset`.
|
|
92
|
-
# The `f"leak_i_{i:02d}"` selects the variable with the name `leak_i_XX`
|
|
93
|
-
# where `XX` is the `i`-th value.
|
|
94
|
-
# - Compare values
|
|
95
|
-
np.testing.assert_array_equal(
|
|
96
|
-
updated_dataset["leak_i"][:, i], housekeeping_dataset[f"leak_i_{i:02d}"]
|
|
97
|
-
)
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def test_process_housekeeping(housekeeping_dataset, attribute_manager):
|
|
101
|
-
"""Test processing of housekeeping dataset"""
|
|
102
|
-
|
|
103
|
-
# Call the function
|
|
104
|
-
processed_hskp_dataset = process_housekeeping(
|
|
105
|
-
housekeeping_dataset, attribute_manager
|
|
106
|
-
)
|
|
107
|
-
|
|
108
|
-
# Define the keys that should have dropped from the dataset
|
|
109
|
-
dropped_keys = {
|
|
110
|
-
"pkt_apid",
|
|
111
|
-
"sc_tick",
|
|
112
|
-
"version",
|
|
113
|
-
"type",
|
|
114
|
-
"sec_hdr_flg",
|
|
115
|
-
"seq_flgs",
|
|
116
|
-
"src_seq_ctr",
|
|
117
|
-
"pkt_len",
|
|
118
|
-
"hskp_spare1",
|
|
119
|
-
"hskp_spare2",
|
|
120
|
-
"hskp_spare3",
|
|
121
|
-
"hskp_spare4",
|
|
122
|
-
"hskp_spare5",
|
|
123
|
-
}
|
|
124
|
-
# Define the keys that should be present
|
|
125
|
-
valid_keys = {
|
|
126
|
-
"heater_on",
|
|
127
|
-
"fsw_version_b",
|
|
128
|
-
"ebox_m12va",
|
|
129
|
-
"phasic_stat",
|
|
130
|
-
"ebox_3d4vd",
|
|
131
|
-
"ebox_p2d0vd",
|
|
132
|
-
"temp1",
|
|
133
|
-
"last_bad_seq_num",
|
|
134
|
-
"ebox_m5d7va",
|
|
135
|
-
"ebox_p12va",
|
|
136
|
-
"table_status",
|
|
137
|
-
"enable_50khz",
|
|
138
|
-
"mram_disabled",
|
|
139
|
-
"temp3",
|
|
140
|
-
"preamp_l1a",
|
|
141
|
-
"l2ab_bias",
|
|
142
|
-
"l34b_bias",
|
|
143
|
-
"fsw_version_c",
|
|
144
|
-
"num_evnt_last_hk",
|
|
145
|
-
"dac1_enable",
|
|
146
|
-
"preamp_l234b",
|
|
147
|
-
"analog_temp",
|
|
148
|
-
"fee_running",
|
|
149
|
-
"fsw_version_a",
|
|
150
|
-
"num_errors",
|
|
151
|
-
"test_pulser_on",
|
|
152
|
-
"dac0_enable",
|
|
153
|
-
"preamp_l1b",
|
|
154
|
-
"l1ab_bias",
|
|
155
|
-
"l34a_bias",
|
|
156
|
-
"leak_i",
|
|
157
|
-
"last_good_cmd",
|
|
158
|
-
"lvps_temp",
|
|
159
|
-
"idpu_temp",
|
|
160
|
-
"temp2",
|
|
161
|
-
"preamp_l234a",
|
|
162
|
-
"last_good_seq_num",
|
|
163
|
-
"num_good_cmds",
|
|
164
|
-
"heater_control",
|
|
165
|
-
"hvps_temp",
|
|
166
|
-
"ebox_p5d7va",
|
|
167
|
-
"spin_period_long",
|
|
168
|
-
"enable_hvps",
|
|
169
|
-
"temp0",
|
|
170
|
-
"spin_period_short",
|
|
171
|
-
"dyn_thresh_lvl",
|
|
172
|
-
"num_bad_cmds",
|
|
173
|
-
"adc_mode",
|
|
174
|
-
"ebox_5d1vd",
|
|
175
|
-
"active_heater",
|
|
176
|
-
"last_error_num",
|
|
177
|
-
"last_bad_cmd",
|
|
178
|
-
"ref_p5v",
|
|
179
|
-
"code_checksum",
|
|
180
|
-
"mode",
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
# Define the dataset attributes
|
|
184
|
-
dataset_attrs = {
|
|
185
|
-
"Data_level": "1A",
|
|
186
|
-
"Data_type": "L1A_HK>Level-1A Housekeeping",
|
|
187
|
-
"Data_version": "001",
|
|
188
|
-
"Descriptor": "HIT>IMAP High-energy Ion Telescope",
|
|
189
|
-
"Discipline": "Solar Physics>Heliospheric Physics",
|
|
190
|
-
"File_naming_convention": "source_descriptor_datatype_yyyyMMdd_vNNN",
|
|
191
|
-
"HTTP_LINK": "https://imap.princeton.edu/",
|
|
192
|
-
"Instrument_type": "Particles (space)",
|
|
193
|
-
"LINK_TITLE": "IMAP The Interstellar Mapping and Acceleration Probe",
|
|
194
|
-
"Logical_file_id": None,
|
|
195
|
-
"Logical_source": "imap_hit_l1a_hk",
|
|
196
|
-
"Logical_source_description": "IMAP Mission HIT Instrument Level-1A "
|
|
197
|
-
"Housekeeping Data.",
|
|
198
|
-
"Mission_group": "IMAP",
|
|
199
|
-
"PI_affiliation": "Princeton University",
|
|
200
|
-
"PI_name": "Prof. David J. McComas",
|
|
201
|
-
"Project": "STP>Solar Terrestrial Probes",
|
|
202
|
-
"Source_name": "IMAP>Interstellar Mapping and Acceleration Probe",
|
|
203
|
-
"TEXT": "The High-energy Ion Telescope (HIT) measures the elemental "
|
|
204
|
-
"composition, energy spectra, angle distributions, and arrival "
|
|
205
|
-
"times of high-energy ions. HIT delivers full-sky coverage from "
|
|
206
|
-
"a wide instrument field-of-view (FOV) to enable a high resolution "
|
|
207
|
-
"of ion measurements, such as observing shock-accelerated ions, "
|
|
208
|
-
"determining the origin of the solar energetic particles (SEPs) "
|
|
209
|
-
"spectra, and resolving particle transport in the heliosphere. "
|
|
210
|
-
"See https://imap.princeton.edu/instruments/hit for more details.\n",
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
# Define the coordinates and dimensions. Both have equivalent values
|
|
214
|
-
dataset_coords_dims = {"epoch", "adc_channels", "adc_channels_label"}
|
|
215
|
-
|
|
216
|
-
# Assertions
|
|
217
|
-
# ----------------
|
|
218
|
-
# Check that the dataset has the correct variables
|
|
219
|
-
assert valid_keys == set(processed_hskp_dataset.data_vars.keys())
|
|
220
|
-
assert set(dropped_keys).isdisjoint(set(processed_hskp_dataset.data_vars.keys()))
|
|
221
|
-
# Check that the dataset has the correct attributes, coordinates, and dimensions
|
|
222
|
-
assert processed_hskp_dataset.attrs == dataset_attrs
|
|
223
|
-
assert processed_hskp_dataset.coords.keys() == dataset_coords_dims
|
|
224
|
-
assert processed_hskp_dataset.sizes.keys() == dataset_coords_dims
|
|
@@ -2,51 +2,182 @@ import pytest
|
|
|
2
2
|
import xarray as xr
|
|
3
3
|
|
|
4
4
|
from imap_processing import imap_module_directory
|
|
5
|
-
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
|
|
6
5
|
from imap_processing.hit.l1a import hit_l1a
|
|
7
6
|
from imap_processing.hit.l1b import hit_l1b
|
|
8
7
|
|
|
9
8
|
|
|
9
|
+
@pytest.fixture(scope="module")
|
|
10
|
+
def packet_filepath():
|
|
11
|
+
"""Set path to test data file"""
|
|
12
|
+
return (
|
|
13
|
+
imap_module_directory / "tests/hit/test_data/imap_hit_l0_raw_20100105_v001.pkts"
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
10
17
|
@pytest.fixture()
|
|
11
|
-
def
|
|
12
|
-
"""Get
|
|
18
|
+
def dependencies(packet_filepath):
|
|
19
|
+
"""Get dependencies for L1B processing"""
|
|
20
|
+
# Create dictionary of dependencies and add CCSDS packet file
|
|
21
|
+
data_dict = {"imap_hit_l0_raw": packet_filepath}
|
|
22
|
+
# Add L1A datasets
|
|
23
|
+
l1a_datasets = hit_l1a.hit_l1a(packet_filepath, "001")
|
|
24
|
+
for dataset in l1a_datasets:
|
|
25
|
+
data_dict[dataset.attrs["Logical_source"]] = dataset
|
|
26
|
+
return data_dict
|
|
13
27
|
|
|
14
|
-
packet_filepath = (
|
|
15
|
-
imap_module_directory / "tests/hit/test_data/imap_hit_l0_hk_20100105_v001.pkts"
|
|
16
|
-
)
|
|
17
|
-
l1a_data = hit_l1a.hit_l1a(packet_filepath, "001")[0]
|
|
18
28
|
|
|
19
|
-
|
|
29
|
+
@pytest.fixture()
|
|
30
|
+
def hk_dataset(dependencies):
|
|
31
|
+
"""Get the housekeeping dataset"""
|
|
32
|
+
datasets = hit_l1b.hit_l1b(dependencies, "001")
|
|
33
|
+
for dataset in datasets:
|
|
34
|
+
if dataset.attrs["Logical_source"] == "imap_hit_l1b_hk":
|
|
35
|
+
return dataset
|
|
20
36
|
|
|
21
37
|
|
|
22
|
-
def
|
|
23
|
-
"""Test
|
|
38
|
+
def test_hit_l1b_hk_dataset_variables(hk_dataset):
|
|
39
|
+
"""Test the variables in the housekeeping dataset"""
|
|
40
|
+
# Define the keys that should have dropped from the housekeeping dataset
|
|
41
|
+
dropped_keys = {
|
|
42
|
+
"pkt_apid",
|
|
43
|
+
"sc_tick",
|
|
44
|
+
"version",
|
|
45
|
+
"type",
|
|
46
|
+
"sec_hdr_flg",
|
|
47
|
+
"seq_flgs",
|
|
48
|
+
"src_seq_ctr",
|
|
49
|
+
"pkt_len",
|
|
50
|
+
"hskp_spare1",
|
|
51
|
+
"hskp_spare2",
|
|
52
|
+
"hskp_spare3",
|
|
53
|
+
"hskp_spare4",
|
|
54
|
+
"hskp_spare5",
|
|
55
|
+
}
|
|
56
|
+
# Define the keys that should be present in the housekeeping dataset
|
|
57
|
+
valid_keys = {
|
|
58
|
+
"heater_on",
|
|
59
|
+
"fsw_version_b",
|
|
60
|
+
"ebox_m12va",
|
|
61
|
+
"phasic_stat",
|
|
62
|
+
"ebox_3d4vd",
|
|
63
|
+
"ebox_p2d0vd",
|
|
64
|
+
"temp1",
|
|
65
|
+
"last_bad_seq_num",
|
|
66
|
+
"ebox_m5d7va",
|
|
67
|
+
"ebox_p12va",
|
|
68
|
+
"table_status",
|
|
69
|
+
"enable_50khz",
|
|
70
|
+
"mram_disabled",
|
|
71
|
+
"temp3",
|
|
72
|
+
"preamp_l1a",
|
|
73
|
+
"l2ab_bias",
|
|
74
|
+
"l34b_bias",
|
|
75
|
+
"fsw_version_c",
|
|
76
|
+
"num_evnt_last_hk",
|
|
77
|
+
"dac1_enable",
|
|
78
|
+
"preamp_l234b",
|
|
79
|
+
"analog_temp",
|
|
80
|
+
"fee_running",
|
|
81
|
+
"fsw_version_a",
|
|
82
|
+
"num_errors",
|
|
83
|
+
"test_pulser_on",
|
|
84
|
+
"dac0_enable",
|
|
85
|
+
"preamp_l1b",
|
|
86
|
+
"l1ab_bias",
|
|
87
|
+
"l34a_bias",
|
|
88
|
+
"leak_i",
|
|
89
|
+
"last_good_cmd",
|
|
90
|
+
"lvps_temp",
|
|
91
|
+
"idpu_temp",
|
|
92
|
+
"temp2",
|
|
93
|
+
"preamp_l234a",
|
|
94
|
+
"last_good_seq_num",
|
|
95
|
+
"num_good_cmds",
|
|
96
|
+
"heater_control",
|
|
97
|
+
"hvps_temp",
|
|
98
|
+
"ebox_p5d7va",
|
|
99
|
+
"spin_period_long",
|
|
100
|
+
"enable_hvps",
|
|
101
|
+
"temp0",
|
|
102
|
+
"spin_period_short",
|
|
103
|
+
"dyn_thresh_lvl",
|
|
104
|
+
"num_bad_cmds",
|
|
105
|
+
"adc_mode",
|
|
106
|
+
"ebox_5d1vd",
|
|
107
|
+
"active_heater",
|
|
108
|
+
"last_error_num",
|
|
109
|
+
"last_bad_cmd",
|
|
110
|
+
"ref_p5v",
|
|
111
|
+
"code_checksum",
|
|
112
|
+
"mode",
|
|
113
|
+
}
|
|
114
|
+
# Check that the dataset has the correct variables
|
|
115
|
+
assert valid_keys == set(hk_dataset.data_vars.keys())
|
|
116
|
+
assert set(dropped_keys).isdisjoint(set(hk_dataset.data_vars.keys()))
|
|
24
117
|
|
|
25
|
-
Creates a xarray dataset for housekeeping data
|
|
26
|
-
"""
|
|
27
118
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
119
|
+
def test_hit_l1b_hk_dataset_attributes(hk_dataset):
|
|
120
|
+
"""Test the attributes, dims, and coords in the housekeeping dataset"""
|
|
121
|
+
# TODO consider removing this test since it may be hard to upkeep if
|
|
122
|
+
# attributes change
|
|
123
|
+
# Define the housekeeping dataset attributes
|
|
124
|
+
dataset_attrs = {
|
|
125
|
+
"Acknowledgement": "Please acknowledge the IMAP Mission Principal "
|
|
126
|
+
"Investigator, Prof. David J. McComas of Princeton "
|
|
127
|
+
"University.\n",
|
|
128
|
+
"Data_level": "1B",
|
|
129
|
+
"Data_type": "L1B_HK>Level-1B Housekeeping",
|
|
130
|
+
"Data_version": "001",
|
|
131
|
+
"Descriptor": "HIT>IMAP High-energy Ion Telescope",
|
|
132
|
+
"Discipline": "Solar Physics>Heliospheric Physics",
|
|
133
|
+
"File_naming_convention": "source_descriptor_datatype_yyyyMMdd_vNNN",
|
|
134
|
+
"HTTP_LINK": "https://imap.princeton.edu/",
|
|
135
|
+
"Instrument_type": "Particles (space)",
|
|
136
|
+
"LINK_TITLE": "IMAP The Interstellar Mapping and Acceleration Probe",
|
|
137
|
+
"Logical_file_id": None,
|
|
138
|
+
"Logical_source": "imap_hit_l1b_hk",
|
|
139
|
+
"Logical_source_description": "IMAP Mission HIT Instrument Level-1B "
|
|
140
|
+
"Housekeeping Data.",
|
|
141
|
+
"Mission_group": "IMAP",
|
|
142
|
+
"PI_affiliation": "Princeton University",
|
|
143
|
+
"PI_name": "Prof. David J. McComas",
|
|
144
|
+
"Project": "STP>Solar Terrestrial Probes",
|
|
145
|
+
"Rules_of_use": "All IMAP data products are publicly released and citable for "
|
|
146
|
+
"use in publications. Please consult the IMAP team "
|
|
147
|
+
"publications and personnel for further details on "
|
|
148
|
+
"production, processing, and usage of these data.\n",
|
|
149
|
+
"Source_name": "IMAP>Interstellar Mapping and Acceleration Probe",
|
|
150
|
+
"TEXT": "The High-energy Ion Telescope (HIT) measures the elemental "
|
|
151
|
+
"composition, energy spectra, angle distributions, and arrival "
|
|
152
|
+
"times of high-energy ions. HIT delivers full-sky coverage from "
|
|
153
|
+
"a wide instrument field-of-view (FOV) to enable a high resolution "
|
|
154
|
+
"of ion measurements, such as observing shock-accelerated ions, "
|
|
155
|
+
"determining the origin of the solar energetic particles (SEPs) "
|
|
156
|
+
"spectra, and resolving particle transport in the heliosphere. "
|
|
157
|
+
"See https://imap.princeton.edu/instruments/hit for more details.\n",
|
|
158
|
+
}
|
|
33
159
|
|
|
34
|
-
|
|
35
|
-
|
|
160
|
+
# Define the coordinates and dimensions. Both have equivalent values
|
|
161
|
+
dataset_coords_dims = {"epoch", "adc_channels", "adc_channels_label"}
|
|
36
162
|
|
|
163
|
+
# Check that the dataset has the correct attributes, coordinates, and dimensions
|
|
164
|
+
assert hk_dataset.attrs == dataset_attrs
|
|
165
|
+
assert hk_dataset.coords.keys() == dataset_coords_dims
|
|
37
166
|
|
|
38
|
-
|
|
167
|
+
|
|
168
|
+
def test_hit_l1b(dependencies):
|
|
39
169
|
"""Test creating L1B CDF files
|
|
40
170
|
|
|
41
|
-
Creates a
|
|
42
|
-
their filepaths in a list
|
|
171
|
+
Creates a list of xarray datasets for each L1B product
|
|
43
172
|
|
|
44
173
|
Parameters
|
|
45
174
|
----------
|
|
46
|
-
|
|
47
|
-
L1A
|
|
175
|
+
dependencies : dict
|
|
176
|
+
Dictionary of L1A datasets and CCSDS packet file path
|
|
48
177
|
"""
|
|
49
|
-
|
|
178
|
+
# TODO: update assertions after science data processing is completed
|
|
179
|
+
datasets = hit_l1b.hit_l1b(dependencies, "001")
|
|
180
|
+
|
|
50
181
|
assert len(datasets) == 1
|
|
51
182
|
assert isinstance(datasets[0], xr.Dataset)
|
|
52
183
|
assert datasets[0].attrs["Logical_source"] == "imap_hit_l1b_hk"
|