imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imap-processing might be problematic. Click here for more details.
- imap_processing/__init__.py +1 -1
- imap_processing/_version.py +2 -2
- imap_processing/ccsds/excel_to_xtce.py +36 -2
- imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
- imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
- imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
- imap_processing/cdf/config/imap_hi_variable_attrs.yaml +136 -9
- imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +14 -0
- imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +63 -1
- imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
- imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +14 -7
- imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +577 -235
- imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +326 -0
- imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +33 -23
- imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
- imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
- imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +137 -79
- imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
- imap_processing/cdf/imap_cdf_manager.py +31 -27
- imap_processing/cdf/utils.py +3 -5
- imap_processing/cli.py +25 -14
- imap_processing/codice/codice_l1a.py +153 -63
- imap_processing/codice/constants.py +10 -10
- imap_processing/codice/decompress.py +10 -11
- imap_processing/codice/utils.py +1 -0
- imap_processing/glows/l1a/glows_l1a.py +1 -2
- imap_processing/glows/l1b/glows_l1b.py +3 -3
- imap_processing/glows/l1b/glows_l1b_data.py +59 -37
- imap_processing/glows/l2/glows_l2_data.py +123 -0
- imap_processing/hi/l1a/hi_l1a.py +4 -4
- imap_processing/hi/l1a/histogram.py +107 -109
- imap_processing/hi/l1a/science_direct_event.py +92 -225
- imap_processing/hi/l1b/hi_l1b.py +85 -11
- imap_processing/hi/l1c/hi_l1c.py +23 -1
- imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +3994 -0
- imap_processing/hi/utils.py +1 -1
- imap_processing/hit/hit_utils.py +221 -0
- imap_processing/hit/l0/constants.py +118 -0
- imap_processing/hit/l0/decom_hit.py +100 -156
- imap_processing/hit/l1a/hit_l1a.py +170 -184
- imap_processing/hit/l1b/hit_l1b.py +33 -153
- imap_processing/ialirt/l0/process_codicelo.py +153 -0
- imap_processing/ialirt/l0/process_hit.py +5 -5
- imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +281 -0
- imap_processing/ialirt/process_ephemeris.py +212 -0
- imap_processing/idex/idex_l1a.py +65 -84
- imap_processing/idex/idex_l1b.py +192 -0
- imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -0
- imap_processing/idex/packet_definitions/idex_packet_definition.xml +97 -595
- imap_processing/lo/l0/decompression_tables/decompression_tables.py +17 -1
- imap_processing/lo/l0/lo_science.py +45 -13
- imap_processing/lo/l1a/lo_l1a.py +76 -8
- imap_processing/lo/packet_definitions/lo_xtce.xml +8344 -1849
- imap_processing/mag/l0/decom_mag.py +4 -3
- imap_processing/mag/l1a/mag_l1a.py +12 -13
- imap_processing/mag/l1a/mag_l1a_data.py +1 -2
- imap_processing/mag/l1b/mag_l1b.py +90 -7
- imap_processing/spice/geometry.py +156 -16
- imap_processing/spice/time.py +144 -2
- imap_processing/swapi/l1/swapi_l1.py +4 -4
- imap_processing/swapi/l2/swapi_l2.py +1 -1
- imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +1535 -446
- imap_processing/swe/l1b/swe_l1b_science.py +8 -8
- imap_processing/swe/l2/swe_l2.py +134 -17
- imap_processing/tests/ccsds/test_data/expected_output.xml +2 -1
- imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
- imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
- imap_processing/tests/codice/conftest.py +1 -17
- imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
- imap_processing/tests/codice/test_codice_l0.py +8 -2
- imap_processing/tests/codice/test_codice_l1a.py +127 -107
- imap_processing/tests/codice/test_codice_l1b.py +1 -0
- imap_processing/tests/codice/test_decompress.py +7 -7
- imap_processing/tests/conftest.py +100 -58
- imap_processing/tests/glows/conftest.py +6 -0
- imap_processing/tests/glows/test_glows_l1b.py +9 -9
- imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
- imap_processing/tests/hi/test_data/l0/H90_NHK_20241104.bin +0 -0
- imap_processing/tests/hi/test_data/l0/H90_sci_cnt_20241104.bin +0 -0
- imap_processing/tests/hi/test_data/l0/H90_sci_de_20241104.bin +0 -0
- imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
- imap_processing/tests/hi/test_hi_l1b.py +73 -3
- imap_processing/tests/hi/test_hi_l1c.py +10 -2
- imap_processing/tests/hi/test_l1a.py +31 -58
- imap_processing/tests/hi/test_science_direct_event.py +58 -0
- imap_processing/tests/hi/test_utils.py +4 -3
- imap_processing/tests/hit/test_data/sci_sample1.ccsds +0 -0
- imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +95 -36
- imap_processing/tests/hit/test_hit_l1a.py +299 -179
- imap_processing/tests/hit/test_hit_l1b.py +231 -24
- imap_processing/tests/hit/test_hit_utils.py +218 -0
- imap_processing/tests/hit/validation_data/hskp_sample_eu.csv +89 -0
- imap_processing/tests/hit/validation_data/sci_sample_raw1.csv +29 -0
- imap_processing/tests/ialirt/test_data/l0/apid01152.tlm +0 -0
- imap_processing/tests/ialirt/test_data/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
- imap_processing/tests/ialirt/unit/test_process_codicelo.py +106 -0
- imap_processing/tests/ialirt/unit/test_process_ephemeris.py +109 -0
- imap_processing/tests/ialirt/unit/test_process_hit.py +9 -6
- imap_processing/tests/idex/conftest.py +2 -2
- imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
- imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
- imap_processing/tests/idex/test_idex_l0.py +4 -4
- imap_processing/tests/idex/test_idex_l1a.py +8 -2
- imap_processing/tests/idex/test_idex_l1b.py +126 -0
- imap_processing/tests/lo/test_lo_l1a.py +7 -16
- imap_processing/tests/lo/test_lo_science.py +69 -5
- imap_processing/tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts +0 -0
- imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv +1999 -0
- imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
- imap_processing/tests/mag/test_mag_l1b.py +97 -7
- imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
- imap_processing/tests/spice/test_geometry.py +115 -9
- imap_processing/tests/spice/test_time.py +135 -6
- imap_processing/tests/swapi/test_swapi_decom.py +75 -69
- imap_processing/tests/swapi/test_swapi_l1.py +4 -4
- imap_processing/tests/swe/conftest.py +33 -0
- imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
- imap_processing/tests/swe/test_swe_l1b.py +29 -8
- imap_processing/tests/swe/test_swe_l2.py +64 -8
- imap_processing/tests/test_utils.py +2 -2
- imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -3314
- imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
- imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
- imap_processing/tests/ultra/unit/test_de.py +113 -0
- imap_processing/tests/ultra/unit/test_spatial_utils.py +125 -0
- imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
- imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
- imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +55 -35
- imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +10 -68
- imap_processing/ultra/constants.py +12 -3
- imap_processing/ultra/l1b/de.py +168 -30
- imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
- imap_processing/ultra/l1b/ultra_l1b_extended.py +46 -80
- imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +60 -144
- imap_processing/ultra/utils/spatial_utils.py +221 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/METADATA +15 -14
- {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/RECORD +142 -139
- imap_processing/cdf/cdf_attribute_manager.py +0 -322
- imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
- imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
- imap_processing/hi/l0/decom_hi.py +0 -24
- imap_processing/hi/packet_definitions/hi_packet_definition.xml +0 -482
- imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
- imap_processing/hit/l0/data_classes/science_packet.py +0 -259
- imap_processing/hit/l0/utils/hit_base.py +0 -57
- imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
- imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
- imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
- imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
- imap_processing/tests/hi/test_decom.py +0 -55
- imap_processing/tests/hi/test_l1a_sci_de.py +0 -72
- imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
- imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
- /imap_processing/{hi/l0/__init__.py → tests/glows/test_glows_l2_data.py} +0 -0
- /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/LICENSE +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/WHEEL +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
"""Tests the L1a processing for decommutated CoDICE data"""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from pathlib import Path
|
|
5
4
|
|
|
6
5
|
import numpy as np
|
|
7
6
|
import pytest
|
|
@@ -10,139 +9,172 @@ import xarray as xr
|
|
|
10
9
|
from imap_processing.cdf.utils import load_cdf, write_cdf
|
|
11
10
|
from imap_processing.codice.codice_l1a import process_codice_l1a
|
|
12
11
|
|
|
13
|
-
from .conftest import
|
|
12
|
+
from .conftest import TEST_L0_FILE, VALIDATION_DATA
|
|
14
13
|
|
|
15
14
|
logger = logging.getLogger(__name__)
|
|
16
15
|
logger.setLevel(logging.INFO)
|
|
17
16
|
|
|
18
|
-
# TODO: Add test that processes a file with multiple APIDs
|
|
19
|
-
|
|
20
17
|
EXPECTED_ARRAY_SHAPES = [
|
|
21
|
-
(
|
|
22
|
-
(
|
|
23
|
-
(
|
|
24
|
-
(
|
|
25
|
-
(
|
|
26
|
-
(1,
|
|
27
|
-
(
|
|
28
|
-
(
|
|
29
|
-
(
|
|
30
|
-
(
|
|
31
|
-
(
|
|
32
|
-
(
|
|
33
|
-
(
|
|
34
|
-
(
|
|
35
|
-
(
|
|
18
|
+
(), # hi-ialirt # TODO: Need to implement
|
|
19
|
+
(), # lo-ialirt # TODO: Need to implement
|
|
20
|
+
(31778,), # hskp
|
|
21
|
+
(77, 128, 6, 6), # lo-counters-aggregated
|
|
22
|
+
(77, 128, 24, 6), # lo-counters-singles
|
|
23
|
+
(77, 128, 1, 12), # lo-sw-priority
|
|
24
|
+
(77, 128, 1, 12), # lo-nsw-priority
|
|
25
|
+
(77, 128, 1, 1), # lo-sw-species
|
|
26
|
+
(77, 128, 1, 1), # lo-nsw-species
|
|
27
|
+
(77, 128, 5, 12), # lo-sw-angular
|
|
28
|
+
(77, 128, 19, 12), # lo-nsw-angular
|
|
29
|
+
(77, 1, 6, 1), # hi-counters-aggregated
|
|
30
|
+
(77, 1, 12, 1), # hi-counters-singles
|
|
31
|
+
(77, 15, 4, 1), # hi-omni
|
|
32
|
+
(77, 8, 12, 12), # hi-sectored
|
|
33
|
+
(), # hi-priority # TODO: Need to implement
|
|
34
|
+
(), # lo-pha # TODO: Need to implement
|
|
35
|
+
(), # hi-pha # TODO: Need to implement
|
|
36
36
|
]
|
|
37
|
-
|
|
37
|
+
|
|
38
|
+
EXPECTED_LOGICAL_SOURCES = [
|
|
39
|
+
"imap_codice_l1a_hi-ialirt",
|
|
40
|
+
"imap_codice_l1a_lo-ialirt",
|
|
38
41
|
"imap_codice_l1a_hskp",
|
|
39
|
-
"imap_codice_l1a_hi-counters-aggregated",
|
|
40
|
-
"imap_codice_l1a_hi-counters-singles",
|
|
41
|
-
"imap_codice_l1a_hi-omni",
|
|
42
|
-
"imap_codice_l1a_hi-sectored",
|
|
43
|
-
"imap_codice_l1a_hi-pha",
|
|
44
42
|
"imap_codice_l1a_lo-counters-aggregated",
|
|
45
43
|
"imap_codice_l1a_lo-counters-singles",
|
|
46
|
-
"imap_codice_l1a_lo-sw-angular",
|
|
47
|
-
"imap_codice_l1a_lo-nsw-angular",
|
|
48
44
|
"imap_codice_l1a_lo-sw-priority",
|
|
49
45
|
"imap_codice_l1a_lo-nsw-priority",
|
|
50
46
|
"imap_codice_l1a_lo-sw-species",
|
|
51
47
|
"imap_codice_l1a_lo-nsw-species",
|
|
48
|
+
"imap_codice_l1a_lo-sw-angular",
|
|
49
|
+
"imap_codice_l1a_lo-nsw-angular",
|
|
50
|
+
"imap_codice_l1a_hi-counters-aggregated",
|
|
51
|
+
"imap_codice_l1a_hi-counters-singles",
|
|
52
|
+
"imap_codice_l1a_hi-omni",
|
|
53
|
+
"imap_codice_l1a_hi-sectored",
|
|
54
|
+
"imap_codice_l1a_hi-priority",
|
|
52
55
|
"imap_codice_l1a_lo-pha",
|
|
56
|
+
"imap_codice_l1a_hi-pha",
|
|
53
57
|
]
|
|
58
|
+
|
|
54
59
|
EXPECTED_NUM_VARIABLES = [
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
8, # hi-omni
|
|
59
|
-
4, # hi-sectored
|
|
60
|
-
0, # hi-pha
|
|
60
|
+
0, # hi-ialirt # TODO: Need to implement
|
|
61
|
+
0, # lo-ialirt # TODO: Need to implement
|
|
62
|
+
148, # hskp
|
|
61
63
|
3, # lo-counters-aggregated
|
|
62
64
|
3, # lo-counters-singles
|
|
63
|
-
6, # lo-sw-angular
|
|
64
|
-
3, # lo-nsw-angular
|
|
65
65
|
7, # lo-sw-priority
|
|
66
66
|
4, # lo-nsw-priority
|
|
67
67
|
18, # lo-sw-species
|
|
68
68
|
10, # lo-nsw-species
|
|
69
|
-
|
|
69
|
+
6, # lo-sw-angular
|
|
70
|
+
3, # lo-nsw-angular
|
|
71
|
+
1, # hi-counters-aggregated
|
|
72
|
+
3, # hi-counters-singles
|
|
73
|
+
8, # hi-omni
|
|
74
|
+
4, # hi-sectored
|
|
75
|
+
0, # hi-priority # TODO: Need to implement
|
|
76
|
+
0, # lo-pha # TODO: Need to implement
|
|
77
|
+
0, # hi-pha # TODO: Need to implement
|
|
70
78
|
]
|
|
71
79
|
|
|
72
80
|
|
|
73
|
-
@pytest.fixture(
|
|
74
|
-
def test_l1a_data(
|
|
81
|
+
@pytest.fixture(scope="session")
|
|
82
|
+
def test_l1a_data() -> xr.Dataset:
|
|
75
83
|
"""Return a ``xarray`` dataset containing test data.
|
|
76
84
|
|
|
77
85
|
Returns
|
|
78
86
|
-------
|
|
79
|
-
|
|
80
|
-
A ``xarray``
|
|
87
|
+
processed_datasets : list[xarray.Dataset]
|
|
88
|
+
A list of ``xarray`` datasets containing the test data
|
|
81
89
|
"""
|
|
82
90
|
|
|
83
|
-
|
|
91
|
+
processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE, data_version="001")
|
|
92
|
+
|
|
93
|
+
return processed_datasets
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@pytest.mark.parametrize("index", range(len(EXPECTED_ARRAY_SHAPES)))
|
|
97
|
+
def test_l1a_data_array_shape(test_l1a_data, index):
|
|
98
|
+
"""Tests that the data arrays in the generated CDFs have the expected shape.
|
|
99
|
+
|
|
100
|
+
Parameters
|
|
101
|
+
----------
|
|
102
|
+
test_l1a_data : list[xarray.Dataset]
|
|
103
|
+
A list of ``xarray`` datasets containing the test data
|
|
104
|
+
index : int
|
|
105
|
+
The index of the list to test
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
processed_dataset = test_l1a_data[index]
|
|
109
|
+
expected_shape = EXPECTED_ARRAY_SHAPES[index]
|
|
110
|
+
|
|
111
|
+
# Mark currently broken/unsupported datasets as expected to fail
|
|
112
|
+
# TODO: Remove these once they are supported
|
|
113
|
+
if index in [0, 1, 15, 16, 17]:
|
|
114
|
+
pytest.xfail("Data product is currently unsupported")
|
|
84
115
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
116
|
+
for variable in processed_dataset:
|
|
117
|
+
if variable in ["energy_table", "acquisition_time_per_step"]:
|
|
118
|
+
assert processed_dataset[variable].data.shape == (128,)
|
|
119
|
+
else:
|
|
120
|
+
assert processed_dataset[variable].data.shape == expected_shape
|
|
88
121
|
|
|
89
|
-
return dataset
|
|
90
122
|
|
|
123
|
+
@pytest.mark.parametrize("index", range(len(EXPECTED_LOGICAL_SOURCES)))
|
|
124
|
+
def test_l1a_logical_sources(test_l1a_data, index):
|
|
125
|
+
"""Tests that the Logical source of the dataset is what is expected.
|
|
91
126
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
"test_l1a_data, expected_logical_source",
|
|
95
|
-
list(zip(TEST_PACKETS, EXPECTED_LOGICAL_SOURCE)),
|
|
96
|
-
indirect=["test_l1a_data"],
|
|
97
|
-
)
|
|
98
|
-
def test_l1a_cdf_filenames(test_l1a_data: xr.Dataset, expected_logical_source: str):
|
|
99
|
-
"""Tests that the ``process_codice_l1a`` function generates datasets
|
|
100
|
-
with the expected logical source.
|
|
127
|
+
Since the logical source gets set by ``write_cdf``, this also tests that
|
|
128
|
+
the dataset can be written to a file.
|
|
101
129
|
|
|
102
130
|
Parameters
|
|
103
131
|
----------
|
|
104
|
-
test_l1a_data : xarray.Dataset
|
|
105
|
-
A ``xarray``
|
|
106
|
-
|
|
107
|
-
The
|
|
132
|
+
test_l1a_data : list[xarray.Dataset]
|
|
133
|
+
A list of ``xarray`` datasets containing the test data
|
|
134
|
+
index : int
|
|
135
|
+
The index of the list to test
|
|
108
136
|
"""
|
|
109
137
|
|
|
110
|
-
|
|
111
|
-
|
|
138
|
+
processed_dataset = test_l1a_data[index]
|
|
139
|
+
expected_logical_source = EXPECTED_LOGICAL_SOURCES[index]
|
|
112
140
|
|
|
141
|
+
# Mark currently broken/unsupported datasets as expected to fail
|
|
142
|
+
# TODO: Remove these once they are supported
|
|
143
|
+
if index in [0, 1, 2, 15, 16, 17]:
|
|
144
|
+
pytest.xfail("Data product is currently unsupported")
|
|
113
145
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
146
|
+
# Write the dataset to a file to set the logical source attribute
|
|
147
|
+
_ = write_cdf(processed_dataset)
|
|
148
|
+
|
|
149
|
+
assert processed_dataset.attrs["Logical_source"] == expected_logical_source
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@pytest.mark.parametrize("index", range(len(EXPECTED_NUM_VARIABLES)))
|
|
153
|
+
def test_l1a_num_variables(test_l1a_data, index):
|
|
154
|
+
"""Tests that the data arrays in the generated CDFs have the expected number
|
|
155
|
+
of variables.
|
|
122
156
|
|
|
123
157
|
Parameters
|
|
124
158
|
----------
|
|
125
|
-
test_l1a_data : xarray.Dataset
|
|
126
|
-
A ``xarray``
|
|
127
|
-
|
|
128
|
-
The
|
|
159
|
+
test_l1a_data : list[xarray.Dataset]
|
|
160
|
+
A list of ``xarray`` datasets containing the test data
|
|
161
|
+
index : int
|
|
162
|
+
The index of the list to test
|
|
129
163
|
"""
|
|
130
164
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
165
|
+
processed_dataset = test_l1a_data[index]
|
|
166
|
+
|
|
167
|
+
# Mark currently broken/unsupported datasets as expected to fail
|
|
168
|
+
# TODO: Remove these once they are supported
|
|
169
|
+
if index in [0, 1, 15, 16, 17]:
|
|
170
|
+
pytest.xfail("Data product is currently unsupported")
|
|
171
|
+
|
|
172
|
+
assert len(processed_dataset) == EXPECTED_NUM_VARIABLES[index]
|
|
137
173
|
|
|
138
174
|
|
|
139
175
|
@pytest.mark.skip("Awaiting validation data")
|
|
140
|
-
@pytest.mark.parametrize(
|
|
141
|
-
|
|
142
|
-
list(zip(TEST_PACKETS, VALIDATION_DATA)),
|
|
143
|
-
indirect=["test_l1a_data"],
|
|
144
|
-
)
|
|
145
|
-
def test_l1a_data_array_values(test_l1a_data: xr.Dataset, validation_data: Path):
|
|
176
|
+
@pytest.mark.parametrize("index", range(len(VALIDATION_DATA)))
|
|
177
|
+
def test_l1a_data_array_values(test_l1a_data: xr.Dataset, index):
|
|
146
178
|
"""Tests that the generated L1a CDF contents are valid.
|
|
147
179
|
|
|
148
180
|
Once proper validation files are acquired, this test function should point
|
|
@@ -151,40 +183,28 @@ def test_l1a_data_array_values(test_l1a_data: xr.Dataset, validation_data: Path)
|
|
|
151
183
|
|
|
152
184
|
Parameters
|
|
153
185
|
----------
|
|
154
|
-
test_l1a_data : xarray.Dataset
|
|
155
|
-
A ``xarray``
|
|
156
|
-
|
|
157
|
-
The
|
|
186
|
+
test_l1a_data : list[xarray.Dataset]
|
|
187
|
+
A list of ``xarray`` datasets containing the test data
|
|
188
|
+
index : int
|
|
189
|
+
The index of the list to test
|
|
158
190
|
"""
|
|
159
191
|
|
|
160
192
|
generated_dataset = test_l1a_data
|
|
161
|
-
validation_dataset = load_cdf(
|
|
193
|
+
validation_dataset = load_cdf(VALIDATION_DATA[index])
|
|
162
194
|
|
|
163
195
|
# Ensure the processed data matches the validation data
|
|
164
196
|
for variable in validation_dataset:
|
|
165
197
|
assert variable in generated_dataset
|
|
166
198
|
if variable != "epoch":
|
|
167
199
|
np.testing.assert_array_equal(
|
|
168
|
-
|
|
200
|
+
validation_dataset[variable].data, generated_dataset[variable].data[0]
|
|
169
201
|
)
|
|
170
202
|
|
|
171
203
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
"test_l1a_data, expected_num_variables",
|
|
175
|
-
list(zip(TEST_PACKETS, EXPECTED_NUM_VARIABLES)),
|
|
176
|
-
indirect=["test_l1a_data"],
|
|
177
|
-
)
|
|
178
|
-
def test_l1a_num_variables(test_l1a_data: xr.Dataset, expected_num_variables: int):
|
|
179
|
-
"""Tests that the data arrays in the generated CDFs have the expected size.
|
|
204
|
+
def test_l1a_multiple_packets():
|
|
205
|
+
"""Tests that an input L0 file containing multiple APIDs can be processed."""
|
|
180
206
|
|
|
181
|
-
|
|
182
|
-
----------
|
|
183
|
-
test_l1a_data : xarray.Dataset
|
|
184
|
-
A ``xarray`` dataset containing the test data
|
|
185
|
-
expected_num_variables : int
|
|
186
|
-
The expected number of data variables in the CDF
|
|
187
|
-
"""
|
|
207
|
+
processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE, data_version="001")
|
|
188
208
|
|
|
189
|
-
|
|
190
|
-
assert len(
|
|
209
|
+
# TODO: Could add some more checks here?
|
|
210
|
+
assert len(processed_datasets) == 18
|
|
@@ -10,14 +10,14 @@ from imap_processing.codice.utils import CoDICECompression
|
|
|
10
10
|
|
|
11
11
|
# Test the algorithms using input value of 234 (picked randomly)
|
|
12
12
|
lzma_bytes = lzma.compress((234).to_bytes(1, byteorder="big"))
|
|
13
|
-
LZMA_EXAMPLE = "".join(format(byte, "08b") for byte in lzma_bytes)
|
|
13
|
+
# LZMA_EXAMPLE = "".join(format(byte, "08b") for byte in lzma_bytes)
|
|
14
14
|
TEST_DATA = [
|
|
15
|
-
("
|
|
16
|
-
("
|
|
17
|
-
("
|
|
18
|
-
(
|
|
19
|
-
(
|
|
20
|
-
(
|
|
15
|
+
(b"\xea", CoDICECompression.NO_COMPRESSION, [234]),
|
|
16
|
+
(b"\xea", CoDICECompression.LOSSY_A, [212992]),
|
|
17
|
+
(b"\xea", CoDICECompression.LOSSY_B, [1310720]),
|
|
18
|
+
(lzma_bytes, CoDICECompression.LOSSLESS, [234]),
|
|
19
|
+
(lzma_bytes, CoDICECompression.LOSSY_A_LOSSLESS, [212992]),
|
|
20
|
+
(lzma_bytes, CoDICECompression.LOSSY_B_LOSSLESS, [1310720]),
|
|
21
21
|
]
|
|
22
22
|
|
|
23
23
|
|
|
@@ -46,45 +46,48 @@ def _autoclear_spice():
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
@pytest.fixture(scope="session")
|
|
49
|
-
def
|
|
50
|
-
"""This fixture downloads the de440s.bsp
|
|
51
|
-
tests/spice/test_data directory if
|
|
49
|
+
def _download_external_kernels(spice_test_data_path):
|
|
50
|
+
"""This fixture downloads the de440s.bsp and pck00011.tpc kernels into the
|
|
51
|
+
tests/spice/test_data directory if they do not already exist there. The
|
|
52
52
|
fixture is not intended to be used directly. It is automatically added to
|
|
53
53
|
tests marked with "external_kernel" in the hook below."""
|
|
54
54
|
logger = logging.getLogger(__name__)
|
|
55
|
-
|
|
56
|
-
"https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp"
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
55
|
+
kernel_urls = [
|
|
56
|
+
"https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp",
|
|
57
|
+
"https://naif.jpl.nasa.gov/pub/naif/generic_kernels/pck/pck00011.tpc",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
for kernel_url in kernel_urls:
|
|
61
|
+
kernel_name = kernel_url.split("/")[-1]
|
|
62
|
+
local_filepath = spice_test_data_path / kernel_name
|
|
63
|
+
|
|
64
|
+
if local_filepath.exists():
|
|
65
|
+
continue
|
|
66
|
+
allowed_attempts = 3
|
|
67
|
+
for attempt_number in range(allowed_attempts):
|
|
68
|
+
try:
|
|
69
|
+
with requests.get(kernel_url, stream=True, timeout=30) as r:
|
|
70
|
+
r.raise_for_status()
|
|
71
|
+
with open(local_filepath, "wb") as f:
|
|
72
|
+
for chunk in r.iter_content(chunk_size=8192):
|
|
73
|
+
f.write(chunk)
|
|
74
|
+
logger.info("Cached kernel file to %s", local_filepath)
|
|
75
|
+
continue
|
|
76
|
+
except requests.exceptions.RequestException as error:
|
|
77
|
+
logger.info(f"Request failed. {error}")
|
|
78
|
+
if attempt_number < allowed_attempts:
|
|
79
|
+
logger.info(
|
|
80
|
+
f"Trying again, retries left "
|
|
81
|
+
f"{allowed_attempts - attempt_number}, "
|
|
82
|
+
f"Exception: {error}"
|
|
83
|
+
)
|
|
84
|
+
time.sleep(1)
|
|
85
|
+
else:
|
|
86
|
+
logger.error(
|
|
87
|
+
f"Failed to download file {kernel_name} after "
|
|
88
|
+
f"{allowed_attempts} attempts, Final Error: {error}"
|
|
89
|
+
)
|
|
90
|
+
raise
|
|
88
91
|
|
|
89
92
|
|
|
90
93
|
def pytest_collection_modifyitems(items):
|
|
@@ -93,12 +96,12 @@ def pytest_collection_modifyitems(items):
|
|
|
93
96
|
been collected. In this case, it automatically adds fixtures based on the
|
|
94
97
|
following table:
|
|
95
98
|
|
|
96
|
-
|
|
97
|
-
| pytest mark | fixture added
|
|
98
|
-
|
|
99
|
-
| external_kernel |
|
|
100
|
-
| use_test_metakernel | use_test_metakernel
|
|
101
|
-
|
|
99
|
+
+---------------------+----------------------------+
|
|
100
|
+
| pytest mark | fixture added |
|
|
101
|
+
+=====================+============================+
|
|
102
|
+
| external_kernel | _download_external_kernels |
|
|
103
|
+
| use_test_metakernel | use_test_metakernel |
|
|
104
|
+
+---------------------+----------------------------+
|
|
102
105
|
|
|
103
106
|
Notes
|
|
104
107
|
-----
|
|
@@ -108,7 +111,7 @@ def pytest_collection_modifyitems(items):
|
|
|
108
111
|
"""
|
|
109
112
|
for item in items:
|
|
110
113
|
if item.get_closest_marker("external_kernel") is not None:
|
|
111
|
-
item.fixturenames.append("
|
|
114
|
+
item.fixturenames.append("_download_external_kernels")
|
|
112
115
|
if item.get_closest_marker("use_test_metakernel") is not None:
|
|
113
116
|
item.fixturenames.append("use_test_metakernel")
|
|
114
117
|
|
|
@@ -297,19 +300,55 @@ def _unset_metakernel_path(monkeypatch):
|
|
|
297
300
|
|
|
298
301
|
|
|
299
302
|
@pytest.fixture()
|
|
300
|
-
def
|
|
301
|
-
"""
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
303
|
+
def use_test_spin_data_csv(monkeypatch):
|
|
304
|
+
"""Sets the SPIN_DATA_FILEPATH environment variable to input path."""
|
|
305
|
+
|
|
306
|
+
def wrapped_set_spin_data_filepath(path: Path):
|
|
307
|
+
monkeypatch.setenv("SPIN_DATA_FILEPATH", str(path))
|
|
308
|
+
|
|
309
|
+
return wrapped_set_spin_data_filepath
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
@pytest.fixture()
|
|
313
|
+
def use_fake_spin_data_for_time(
|
|
314
|
+
request, use_test_spin_data_csv, tmpdir, generate_spin_data
|
|
315
|
+
):
|
|
316
|
+
"""
|
|
317
|
+
Generate and use fake spin data for testing.
|
|
318
|
+
|
|
319
|
+
Returns
|
|
320
|
+
-------
|
|
321
|
+
callable
|
|
322
|
+
Returns a callable function that takes start_met and optionally end_met
|
|
323
|
+
as inputs, generates fake spin data, writes the data to a csv file,
|
|
324
|
+
and sets the SPIN_DATA_FILEPATH environment variable to point to the
|
|
325
|
+
fake spin data file.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
def wrapped_set_spin_data_filepath(
|
|
329
|
+
start_met: float, end_met: Optional[int] = None
|
|
330
|
+
) -> pd.DataFrame:
|
|
331
|
+
"""
|
|
332
|
+
Generate and use fake spin data for testing.
|
|
333
|
+
Parameters
|
|
334
|
+
----------
|
|
335
|
+
start_met : int
|
|
336
|
+
Provides the start time in Mission Elapsed Time (MET).
|
|
337
|
+
end_met : int
|
|
338
|
+
Provides the end time in MET. If not provided, default to one day
|
|
339
|
+
from start time.
|
|
340
|
+
"""
|
|
341
|
+
spin_df = generate_spin_data(start_met, end_met=end_met)
|
|
342
|
+
spin_csv_file_path = tmpdir / "spin_data.spin.csv"
|
|
343
|
+
spin_df.to_csv(spin_csv_file_path, index=False)
|
|
344
|
+
use_test_spin_data_csv(spin_csv_file_path)
|
|
345
|
+
|
|
346
|
+
return wrapped_set_spin_data_filepath
|
|
308
347
|
|
|
309
348
|
|
|
310
349
|
@pytest.fixture()
|
|
311
350
|
def generate_spin_data():
|
|
312
|
-
def make_data(start_met:
|
|
351
|
+
def make_data(start_met: float, end_met: Optional[float] = None) -> pd.DataFrame:
|
|
313
352
|
"""
|
|
314
353
|
Generate a spin table CSV covering one or more days.
|
|
315
354
|
Spin table contains the following fields:
|
|
@@ -324,14 +363,14 @@ def generate_spin_data():
|
|
|
324
363
|
thruster_firing
|
|
325
364
|
)
|
|
326
365
|
This function creates spin data using start MET and end MET time.
|
|
327
|
-
Each spin start data uses the nominal 15
|
|
366
|
+
Each spin start data uses the nominal 15-second spin period. The spins that
|
|
328
367
|
occur from 00:00(Mid-night) to 00:10 UTC are marked with flags for
|
|
329
368
|
thruster firing, invalid spin period, and invalid spin phase.
|
|
330
369
|
Parameters
|
|
331
370
|
----------
|
|
332
|
-
start_met :
|
|
371
|
+
start_met : float
|
|
333
372
|
Provides the start time in Mission Elapsed Time (MET).
|
|
334
|
-
end_met :
|
|
373
|
+
end_met : float
|
|
335
374
|
Provides the end time in MET. If not provided, default to one day
|
|
336
375
|
from start time.
|
|
337
376
|
Returns
|
|
@@ -344,7 +383,8 @@ def generate_spin_data():
|
|
|
344
383
|
end_met = start_met + 86400
|
|
345
384
|
|
|
346
385
|
# Create spin start second data of 15 seconds increment
|
|
347
|
-
spin_start_sec = np.arange(start_met, end_met + 1, 15)
|
|
386
|
+
spin_start_sec = np.arange(np.floor(start_met), end_met + 1, 15)
|
|
387
|
+
spin_start_subsec = int((start_met - spin_start_sec[0]) * 1000)
|
|
348
388
|
|
|
349
389
|
nspins = len(spin_start_sec)
|
|
350
390
|
|
|
@@ -352,7 +392,9 @@ def generate_spin_data():
|
|
|
352
392
|
{
|
|
353
393
|
"spin_number": np.arange(nspins, dtype=np.uint32),
|
|
354
394
|
"spin_start_sec": spin_start_sec,
|
|
355
|
-
"spin_start_subsec": np.
|
|
395
|
+
"spin_start_subsec": np.full(
|
|
396
|
+
nspins, spin_start_subsec, dtype=np.uint32
|
|
397
|
+
),
|
|
356
398
|
"spin_period_sec": np.full(nspins, 15.0, dtype=np.float32),
|
|
357
399
|
"spin_period_valid": np.ones(nspins, dtype=np.uint8),
|
|
358
400
|
"spin_phase_valid": np.ones(nspins, dtype=np.uint8),
|
|
@@ -362,7 +404,7 @@ def generate_spin_data():
|
|
|
362
404
|
)
|
|
363
405
|
|
|
364
406
|
# Convert spin_start_sec to datetime to set repointing times flags
|
|
365
|
-
spin_start_dates = met_to_j2000ns(spin_start_sec)
|
|
407
|
+
spin_start_dates = met_to_j2000ns(spin_start_sec + spin_start_subsec / 1000)
|
|
366
408
|
spin_start_dates = cdflib.cdfepoch.to_datetime(spin_start_dates)
|
|
367
409
|
|
|
368
410
|
# Convert DatetimeIndex to Series for using .dt accessor
|
|
@@ -6,6 +6,7 @@ import pytest
|
|
|
6
6
|
from imap_processing.glows.l0 import decom_glows
|
|
7
7
|
from imap_processing.glows.l1a.glows_l1a import glows_l1a, process_de_l0
|
|
8
8
|
from imap_processing.glows.l1a.glows_l1a_data import HistogramL1A
|
|
9
|
+
from imap_processing.glows.l1b.glows_l1b import glows_l1b
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
@pytest.fixture()
|
|
@@ -40,3 +41,8 @@ def l1a_test_data(decom_test_data):
|
|
|
40
41
|
@pytest.fixture()
|
|
41
42
|
def l1a_dataset(packet_path):
|
|
42
43
|
return glows_l1a(packet_path, "v001")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@pytest.fixture()
|
|
47
|
+
def l1b_hist_dataset(l1a_dataset):
|
|
48
|
+
return glows_l1b(l1a_dataset[0], "v001")
|
|
@@ -318,23 +318,23 @@ def test_glows_l1b(de_dataset, hist_dataset):
|
|
|
318
318
|
"imap_spin_angle_bin_cntr",
|
|
319
319
|
"histogram_flag_array",
|
|
320
320
|
"filter_temperature_average",
|
|
321
|
-
"
|
|
321
|
+
"filter_temperature_std_dev",
|
|
322
322
|
"hv_voltage_average",
|
|
323
|
-
"
|
|
323
|
+
"hv_voltage_std_dev",
|
|
324
324
|
"spin_period_average",
|
|
325
|
-
"
|
|
325
|
+
"spin_period_std_dev",
|
|
326
326
|
"pulse_length_average",
|
|
327
|
-
"
|
|
327
|
+
"pulse_length_std_dev",
|
|
328
328
|
"spin_period_ground_average",
|
|
329
|
-
"
|
|
329
|
+
"spin_period_ground_std_dev",
|
|
330
330
|
"position_angle_offset_average",
|
|
331
|
-
"
|
|
332
|
-
"
|
|
331
|
+
"position_angle_offset_std_dev",
|
|
332
|
+
"spin_axis_orientation_std_dev",
|
|
333
333
|
"spin_axis_orientation_average",
|
|
334
334
|
"spacecraft_location_average",
|
|
335
|
-
"
|
|
335
|
+
"spacecraft_location_std_dev",
|
|
336
336
|
"spacecraft_velocity_average",
|
|
337
|
-
"
|
|
337
|
+
"spacecraft_velocity_std_dev",
|
|
338
338
|
"flags",
|
|
339
339
|
]
|
|
340
340
|
|