imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (172) hide show
  1. imap_processing/__init__.py +1 -1
  2. imap_processing/_version.py +2 -2
  3. imap_processing/ccsds/excel_to_xtce.py +36 -2
  4. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
  5. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
  6. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
  7. imap_processing/cdf/config/imap_hi_variable_attrs.yaml +136 -9
  8. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +14 -0
  9. imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +63 -1
  10. imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
  11. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +14 -7
  12. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +577 -235
  13. imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +326 -0
  14. imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +33 -23
  15. imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
  16. imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
  17. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +137 -79
  18. imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
  19. imap_processing/cdf/imap_cdf_manager.py +31 -27
  20. imap_processing/cdf/utils.py +3 -5
  21. imap_processing/cli.py +25 -14
  22. imap_processing/codice/codice_l1a.py +153 -63
  23. imap_processing/codice/constants.py +10 -10
  24. imap_processing/codice/decompress.py +10 -11
  25. imap_processing/codice/utils.py +1 -0
  26. imap_processing/glows/l1a/glows_l1a.py +1 -2
  27. imap_processing/glows/l1b/glows_l1b.py +3 -3
  28. imap_processing/glows/l1b/glows_l1b_data.py +59 -37
  29. imap_processing/glows/l2/glows_l2_data.py +123 -0
  30. imap_processing/hi/l1a/hi_l1a.py +4 -4
  31. imap_processing/hi/l1a/histogram.py +107 -109
  32. imap_processing/hi/l1a/science_direct_event.py +92 -225
  33. imap_processing/hi/l1b/hi_l1b.py +85 -11
  34. imap_processing/hi/l1c/hi_l1c.py +23 -1
  35. imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +3994 -0
  36. imap_processing/hi/utils.py +1 -1
  37. imap_processing/hit/hit_utils.py +221 -0
  38. imap_processing/hit/l0/constants.py +118 -0
  39. imap_processing/hit/l0/decom_hit.py +100 -156
  40. imap_processing/hit/l1a/hit_l1a.py +170 -184
  41. imap_processing/hit/l1b/hit_l1b.py +33 -153
  42. imap_processing/ialirt/l0/process_codicelo.py +153 -0
  43. imap_processing/ialirt/l0/process_hit.py +5 -5
  44. imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +281 -0
  45. imap_processing/ialirt/process_ephemeris.py +212 -0
  46. imap_processing/idex/idex_l1a.py +65 -84
  47. imap_processing/idex/idex_l1b.py +192 -0
  48. imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -0
  49. imap_processing/idex/packet_definitions/idex_packet_definition.xml +97 -595
  50. imap_processing/lo/l0/decompression_tables/decompression_tables.py +17 -1
  51. imap_processing/lo/l0/lo_science.py +45 -13
  52. imap_processing/lo/l1a/lo_l1a.py +76 -8
  53. imap_processing/lo/packet_definitions/lo_xtce.xml +8344 -1849
  54. imap_processing/mag/l0/decom_mag.py +4 -3
  55. imap_processing/mag/l1a/mag_l1a.py +12 -13
  56. imap_processing/mag/l1a/mag_l1a_data.py +1 -2
  57. imap_processing/mag/l1b/mag_l1b.py +90 -7
  58. imap_processing/spice/geometry.py +156 -16
  59. imap_processing/spice/time.py +144 -2
  60. imap_processing/swapi/l1/swapi_l1.py +4 -4
  61. imap_processing/swapi/l2/swapi_l2.py +1 -1
  62. imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +1535 -446
  63. imap_processing/swe/l1b/swe_l1b_science.py +8 -8
  64. imap_processing/swe/l2/swe_l2.py +134 -17
  65. imap_processing/tests/ccsds/test_data/expected_output.xml +2 -1
  66. imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
  67. imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
  68. imap_processing/tests/codice/conftest.py +1 -17
  69. imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
  70. imap_processing/tests/codice/test_codice_l0.py +8 -2
  71. imap_processing/tests/codice/test_codice_l1a.py +127 -107
  72. imap_processing/tests/codice/test_codice_l1b.py +1 -0
  73. imap_processing/tests/codice/test_decompress.py +7 -7
  74. imap_processing/tests/conftest.py +100 -58
  75. imap_processing/tests/glows/conftest.py +6 -0
  76. imap_processing/tests/glows/test_glows_l1b.py +9 -9
  77. imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
  78. imap_processing/tests/hi/test_data/l0/H90_NHK_20241104.bin +0 -0
  79. imap_processing/tests/hi/test_data/l0/H90_sci_cnt_20241104.bin +0 -0
  80. imap_processing/tests/hi/test_data/l0/H90_sci_de_20241104.bin +0 -0
  81. imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
  82. imap_processing/tests/hi/test_hi_l1b.py +73 -3
  83. imap_processing/tests/hi/test_hi_l1c.py +10 -2
  84. imap_processing/tests/hi/test_l1a.py +31 -58
  85. imap_processing/tests/hi/test_science_direct_event.py +58 -0
  86. imap_processing/tests/hi/test_utils.py +4 -3
  87. imap_processing/tests/hit/test_data/sci_sample1.ccsds +0 -0
  88. imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +95 -36
  89. imap_processing/tests/hit/test_hit_l1a.py +299 -179
  90. imap_processing/tests/hit/test_hit_l1b.py +231 -24
  91. imap_processing/tests/hit/test_hit_utils.py +218 -0
  92. imap_processing/tests/hit/validation_data/hskp_sample_eu.csv +89 -0
  93. imap_processing/tests/hit/validation_data/sci_sample_raw1.csv +29 -0
  94. imap_processing/tests/ialirt/test_data/l0/apid01152.tlm +0 -0
  95. imap_processing/tests/ialirt/test_data/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
  96. imap_processing/tests/ialirt/unit/test_process_codicelo.py +106 -0
  97. imap_processing/tests/ialirt/unit/test_process_ephemeris.py +109 -0
  98. imap_processing/tests/ialirt/unit/test_process_hit.py +9 -6
  99. imap_processing/tests/idex/conftest.py +2 -2
  100. imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
  101. imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
  102. imap_processing/tests/idex/test_idex_l0.py +4 -4
  103. imap_processing/tests/idex/test_idex_l1a.py +8 -2
  104. imap_processing/tests/idex/test_idex_l1b.py +126 -0
  105. imap_processing/tests/lo/test_lo_l1a.py +7 -16
  106. imap_processing/tests/lo/test_lo_science.py +69 -5
  107. imap_processing/tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts +0 -0
  108. imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv +1999 -0
  109. imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
  110. imap_processing/tests/mag/test_mag_l1b.py +97 -7
  111. imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
  112. imap_processing/tests/spice/test_geometry.py +115 -9
  113. imap_processing/tests/spice/test_time.py +135 -6
  114. imap_processing/tests/swapi/test_swapi_decom.py +75 -69
  115. imap_processing/tests/swapi/test_swapi_l1.py +4 -4
  116. imap_processing/tests/swe/conftest.py +33 -0
  117. imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
  118. imap_processing/tests/swe/test_swe_l1b.py +29 -8
  119. imap_processing/tests/swe/test_swe_l2.py +64 -8
  120. imap_processing/tests/test_utils.py +2 -2
  121. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -3314
  122. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
  123. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
  124. imap_processing/tests/ultra/unit/test_de.py +113 -0
  125. imap_processing/tests/ultra/unit/test_spatial_utils.py +125 -0
  126. imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
  127. imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
  128. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +55 -35
  129. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +10 -68
  130. imap_processing/ultra/constants.py +12 -3
  131. imap_processing/ultra/l1b/de.py +168 -30
  132. imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
  133. imap_processing/ultra/l1b/ultra_l1b_extended.py +46 -80
  134. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +60 -144
  135. imap_processing/ultra/utils/spatial_utils.py +221 -0
  136. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/METADATA +15 -14
  137. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/RECORD +142 -139
  138. imap_processing/cdf/cdf_attribute_manager.py +0 -322
  139. imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
  140. imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  141. imap_processing/hi/l0/decom_hi.py +0 -24
  142. imap_processing/hi/packet_definitions/hi_packet_definition.xml +0 -482
  143. imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
  144. imap_processing/hit/l0/data_classes/science_packet.py +0 -259
  145. imap_processing/hit/l0/utils/hit_base.py +0 -57
  146. imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
  147. imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  148. imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
  149. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
  150. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
  151. imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
  152. imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
  153. imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
  154. imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
  155. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
  156. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
  157. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
  158. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
  159. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
  160. imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
  161. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
  162. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
  163. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
  164. imap_processing/tests/hi/test_decom.py +0 -55
  165. imap_processing/tests/hi/test_l1a_sci_de.py +0 -72
  166. imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
  167. imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
  168. /imap_processing/{hi/l0/__init__.py → tests/glows/test_glows_l2_data.py} +0 -0
  169. /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
  170. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/LICENSE +0 -0
  171. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/WHEEL +0 -0
  172. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/entry_points.txt +0 -0
@@ -1,7 +1,6 @@
1
1
  """Tests the L1a processing for decommutated CoDICE data"""
2
2
 
3
3
  import logging
4
- from pathlib import Path
5
4
 
6
5
  import numpy as np
7
6
  import pytest
@@ -10,139 +9,172 @@ import xarray as xr
10
9
  from imap_processing.cdf.utils import load_cdf, write_cdf
11
10
  from imap_processing.codice.codice_l1a import process_codice_l1a
12
11
 
13
- from .conftest import TEST_PACKETS, VALIDATION_DATA
12
+ from .conftest import TEST_L0_FILE, VALIDATION_DATA
14
13
 
15
14
  logger = logging.getLogger(__name__)
16
15
  logger.setLevel(logging.INFO)
17
16
 
18
- # TODO: Add test that processes a file with multiple APIDs
19
-
20
17
  EXPECTED_ARRAY_SHAPES = [
21
- (99,), # hskp
22
- (1, 1, 6, 1), # hi-counters-aggregated # TODO: Double check with Joey
23
- (1, 1, 16, 1), # hi-counters-singles # TODO: Double check with Joey
24
- (1, 15, 4, 1), # hi-omni # TODO: Double check with Joey
25
- (1, 8, 12, 12), # hi-sectored
26
- (1, 1), # hi-pha
27
- (1, 6, 6, 128), # lo-counters-aggregated
28
- (1, 24, 6, 128), # lo-counters-singles
29
- (1, 5, 12, 128), # lo-sw-angular
30
- (1, 19, 12, 128), # lo-nsw-angular
31
- (1, 1, 12, 128), # lo-sw-priority
32
- (1, 1, 12, 128), # lo-nsw-priority
33
- (1, 1, 1, 128), # lo-sw-species
34
- (1, 1, 1, 128), # lo-nsw-species
35
- (1, 128), # lo-pha
18
+ (), # hi-ialirt # TODO: Need to implement
19
+ (), # lo-ialirt # TODO: Need to implement
20
+ (31778,), # hskp
21
+ (77, 128, 6, 6), # lo-counters-aggregated
22
+ (77, 128, 24, 6), # lo-counters-singles
23
+ (77, 128, 1, 12), # lo-sw-priority
24
+ (77, 128, 1, 12), # lo-nsw-priority
25
+ (77, 128, 1, 1), # lo-sw-species
26
+ (77, 128, 1, 1), # lo-nsw-species
27
+ (77, 128, 5, 12), # lo-sw-angular
28
+ (77, 128, 19, 12), # lo-nsw-angular
29
+ (77, 1, 6, 1), # hi-counters-aggregated
30
+ (77, 1, 12, 1), # hi-counters-singles
31
+ (77, 15, 4, 1), # hi-omni
32
+ (77, 8, 12, 12), # hi-sectored
33
+ (), # hi-priority # TODO: Need to implement
34
+ (), # lo-pha # TODO: Need to implement
35
+ (), # hi-pha # TODO: Need to implement
36
36
  ]
37
- EXPECTED_LOGICAL_SOURCE = [
37
+
38
+ EXPECTED_LOGICAL_SOURCES = [
39
+ "imap_codice_l1a_hi-ialirt",
40
+ "imap_codice_l1a_lo-ialirt",
38
41
  "imap_codice_l1a_hskp",
39
- "imap_codice_l1a_hi-counters-aggregated",
40
- "imap_codice_l1a_hi-counters-singles",
41
- "imap_codice_l1a_hi-omni",
42
- "imap_codice_l1a_hi-sectored",
43
- "imap_codice_l1a_hi-pha",
44
42
  "imap_codice_l1a_lo-counters-aggregated",
45
43
  "imap_codice_l1a_lo-counters-singles",
46
- "imap_codice_l1a_lo-sw-angular",
47
- "imap_codice_l1a_lo-nsw-angular",
48
44
  "imap_codice_l1a_lo-sw-priority",
49
45
  "imap_codice_l1a_lo-nsw-priority",
50
46
  "imap_codice_l1a_lo-sw-species",
51
47
  "imap_codice_l1a_lo-nsw-species",
48
+ "imap_codice_l1a_lo-sw-angular",
49
+ "imap_codice_l1a_lo-nsw-angular",
50
+ "imap_codice_l1a_hi-counters-aggregated",
51
+ "imap_codice_l1a_hi-counters-singles",
52
+ "imap_codice_l1a_hi-omni",
53
+ "imap_codice_l1a_hi-sectored",
54
+ "imap_codice_l1a_hi-priority",
52
55
  "imap_codice_l1a_lo-pha",
56
+ "imap_codice_l1a_hi-pha",
53
57
  ]
58
+
54
59
  EXPECTED_NUM_VARIABLES = [
55
- 129, # hskp
56
- 1, # hi-counters-aggregated
57
- 3, # hi-counters-singles
58
- 8, # hi-omni
59
- 4, # hi-sectored
60
- 0, # hi-pha
60
+ 0, # hi-ialirt # TODO: Need to implement
61
+ 0, # lo-ialirt # TODO: Need to implement
62
+ 148, # hskp
61
63
  3, # lo-counters-aggregated
62
64
  3, # lo-counters-singles
63
- 6, # lo-sw-angular
64
- 3, # lo-nsw-angular
65
65
  7, # lo-sw-priority
66
66
  4, # lo-nsw-priority
67
67
  18, # lo-sw-species
68
68
  10, # lo-nsw-species
69
- 0, # lo-pha
69
+ 6, # lo-sw-angular
70
+ 3, # lo-nsw-angular
71
+ 1, # hi-counters-aggregated
72
+ 3, # hi-counters-singles
73
+ 8, # hi-omni
74
+ 4, # hi-sectored
75
+ 0, # hi-priority # TODO: Need to implement
76
+ 0, # lo-pha # TODO: Need to implement
77
+ 0, # hi-pha # TODO: Need to implement
70
78
  ]
71
79
 
72
80
 
73
- @pytest.fixture(params=TEST_PACKETS)
74
- def test_l1a_data(request) -> xr.Dataset:
81
+ @pytest.fixture(scope="session")
82
+ def test_l1a_data() -> xr.Dataset:
75
83
  """Return a ``xarray`` dataset containing test data.
76
84
 
77
85
  Returns
78
86
  -------
79
- dataset : xarray.Dataset
80
- A ``xarray`` dataset containing the test data
87
+ processed_datasets : list[xarray.Dataset]
88
+ A list of ``xarray`` datasets containing the test data
81
89
  """
82
90
 
83
- dataset = process_codice_l1a(file_path=request.param, data_version="001")
91
+ processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE, data_version="001")
92
+
93
+ return processed_datasets
94
+
95
+
96
+ @pytest.mark.parametrize("index", range(len(EXPECTED_ARRAY_SHAPES)))
97
+ def test_l1a_data_array_shape(test_l1a_data, index):
98
+ """Tests that the data arrays in the generated CDFs have the expected shape.
99
+
100
+ Parameters
101
+ ----------
102
+ test_l1a_data : list[xarray.Dataset]
103
+ A list of ``xarray`` datasets containing the test data
104
+ index : int
105
+ The index of the list to test
106
+ """
107
+
108
+ processed_dataset = test_l1a_data[index]
109
+ expected_shape = EXPECTED_ARRAY_SHAPES[index]
110
+
111
+ # Mark currently broken/unsupported datasets as expected to fail
112
+ # TODO: Remove these once they are supported
113
+ if index in [0, 1, 15, 16, 17]:
114
+ pytest.xfail("Data product is currently unsupported")
84
115
 
85
- # Write the dataset to a CDF so it can be manually inspected as well
86
- file_path = write_cdf(dataset)
87
- logger.info(f"CDF file written to {file_path}")
116
+ for variable in processed_dataset:
117
+ if variable in ["energy_table", "acquisition_time_per_step"]:
118
+ assert processed_dataset[variable].data.shape == (128,)
119
+ else:
120
+ assert processed_dataset[variable].data.shape == expected_shape
88
121
 
89
- return dataset
90
122
 
123
+ @pytest.mark.parametrize("index", range(len(EXPECTED_LOGICAL_SOURCES)))
124
+ def test_l1a_logical_sources(test_l1a_data, index):
125
+ """Tests that the Logical source of the dataset is what is expected.
91
126
 
92
- @pytest.mark.xfail(reason="Epoch variable data needs to monotonically increase")
93
- @pytest.mark.parametrize(
94
- "test_l1a_data, expected_logical_source",
95
- list(zip(TEST_PACKETS, EXPECTED_LOGICAL_SOURCE)),
96
- indirect=["test_l1a_data"],
97
- )
98
- def test_l1a_cdf_filenames(test_l1a_data: xr.Dataset, expected_logical_source: str):
99
- """Tests that the ``process_codice_l1a`` function generates datasets
100
- with the expected logical source.
127
+ Since the logical source gets set by ``write_cdf``, this also tests that
128
+ the dataset can be written to a file.
101
129
 
102
130
  Parameters
103
131
  ----------
104
- test_l1a_data : xarray.Dataset
105
- A ``xarray`` dataset containing the test data
106
- expected_logical_source : str
107
- The expected CDF filename
132
+ test_l1a_data : list[xarray.Dataset]
133
+ A list of ``xarray`` datasets containing the test data
134
+ index : int
135
+ The index of the list to test
108
136
  """
109
137
 
110
- dataset = test_l1a_data
111
- assert dataset.attrs["Logical_source"] == expected_logical_source
138
+ processed_dataset = test_l1a_data[index]
139
+ expected_logical_source = EXPECTED_LOGICAL_SOURCES[index]
112
140
 
141
+ # Mark currently broken/unsupported datasets as expected to fail
142
+ # TODO: Remove these once they are supported
143
+ if index in [0, 1, 2, 15, 16, 17]:
144
+ pytest.xfail("Data product is currently unsupported")
113
145
 
114
- @pytest.mark.xfail(reason="Epoch variable data needs to monotonically increase")
115
- @pytest.mark.parametrize(
116
- "test_l1a_data, expected_shape",
117
- list(zip(TEST_PACKETS, EXPECTED_ARRAY_SHAPES)),
118
- indirect=["test_l1a_data"],
119
- )
120
- def test_l1a_data_array_shape(test_l1a_data: xr.Dataset, expected_shape: tuple):
121
- """Tests that the data arrays in the generated CDFs have the expected shape.
146
+ # Write the dataset to a file to set the logical source attribute
147
+ _ = write_cdf(processed_dataset)
148
+
149
+ assert processed_dataset.attrs["Logical_source"] == expected_logical_source
150
+
151
+
152
+ @pytest.mark.parametrize("index", range(len(EXPECTED_NUM_VARIABLES)))
153
+ def test_l1a_num_variables(test_l1a_data, index):
154
+ """Tests that the data arrays in the generated CDFs have the expected number
155
+ of variables.
122
156
 
123
157
  Parameters
124
158
  ----------
125
- test_l1a_data : xarray.Dataset
126
- A ``xarray`` dataset containing the test data
127
- expected_shape : tuple
128
- The expected shape of the data array
159
+ test_l1a_data : list[xarray.Dataset]
160
+ A list of ``xarray`` datasets containing the test data
161
+ index : int
162
+ The index of the list to test
129
163
  """
130
164
 
131
- dataset = test_l1a_data
132
- for variable in dataset:
133
- if variable in ["energy_table", "acquisition_time_per_step"]:
134
- assert dataset[variable].data.shape == (128,)
135
- else:
136
- assert dataset[variable].data.shape == expected_shape
165
+ processed_dataset = test_l1a_data[index]
166
+
167
+ # Mark currently broken/unsupported datasets as expected to fail
168
+ # TODO: Remove these once they are supported
169
+ if index in [0, 1, 15, 16, 17]:
170
+ pytest.xfail("Data product is currently unsupported")
171
+
172
+ assert len(processed_dataset) == EXPECTED_NUM_VARIABLES[index]
137
173
 
138
174
 
139
175
  @pytest.mark.skip("Awaiting validation data")
140
- @pytest.mark.parametrize(
141
- "test_l1a_data, validation_data",
142
- list(zip(TEST_PACKETS, VALIDATION_DATA)),
143
- indirect=["test_l1a_data"],
144
- )
145
- def test_l1a_data_array_values(test_l1a_data: xr.Dataset, validation_data: Path):
176
+ @pytest.mark.parametrize("index", range(len(VALIDATION_DATA)))
177
+ def test_l1a_data_array_values(test_l1a_data: xr.Dataset, index):
146
178
  """Tests that the generated L1a CDF contents are valid.
147
179
 
148
180
  Once proper validation files are acquired, this test function should point
@@ -151,40 +183,28 @@ def test_l1a_data_array_values(test_l1a_data: xr.Dataset, validation_data: Path)
151
183
 
152
184
  Parameters
153
185
  ----------
154
- test_l1a_data : xarray.Dataset
155
- A ``xarray`` dataset containing the test data
156
- validataion_data : pathlib.Path
157
- The path to the file containing the validation data
186
+ test_l1a_data : list[xarray.Dataset]
187
+ A list of ``xarray`` datasets containing the test data
188
+ index : int
189
+ The index of the list to test
158
190
  """
159
191
 
160
192
  generated_dataset = test_l1a_data
161
- validation_dataset = load_cdf(validation_data)
193
+ validation_dataset = load_cdf(VALIDATION_DATA[index])
162
194
 
163
195
  # Ensure the processed data matches the validation data
164
196
  for variable in validation_dataset:
165
197
  assert variable in generated_dataset
166
198
  if variable != "epoch":
167
199
  np.testing.assert_array_equal(
168
- validation_data[variable].data, generated_dataset[variable].data[0]
200
+ validation_dataset[variable].data, generated_dataset[variable].data[0]
169
201
  )
170
202
 
171
203
 
172
- @pytest.mark.xfail(reason="Epoch variable data needs to monotonically increase")
173
- @pytest.mark.parametrize(
174
- "test_l1a_data, expected_num_variables",
175
- list(zip(TEST_PACKETS, EXPECTED_NUM_VARIABLES)),
176
- indirect=["test_l1a_data"],
177
- )
178
- def test_l1a_num_variables(test_l1a_data: xr.Dataset, expected_num_variables: int):
179
- """Tests that the data arrays in the generated CDFs have the expected size.
204
+ def test_l1a_multiple_packets():
205
+ """Tests that an input L0 file containing multiple APIDs can be processed."""
180
206
 
181
- Parameters
182
- ----------
183
- test_l1a_data : xarray.Dataset
184
- A ``xarray`` dataset containing the test data
185
- expected_num_variables : int
186
- The expected number of data variables in the CDF
187
- """
207
+ processed_datasets = process_codice_l1a(file_path=TEST_L0_FILE, data_version="001")
188
208
 
189
- dataset = test_l1a_data
190
- assert len(dataset) == expected_num_variables
209
+ # TODO: Could add some more checks here?
210
+ assert len(processed_datasets) == 18
@@ -39,6 +39,7 @@ def test_l1b_data(request) -> xr.Dataset:
39
39
  return dataset
40
40
 
41
41
 
42
+ @pytest.mark.skip("Awaiting proper implementation of L1B")
42
43
  @pytest.mark.parametrize(
43
44
  "test_l1b_data, expected_logical_source",
44
45
  list(zip(TEST_L1A_FILES, EXPECTED_LOGICAL_SOURCE)),
@@ -10,14 +10,14 @@ from imap_processing.codice.utils import CoDICECompression
10
10
 
11
11
  # Test the algorithms using input value of 234 (picked randomly)
12
12
  lzma_bytes = lzma.compress((234).to_bytes(1, byteorder="big"))
13
- LZMA_EXAMPLE = "".join(format(byte, "08b") for byte in lzma_bytes)
13
+ # LZMA_EXAMPLE = "".join(format(byte, "08b") for byte in lzma_bytes)
14
14
  TEST_DATA = [
15
- ("11101010", CoDICECompression.NO_COMPRESSION, [234]),
16
- ("11101010", CoDICECompression.LOSSY_A, [221184]),
17
- ("11101010", CoDICECompression.LOSSY_B, [1441792]),
18
- (LZMA_EXAMPLE, CoDICECompression.LOSSLESS, [234]),
19
- (LZMA_EXAMPLE, CoDICECompression.LOSSY_A_LOSSLESS, [221184]),
20
- (LZMA_EXAMPLE, CoDICECompression.LOSSY_B_LOSSLESS, [1441792]),
15
+ (b"\xea", CoDICECompression.NO_COMPRESSION, [234]),
16
+ (b"\xea", CoDICECompression.LOSSY_A, [212992]),
17
+ (b"\xea", CoDICECompression.LOSSY_B, [1310720]),
18
+ (lzma_bytes, CoDICECompression.LOSSLESS, [234]),
19
+ (lzma_bytes, CoDICECompression.LOSSY_A_LOSSLESS, [212992]),
20
+ (lzma_bytes, CoDICECompression.LOSSY_B_LOSSLESS, [1310720]),
21
21
  ]
22
22
 
23
23
 
@@ -46,45 +46,48 @@ def _autoclear_spice():
46
46
 
47
47
 
48
48
  @pytest.fixture(scope="session")
49
- def _download_de440s(spice_test_data_path):
50
- """This fixture downloads the de440s.bsp kernel into the
51
- tests/spice/test_data directory if it does not already exist there. The
49
+ def _download_external_kernels(spice_test_data_path):
50
+ """This fixture downloads the de440s.bsp and pck00011.tpc kernels into the
51
+ tests/spice/test_data directory if they do not already exist there. The
52
52
  fixture is not intended to be used directly. It is automatically added to
53
53
  tests marked with "external_kernel" in the hook below."""
54
54
  logger = logging.getLogger(__name__)
55
- kernel_url = (
56
- "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp"
57
- )
58
- kernel_name = kernel_url.split("/")[-1]
59
- local_filepath = spice_test_data_path / kernel_name
60
-
61
- if local_filepath.exists():
62
- return
63
- allowed_attempts = 3
64
- for attempt_number in range(allowed_attempts):
65
- try:
66
- with requests.get(kernel_url, stream=True, timeout=30) as r:
67
- r.raise_for_status()
68
- with open(local_filepath, "wb") as f:
69
- for chunk in r.iter_content(chunk_size=8192):
70
- f.write(chunk)
71
- logger.info("Cached kernel file to %s", local_filepath)
72
- break
73
- except requests.exceptions.RequestException as error:
74
- logger.info(f"Request failed. {error}")
75
- if attempt_number < allowed_attempts:
76
- logger.info(
77
- f"Trying again, retries left "
78
- f"{allowed_attempts - attempt_number}, "
79
- f"Exception: {error}"
80
- )
81
- time.sleep(1)
82
- else:
83
- logger.error(
84
- f"Failed to download file after {allowed_attempts} "
85
- f"attempts, Final Error: {error}"
86
- )
87
- raise
55
+ kernel_urls = [
56
+ "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp",
57
+ "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/pck/pck00011.tpc",
58
+ ]
59
+
60
+ for kernel_url in kernel_urls:
61
+ kernel_name = kernel_url.split("/")[-1]
62
+ local_filepath = spice_test_data_path / kernel_name
63
+
64
+ if local_filepath.exists():
65
+ continue
66
+ allowed_attempts = 3
67
+ for attempt_number in range(allowed_attempts):
68
+ try:
69
+ with requests.get(kernel_url, stream=True, timeout=30) as r:
70
+ r.raise_for_status()
71
+ with open(local_filepath, "wb") as f:
72
+ for chunk in r.iter_content(chunk_size=8192):
73
+ f.write(chunk)
74
+ logger.info("Cached kernel file to %s", local_filepath)
75
+ continue
76
+ except requests.exceptions.RequestException as error:
77
+ logger.info(f"Request failed. {error}")
78
+ if attempt_number < allowed_attempts:
79
+ logger.info(
80
+ f"Trying again, retries left "
81
+ f"{allowed_attempts - attempt_number}, "
82
+ f"Exception: {error}"
83
+ )
84
+ time.sleep(1)
85
+ else:
86
+ logger.error(
87
+ f"Failed to download file {kernel_name} after "
88
+ f"{allowed_attempts} attempts, Final Error: {error}"
89
+ )
90
+ raise
88
91
 
89
92
 
90
93
  def pytest_collection_modifyitems(items):
@@ -93,12 +96,12 @@ def pytest_collection_modifyitems(items):
93
96
  been collected. In this case, it automatically adds fixtures based on the
94
97
  following table:
95
98
 
96
- +---------------------+---------------------+
97
- | pytest mark | fixture added |
98
- +=====================+=====================+
99
- | external_kernel | _download_de440s |
100
- | use_test_metakernel | use_test_metakernel |
101
- +---------------------+---------------------+
99
+ +---------------------+----------------------------+
100
+ | pytest mark | fixture added |
101
+ +=====================+============================+
102
+ | external_kernel | _download_external_kernels |
103
+ | use_test_metakernel | use_test_metakernel |
104
+ +---------------------+----------------------------+
102
105
 
103
106
  Notes
104
107
  -----
@@ -108,7 +111,7 @@ def pytest_collection_modifyitems(items):
108
111
  """
109
112
  for item in items:
110
113
  if item.get_closest_marker("external_kernel") is not None:
111
- item.fixturenames.append("_download_de440s")
114
+ item.fixturenames.append("_download_external_kernels")
112
115
  if item.get_closest_marker("use_test_metakernel") is not None:
113
116
  item.fixturenames.append("use_test_metakernel")
114
117
 
@@ -297,19 +300,55 @@ def _unset_metakernel_path(monkeypatch):
297
300
 
298
301
 
299
302
  @pytest.fixture()
300
- def _set_spin_data_filepath(monkeypatch, tmpdir, generate_spin_data):
301
- """Set the SPIN_DATA_FILEPATH environment variable"""
302
- # SWE test data time minus 56120 seconds to get mid-night time
303
- start_time = 453051323.0 - 56120
304
- spin_df = generate_spin_data(start_time)
305
- spin_csv_file_path = tmpdir / "spin_data.spin.csv"
306
- spin_df.to_csv(spin_csv_file_path, index=False)
307
- monkeypatch.setenv("SPIN_DATA_FILEPATH", str(spin_csv_file_path))
303
+ def use_test_spin_data_csv(monkeypatch):
304
+ """Sets the SPIN_DATA_FILEPATH environment variable to input path."""
305
+
306
+ def wrapped_set_spin_data_filepath(path: Path):
307
+ monkeypatch.setenv("SPIN_DATA_FILEPATH", str(path))
308
+
309
+ return wrapped_set_spin_data_filepath
310
+
311
+
312
+ @pytest.fixture()
313
+ def use_fake_spin_data_for_time(
314
+ request, use_test_spin_data_csv, tmpdir, generate_spin_data
315
+ ):
316
+ """
317
+ Generate and use fake spin data for testing.
318
+
319
+ Returns
320
+ -------
321
+ callable
322
+ Returns a callable function that takes start_met and optionally end_met
323
+ as inputs, generates fake spin data, writes the data to a csv file,
324
+ and sets the SPIN_DATA_FILEPATH environment variable to point to the
325
+ fake spin data file.
326
+ """
327
+
328
+ def wrapped_set_spin_data_filepath(
329
+ start_met: float, end_met: Optional[int] = None
330
+ ) -> pd.DataFrame:
331
+ """
332
+ Generate and use fake spin data for testing.
333
+ Parameters
334
+ ----------
335
+ start_met : int
336
+ Provides the start time in Mission Elapsed Time (MET).
337
+ end_met : int
338
+ Provides the end time in MET. If not provided, default to one day
339
+ from start time.
340
+ """
341
+ spin_df = generate_spin_data(start_met, end_met=end_met)
342
+ spin_csv_file_path = tmpdir / "spin_data.spin.csv"
343
+ spin_df.to_csv(spin_csv_file_path, index=False)
344
+ use_test_spin_data_csv(spin_csv_file_path)
345
+
346
+ return wrapped_set_spin_data_filepath
308
347
 
309
348
 
310
349
  @pytest.fixture()
311
350
  def generate_spin_data():
312
- def make_data(start_met: int, end_met: Optional[int] = None) -> pd.DataFrame:
351
+ def make_data(start_met: float, end_met: Optional[float] = None) -> pd.DataFrame:
313
352
  """
314
353
  Generate a spin table CSV covering one or more days.
315
354
  Spin table contains the following fields:
@@ -324,14 +363,14 @@ def generate_spin_data():
324
363
  thruster_firing
325
364
  )
326
365
  This function creates spin data using start MET and end MET time.
327
- Each spin start data uses the nominal 15 second spin period. The spins that
366
+ Each spin start data uses the nominal 15-second spin period. The spins that
328
367
  occur from 00:00(Mid-night) to 00:10 UTC are marked with flags for
329
368
  thruster firing, invalid spin period, and invalid spin phase.
330
369
  Parameters
331
370
  ----------
332
- start_met : int
371
+ start_met : float
333
372
  Provides the start time in Mission Elapsed Time (MET).
334
- end_met : int
373
+ end_met : float
335
374
  Provides the end time in MET. If not provided, default to one day
336
375
  from start time.
337
376
  Returns
@@ -344,7 +383,8 @@ def generate_spin_data():
344
383
  end_met = start_met + 86400
345
384
 
346
385
  # Create spin start second data of 15 seconds increment
347
- spin_start_sec = np.arange(start_met, end_met + 1, 15)
386
+ spin_start_sec = np.arange(np.floor(start_met), end_met + 1, 15)
387
+ spin_start_subsec = int((start_met - spin_start_sec[0]) * 1000)
348
388
 
349
389
  nspins = len(spin_start_sec)
350
390
 
@@ -352,7 +392,9 @@ def generate_spin_data():
352
392
  {
353
393
  "spin_number": np.arange(nspins, dtype=np.uint32),
354
394
  "spin_start_sec": spin_start_sec,
355
- "spin_start_subsec": np.zeros(nspins, dtype=np.uint32),
395
+ "spin_start_subsec": np.full(
396
+ nspins, spin_start_subsec, dtype=np.uint32
397
+ ),
356
398
  "spin_period_sec": np.full(nspins, 15.0, dtype=np.float32),
357
399
  "spin_period_valid": np.ones(nspins, dtype=np.uint8),
358
400
  "spin_phase_valid": np.ones(nspins, dtype=np.uint8),
@@ -362,7 +404,7 @@ def generate_spin_data():
362
404
  )
363
405
 
364
406
  # Convert spin_start_sec to datetime to set repointing times flags
365
- spin_start_dates = met_to_j2000ns(spin_start_sec)
407
+ spin_start_dates = met_to_j2000ns(spin_start_sec + spin_start_subsec / 1000)
366
408
  spin_start_dates = cdflib.cdfepoch.to_datetime(spin_start_dates)
367
409
 
368
410
  # Convert DatetimeIndex to Series for using .dt accessor
@@ -6,6 +6,7 @@ import pytest
6
6
  from imap_processing.glows.l0 import decom_glows
7
7
  from imap_processing.glows.l1a.glows_l1a import glows_l1a, process_de_l0
8
8
  from imap_processing.glows.l1a.glows_l1a_data import HistogramL1A
9
+ from imap_processing.glows.l1b.glows_l1b import glows_l1b
9
10
 
10
11
 
11
12
  @pytest.fixture()
@@ -40,3 +41,8 @@ def l1a_test_data(decom_test_data):
40
41
  @pytest.fixture()
41
42
  def l1a_dataset(packet_path):
42
43
  return glows_l1a(packet_path, "v001")
44
+
45
+
46
+ @pytest.fixture()
47
+ def l1b_hist_dataset(l1a_dataset):
48
+ return glows_l1b(l1a_dataset[0], "v001")
@@ -318,23 +318,23 @@ def test_glows_l1b(de_dataset, hist_dataset):
318
318
  "imap_spin_angle_bin_cntr",
319
319
  "histogram_flag_array",
320
320
  "filter_temperature_average",
321
- "filter_temperature_variance",
321
+ "filter_temperature_std_dev",
322
322
  "hv_voltage_average",
323
- "hv_voltage_variance",
323
+ "hv_voltage_std_dev",
324
324
  "spin_period_average",
325
- "spin_period_variance",
325
+ "spin_period_std_dev",
326
326
  "pulse_length_average",
327
- "pulse_length_variance",
327
+ "pulse_length_std_dev",
328
328
  "spin_period_ground_average",
329
- "spin_period_ground_variance",
329
+ "spin_period_ground_std_dev",
330
330
  "position_angle_offset_average",
331
- "position_angle_offset_variance",
332
- "spin_axis_orientation_variance",
331
+ "position_angle_offset_std_dev",
332
+ "spin_axis_orientation_std_dev",
333
333
  "spin_axis_orientation_average",
334
334
  "spacecraft_location_average",
335
- "spacecraft_location_variance",
335
+ "spacecraft_location_std_dev",
336
336
  "spacecraft_velocity_average",
337
- "spacecraft_velocity_variance",
337
+ "spacecraft_velocity_std_dev",
338
338
  "flags",
339
339
  ]
340
340