imap-processing 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (398) hide show
  1. imap_processing/__init__.py +34 -0
  2. imap_processing/_version.py +3 -0
  3. imap_processing/ccsds/__init__.py +0 -0
  4. imap_processing/ccsds/ccsds_data.py +55 -0
  5. imap_processing/ccsds/excel_to_xtce.py +477 -0
  6. imap_processing/cdf/__init__.py +0 -0
  7. imap_processing/cdf/cdf_attribute_manager.py +322 -0
  8. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +212 -0
  9. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +1358 -0
  10. imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +391 -0
  11. imap_processing/cdf/config/imap_constant_attrs.yaml +33 -0
  12. imap_processing/cdf/config/imap_default_global_cdf_attrs.yaml +17 -0
  13. imap_processing/cdf/config/imap_glows_global_cdf_attrs.yaml +41 -0
  14. imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml +499 -0
  15. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +334 -0
  16. imap_processing/cdf/config/imap_hi_global_cdf_attrs.yaml +51 -0
  17. imap_processing/cdf/config/imap_hi_variable_attrs.yaml +435 -0
  18. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +27 -0
  19. imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +493 -0
  20. imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +564 -0
  21. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +24 -0
  22. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +426 -0
  23. imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml +90 -0
  24. imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +487 -0
  25. imap_processing/cdf/config/imap_lo_l1b_variable_attrs.yaml +121 -0
  26. imap_processing/cdf/config/imap_lo_l1c_variable_attrs.yaml +179 -0
  27. imap_processing/cdf/config/imap_mag_global_cdf_attrs.yaml +97 -0
  28. imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +201 -0
  29. imap_processing/cdf/config/imap_swapi_global_cdf_attrs.yaml +33 -0
  30. imap_processing/cdf/config/imap_swapi_variable_attrs.yaml +137 -0
  31. imap_processing/cdf/config/imap_swe_global_cdf_attrs.yaml +24 -0
  32. imap_processing/cdf/config/imap_swe_l1a_variable_attrs.yaml +234 -0
  33. imap_processing/cdf/config/imap_swe_l1b_variable_attrs.yaml +273 -0
  34. imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +100 -0
  35. imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +52 -0
  36. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +297 -0
  37. imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +121 -0
  38. imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +246 -0
  39. imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +466 -0
  40. imap_processing/cdf/imap_cdf_manager.py +64 -0
  41. imap_processing/cdf/utils.py +147 -0
  42. imap_processing/cli.py +863 -0
  43. imap_processing/codice/__init__.py +1 -0
  44. imap_processing/codice/codice_l0.py +54 -0
  45. imap_processing/codice/codice_l1a.py +558 -0
  46. imap_processing/codice/codice_l1b.py +194 -0
  47. imap_processing/codice/constants.py +986 -0
  48. imap_processing/codice/data/esa_sweep_values.csv +257 -0
  49. imap_processing/codice/data/lo_stepping_values.csv +129 -0
  50. imap_processing/codice/decompress.py +142 -0
  51. imap_processing/codice/packet_definitions/P_COD_NHK.xml +618 -0
  52. imap_processing/codice/packet_definitions/codice_packet_definition.xml +5073 -0
  53. imap_processing/codice/utils.py +95 -0
  54. imap_processing/decom.py +40 -0
  55. imap_processing/glows/__init__.py +1 -0
  56. imap_processing/glows/ancillary/l1b_conversion_table_v001.json +42 -0
  57. imap_processing/glows/l0/__init__.py +0 -0
  58. imap_processing/glows/l0/decom_glows.py +91 -0
  59. imap_processing/glows/l0/glows_l0_data.py +194 -0
  60. imap_processing/glows/l1a/glows_l1a.py +424 -0
  61. imap_processing/glows/l1a/glows_l1a_data.py +555 -0
  62. imap_processing/glows/l1b/glows_l1b.py +270 -0
  63. imap_processing/glows/l1b/glows_l1b_data.py +583 -0
  64. imap_processing/glows/packet_definitions/GLX_COMBINED.xml +254 -0
  65. imap_processing/glows/packet_definitions/P_GLX_TMSCDE.xml +97 -0
  66. imap_processing/glows/packet_definitions/P_GLX_TMSCHIST.xml +215 -0
  67. imap_processing/glows/utils/__init__.py +0 -0
  68. imap_processing/glows/utils/constants.py +105 -0
  69. imap_processing/hi/__init__.py +1 -0
  70. imap_processing/hi/l0/__init__.py +0 -0
  71. imap_processing/hi/l0/decom_hi.py +24 -0
  72. imap_processing/hi/l1a/__init__.py +0 -0
  73. imap_processing/hi/l1a/hi_l1a.py +73 -0
  74. imap_processing/hi/l1a/histogram.py +142 -0
  75. imap_processing/hi/l1a/housekeeping.py +27 -0
  76. imap_processing/hi/l1a/science_direct_event.py +341 -0
  77. imap_processing/hi/l1b/__init__.py +0 -0
  78. imap_processing/hi/l1b/hi_eng_unit_convert_table.csv +154 -0
  79. imap_processing/hi/l1b/hi_l1b.py +127 -0
  80. imap_processing/hi/l1c/__init__.py +0 -0
  81. imap_processing/hi/l1c/hi_l1c.py +228 -0
  82. imap_processing/hi/packet_definitions/__init__.py +0 -0
  83. imap_processing/hi/packet_definitions/hi_packet_definition.xml +482 -0
  84. imap_processing/hi/utils.py +27 -0
  85. imap_processing/hit/__init__.py +1 -0
  86. imap_processing/hit/l0/__init__.py +0 -0
  87. imap_processing/hit/l0/data_classes/housekeeping.py +240 -0
  88. imap_processing/hit/l0/data_classes/science_packet.py +259 -0
  89. imap_processing/hit/l0/decom_hit.py +467 -0
  90. imap_processing/hit/l0/utils/hit_base.py +57 -0
  91. imap_processing/hit/l1a/__init__.py +0 -0
  92. imap_processing/hit/l1a/hit_l1a.py +254 -0
  93. imap_processing/hit/l1b/hit_l1b.py +179 -0
  94. imap_processing/hit/packet_definitions/hit_packet_definitions.xml +1276 -0
  95. imap_processing/ialirt/__init__.py +0 -0
  96. imap_processing/ialirt/l0/__init__.py +0 -0
  97. imap_processing/ialirt/l0/process_hit.py +220 -0
  98. imap_processing/ialirt/packet_definitions/__init__.py +0 -0
  99. imap_processing/ialirt/packet_definitions/ialirt.xml +778 -0
  100. imap_processing/ialirt/packet_definitions/ialirt_hit.xml +186 -0
  101. imap_processing/idex/__init__.py +2 -0
  102. imap_processing/idex/idex_constants.py +27 -0
  103. imap_processing/idex/idex_l0.py +31 -0
  104. imap_processing/idex/idex_l1a.py +631 -0
  105. imap_processing/idex/packet_definitions/idex_packet_definition.xml +3162 -0
  106. imap_processing/lo/__init__.py +1 -0
  107. imap_processing/lo/l0/__init__.py +0 -0
  108. imap_processing/lo/l0/data_classes/science_direct_events.py +215 -0
  109. imap_processing/lo/l0/data_classes/star_sensor.py +98 -0
  110. imap_processing/lo/l0/decompression_tables/12_to_16_bit.csv +4097 -0
  111. imap_processing/lo/l0/decompression_tables/8_to_12_bit.csv +257 -0
  112. imap_processing/lo/l0/decompression_tables/8_to_16_bit.csv +257 -0
  113. imap_processing/lo/l0/decompression_tables/decompression_tables.py +75 -0
  114. imap_processing/lo/l0/lo_apid.py +15 -0
  115. imap_processing/lo/l0/lo_science.py +150 -0
  116. imap_processing/lo/l0/utils/binary_string.py +59 -0
  117. imap_processing/lo/l0/utils/bit_decompression.py +62 -0
  118. imap_processing/lo/l0/utils/lo_base.py +57 -0
  119. imap_processing/lo/l1a/__init__.py +0 -0
  120. imap_processing/lo/l1a/lo_l1a.py +157 -0
  121. imap_processing/lo/l1b/lo_l1b.py +160 -0
  122. imap_processing/lo/l1c/lo_l1c.py +180 -0
  123. imap_processing/lo/packet_definitions/lo_xtce.xml +3541 -0
  124. imap_processing/mag/__init__.py +2 -0
  125. imap_processing/mag/constants.py +108 -0
  126. imap_processing/mag/l0/decom_mag.py +170 -0
  127. imap_processing/mag/l0/mag_l0_data.py +118 -0
  128. imap_processing/mag/l1a/mag_l1a.py +317 -0
  129. imap_processing/mag/l1a/mag_l1a_data.py +1007 -0
  130. imap_processing/mag/l1b/__init__.py +0 -0
  131. imap_processing/mag/l1b/imap_calibration_mag_20240229_v01.cdf +0 -0
  132. imap_processing/mag/l1b/mag_l1b.py +125 -0
  133. imap_processing/mag/l1c/mag_l1c.py +57 -0
  134. imap_processing/mag/packet_definitions/MAG_SCI_COMBINED.xml +235 -0
  135. imap_processing/quality_flags.py +91 -0
  136. imap_processing/spice/__init__.py +1 -0
  137. imap_processing/spice/geometry.py +322 -0
  138. imap_processing/spice/kernels.py +459 -0
  139. imap_processing/spice/time.py +72 -0
  140. imap_processing/swapi/__init__.py +1 -0
  141. imap_processing/swapi/l1/__init__.py +0 -0
  142. imap_processing/swapi/l1/swapi_l1.py +685 -0
  143. imap_processing/swapi/l2/__init__.py +0 -0
  144. imap_processing/swapi/l2/swapi_l2.py +107 -0
  145. imap_processing/swapi/packet_definitions/__init__.py +0 -0
  146. imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +708 -0
  147. imap_processing/swapi/swapi_utils.py +25 -0
  148. imap_processing/swe/__init__.py +1 -0
  149. imap_processing/swe/l1a/__init__.py +0 -0
  150. imap_processing/swe/l1a/swe_l1a.py +48 -0
  151. imap_processing/swe/l1a/swe_science.py +223 -0
  152. imap_processing/swe/l1b/engineering_unit_convert_table.csv +65 -0
  153. imap_processing/swe/l1b/swe_esa_lookup_table.csv +1441 -0
  154. imap_processing/swe/l1b/swe_l1b.py +49 -0
  155. imap_processing/swe/l1b/swe_l1b_science.py +557 -0
  156. imap_processing/swe/packet_definitions/__init__.py +0 -0
  157. imap_processing/swe/packet_definitions/swe_packet_definition.xml +303 -0
  158. imap_processing/swe/utils/__init__.py +0 -0
  159. imap_processing/swe/utils/swe_utils.py +9 -0
  160. imap_processing/tests/__init__.py +0 -0
  161. imap_processing/tests/ccsds/test_data/expected_output.xml +171 -0
  162. imap_processing/tests/ccsds/test_excel_to_xtce.py +285 -0
  163. imap_processing/tests/cdf/__init__.py +0 -0
  164. imap_processing/tests/cdf/imap_default_global_cdf_attrs.yaml +8 -0
  165. imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +246 -0
  166. imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +466 -0
  167. imap_processing/tests/cdf/test_cdf_attribute_manager.py +353 -0
  168. imap_processing/tests/cdf/test_data/imap_default_global_test_cdf_attrs.yaml +7 -0
  169. imap_processing/tests/cdf/test_data/imap_instrument1_global_cdf_attrs.yaml +14 -0
  170. imap_processing/tests/cdf/test_data/imap_instrument1_level1_variable_attrs.yaml +23 -0
  171. imap_processing/tests/cdf/test_data/imap_instrument2_global_cdf_attrs.yaml +23 -0
  172. imap_processing/tests/cdf/test_data/imap_instrument2_level2_variable_attrs.yaml +30 -0
  173. imap_processing/tests/cdf/test_data/imap_test_global.yaml +26 -0
  174. imap_processing/tests/cdf/test_data/imap_test_variable.yaml +41 -0
  175. imap_processing/tests/cdf/test_imap_cdf_manager.py +62 -0
  176. imap_processing/tests/cdf/test_utils.py +109 -0
  177. imap_processing/tests/codice/__init__.py +0 -0
  178. imap_processing/tests/codice/conftest.py +56 -0
  179. imap_processing/tests/codice/data/eu_unit_lookup_table.csv +101 -0
  180. imap_processing/tests/codice/data/idle_export_eu.COD_NHK_20230822_122700 2.csv +100 -0
  181. imap_processing/tests/codice/data/idle_export_raw.COD_NHK_20230822_122700.csv +100 -0
  182. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
  183. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
  184. imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
  185. imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
  186. imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
  187. imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
  188. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
  189. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
  190. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
  191. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
  192. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
  193. imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
  194. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
  195. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
  196. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
  197. imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.cdf +0 -0
  198. imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.cdf +0 -0
  199. imap_processing/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.cdf +0 -0
  200. imap_processing/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.cdf +0 -0
  201. imap_processing/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.cdf +0 -0
  202. imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.cdf +0 -0
  203. imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.cdf +0 -0
  204. imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.cdf +0 -0
  205. imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.cdf +0 -0
  206. imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.cdf +0 -0
  207. imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.cdf +0 -0
  208. imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.cdf +0 -0
  209. imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.cdf +0 -0
  210. imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-aggregated_20240429_v001.cdf +0 -0
  211. imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-singles_20240429_v001.cdf +0 -0
  212. imap_processing/tests/codice/data/imap_codice_l1b_hi-omni_20240429_v001.cdf +0 -0
  213. imap_processing/tests/codice/data/imap_codice_l1b_hi-sectored_20240429_v001.cdf +0 -0
  214. imap_processing/tests/codice/data/imap_codice_l1b_hskp_20100101_v001.cdf +0 -0
  215. imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-aggregated_20240429_v001.cdf +0 -0
  216. imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-singles_20240429_v001.cdf +0 -0
  217. imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-angular_20240429_v001.cdf +0 -0
  218. imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-priority_20240429_v001.cdf +0 -0
  219. imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-species_20240429_v001.cdf +0 -0
  220. imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-angular_20240429_v001.cdf +0 -0
  221. imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-priority_20240429_v001.cdf +0 -0
  222. imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-species_20240429_v001.cdf +0 -0
  223. imap_processing/tests/codice/test_codice_l0.py +144 -0
  224. imap_processing/tests/codice/test_codice_l1a.py +187 -0
  225. imap_processing/tests/codice/test_codice_l1b.py +60 -0
  226. imap_processing/tests/codice/test_decompress.py +50 -0
  227. imap_processing/tests/conftest.py +372 -0
  228. imap_processing/tests/glows/direct_events_validation_data_l1a.csv +5704 -0
  229. imap_processing/tests/glows/glows_test_packet_20110921_v01.pkts +0 -0
  230. imap_processing/tests/glows/test_glows_decom.py +133 -0
  231. imap_processing/tests/glows/test_glows_l1a_cdf.py +85 -0
  232. imap_processing/tests/glows/test_glows_l1a_data.py +510 -0
  233. imap_processing/tests/glows/test_glows_l1b.py +348 -0
  234. imap_processing/tests/glows/test_glows_l1b_data.py +70 -0
  235. imap_processing/tests/hi/__init__.py +0 -0
  236. imap_processing/tests/hi/conftest.py +133 -0
  237. imap_processing/tests/hi/test_data/l0/20231030_H45_APP_NHK.bin +0 -0
  238. imap_processing/tests/hi/test_data/l0/20231030_H45_APP_NHK.csv +201 -0
  239. imap_processing/tests/hi/test_data/l0/20231030_H45_SCI_CNT.bin +0 -0
  240. imap_processing/tests/hi/test_data/l0/20231030_H45_SCI_DE.bin +0 -0
  241. imap_processing/tests/hi/test_data/l0/README.txt +54 -0
  242. imap_processing/tests/hi/test_decom.py +55 -0
  243. imap_processing/tests/hi/test_hi_l1b.py +31 -0
  244. imap_processing/tests/hi/test_hi_l1c.py +69 -0
  245. imap_processing/tests/hi/test_l1a.py +96 -0
  246. imap_processing/tests/hi/test_l1a_sci_de.py +72 -0
  247. imap_processing/tests/hi/test_utils.py +15 -0
  248. imap_processing/tests/hit/PREFLIGHT_raw_record_2023_256_15_59_04_apid1251.pkts +0 -0
  249. imap_processing/tests/hit/PREFLIGHT_raw_record_2023_256_15_59_04_apid1252.pkts +0 -0
  250. imap_processing/tests/hit/__init__.py +0 -0
  251. imap_processing/tests/hit/test_data/imap_hit_l0_hk_20100105_v001.pkts +0 -0
  252. imap_processing/tests/hit/test_data/sci_sample.ccsds +0 -0
  253. imap_processing/tests/hit/test_hit_decom.py +230 -0
  254. imap_processing/tests/hit/test_hit_l1a.py +224 -0
  255. imap_processing/tests/hit/test_hit_l1b.py +52 -0
  256. imap_processing/tests/hit/validation_data/hskp_sample_raw.csv +88 -0
  257. imap_processing/tests/ialirt/__init__.py +0 -0
  258. imap_processing/tests/ialirt/test_data/l0/IALiRT Raw Packet Telemetry.txt +33 -0
  259. imap_processing/tests/ialirt/test_data/l0/hit_ialirt_sample.ccsds +0 -0
  260. imap_processing/tests/ialirt/test_data/l0/hit_ialirt_sample.csv +1001 -0
  261. imap_processing/tests/ialirt/unit/__init__.py +0 -0
  262. imap_processing/tests/ialirt/unit/test_decom_ialirt.py +94 -0
  263. imap_processing/tests/ialirt/unit/test_process_hit.py +226 -0
  264. imap_processing/tests/idex/__init__.py +0 -0
  265. imap_processing/tests/idex/conftest.py +22 -0
  266. imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
  267. imap_processing/tests/idex/impact_14_tof_high_data.txt +8189 -0
  268. imap_processing/tests/idex/test_idex_l0.py +45 -0
  269. imap_processing/tests/idex/test_idex_l1a.py +91 -0
  270. imap_processing/tests/lo/__init__.py +0 -0
  271. imap_processing/tests/lo/test_binary_string.py +21 -0
  272. imap_processing/tests/lo/test_bit_decompression.py +39 -0
  273. imap_processing/tests/lo/test_cdfs/imap_lo_l0_raw_20240627_v001.pkts +0 -0
  274. imap_processing/tests/lo/test_cdfs/imap_lo_l1a_de_20100101_v001.cdf +0 -0
  275. imap_processing/tests/lo/test_cdfs/imap_lo_l1a_spin_20100101_v001.cdf +0 -0
  276. imap_processing/tests/lo/test_cdfs/imap_lo_l1b_de_20100101_v001.cdf +0 -0
  277. imap_processing/tests/lo/test_lo_l1a.py +66 -0
  278. imap_processing/tests/lo/test_lo_l1b.py +74 -0
  279. imap_processing/tests/lo/test_lo_l1c.py +66 -0
  280. imap_processing/tests/lo/test_science_counts.py +41 -0
  281. imap_processing/tests/lo/test_science_direct_events.py +209 -0
  282. imap_processing/tests/lo/test_star_sensor.py +35 -0
  283. imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
  284. imap_processing/tests/mag/mag_l0_test_data.pkts +0 -0
  285. imap_processing/tests/mag/mag_l0_test_output.csv +37 -0
  286. imap_processing/tests/mag/mag_l1_test_data.pkts +0 -0
  287. imap_processing/tests/mag/mag_l1a_test_output.csv +97 -0
  288. imap_processing/tests/mag/test_mag_decom.py +117 -0
  289. imap_processing/tests/mag/test_mag_l1a.py +856 -0
  290. imap_processing/tests/mag/test_mag_l1b.py +77 -0
  291. imap_processing/tests/mag/test_mag_l1c.py +40 -0
  292. imap_processing/tests/spice/__init__.py +0 -0
  293. imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +4 -0
  294. imap_processing/tests/spice/test_data/imap_science_0001.tf +171 -0
  295. imap_processing/tests/spice/test_data/imap_sclk_0000.tsc +156 -0
  296. imap_processing/tests/spice/test_data/imap_sim_ck_2hr_2secsampling_with_nutation.bc +0 -0
  297. imap_processing/tests/spice/test_data/imap_simple_metakernel.template +3 -0
  298. imap_processing/tests/spice/test_data/imap_spk_demo.bsp +0 -0
  299. imap_processing/tests/spice/test_data/imap_wkcp.tf +1806 -0
  300. imap_processing/tests/spice/test_data/naif0012.tls +150 -0
  301. imap_processing/tests/spice/test_data/sim_1yr_imap_attitude.bc +0 -0
  302. imap_processing/tests/spice/test_data/sim_1yr_imap_pointing_frame.bc +0 -0
  303. imap_processing/tests/spice/test_geometry.py +214 -0
  304. imap_processing/tests/spice/test_kernels.py +272 -0
  305. imap_processing/tests/spice/test_time.py +35 -0
  306. imap_processing/tests/swapi/__init__.py +0 -0
  307. imap_processing/tests/swapi/conftest.py +16 -0
  308. imap_processing/tests/swapi/l0_data/__init__.py +0 -0
  309. imap_processing/tests/swapi/l0_data/imap_swapi_l0_raw_20231012_v001.pkts +0 -0
  310. imap_processing/tests/swapi/l0_validation_data/__init__.py +0 -0
  311. imap_processing/tests/swapi/l0_validation_data/idle_export_eu.SWP_AUT_20231012_125245.csv +124 -0
  312. imap_processing/tests/swapi/l0_validation_data/idle_export_eu.SWP_HK_20231012_125245.csv +98 -0
  313. imap_processing/tests/swapi/l0_validation_data/idle_export_eu.SWP_MG_20231012_125245.csv +9 -0
  314. imap_processing/tests/swapi/l0_validation_data/idle_export_eu.SWP_SCI_20231012_125245.csv +72 -0
  315. imap_processing/tests/swapi/l0_validation_data/idle_export_raw.SWP_AUT_20231012_125245.csv +124 -0
  316. imap_processing/tests/swapi/l0_validation_data/idle_export_raw.SWP_HK_20231012_125245.csv +98 -0
  317. imap_processing/tests/swapi/l0_validation_data/idle_export_raw.SWP_MG_20231012_125245.csv +9 -0
  318. imap_processing/tests/swapi/l0_validation_data/idle_export_raw.SWP_SCI_20231012_125245.csv +72 -0
  319. imap_processing/tests/swapi/test_swapi_decom.py +135 -0
  320. imap_processing/tests/swapi/test_swapi_l1.py +354 -0
  321. imap_processing/tests/swapi/test_swapi_l2.py +21 -0
  322. imap_processing/tests/swe/__init__.py +0 -0
  323. imap_processing/tests/swe/conftest.py +35 -0
  324. imap_processing/tests/swe/decompressed/20230927173238_4th_quarter_decompressed.csv +181 -0
  325. imap_processing/tests/swe/decompressed/20230927173253_1st_quarter_decompressed.csv +181 -0
  326. imap_processing/tests/swe/decompressed/20230927173308_2nd_quarter_decompressed.csv +181 -0
  327. imap_processing/tests/swe/decompressed/20230927173323_3rd_quarter_decompressed.csv +181 -0
  328. imap_processing/tests/swe/l0_data/2024051010_SWE_SCIENCE_packet.bin +0 -0
  329. imap_processing/tests/swe/l0_validation_data/idle_export_eu.SWE_SCIENCE_20240510_092742.csv +544 -0
  330. imap_processing/tests/swe/l0_validation_data/idle_export_raw.SWE_SCIENCE_20240510_092742.csv +363 -0
  331. imap_processing/tests/swe/test_swe_l1a.py +12 -0
  332. imap_processing/tests/swe/test_swe_l1a_science.py +129 -0
  333. imap_processing/tests/swe/test_swe_l1b.py +61 -0
  334. imap_processing/tests/swe/test_swe_l1b_science.py +65 -0
  335. imap_processing/tests/test_cli.py +229 -0
  336. imap_processing/tests/test_decom.py +66 -0
  337. imap_processing/tests/test_quality_flags.py +71 -0
  338. imap_processing/tests/test_utils.py +107 -0
  339. imap_processing/tests/ultra/__init__.py +0 -0
  340. imap_processing/tests/ultra/test_data/l0/FM45_40P_Phi28p5_BeamCal_LinearScan_phi28.50_theta-0.00_20240207T102740.CCSDS +0 -0
  341. imap_processing/tests/ultra/test_data/l0/FM45_7P_Phi0.0_BeamCal_LinearScan_phi0.04_theta-0.01_20230821T121304.CCSDS +0 -0
  342. imap_processing/tests/ultra/test_data/l0/FM45_TV_Cycle6_Hot_Ops_Front212_20240124T063837.CCSDS +0 -0
  343. imap_processing/tests/ultra/test_data/l0/Ultra45_EM_SwRI_Cal_Run7_ThetaScan_20220530T225054.CCSDS +0 -0
  344. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_auxdata_Ultra45_EM_SwRI_Cal_Run7_ThetaScan_20220530T225054.csv +24 -0
  345. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_enaphxtofhangimg_FM45_TV_Cycle6_Hot_Ops_Front212_20240124T063837.csv +105 -0
  346. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultraimgrates_Ultra45_EM_SwRI_Cal_Run7_ThetaScan_20220530T225054.csv +24 -0
  347. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -0
  348. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimgevent_FM45_7P_Phi00_BeamCal_LinearScan_phi004_theta-001_20230821T121304.csv +702 -0
  349. imap_processing/tests/ultra/unit/__init__.py +0 -0
  350. imap_processing/tests/ultra/unit/conftest.py +210 -0
  351. imap_processing/tests/ultra/unit/test_decom_apid_880.py +98 -0
  352. imap_processing/tests/ultra/unit/test_decom_apid_881.py +50 -0
  353. imap_processing/tests/ultra/unit/test_decom_apid_883.py +44 -0
  354. imap_processing/tests/ultra/unit/test_decom_apid_896.py +104 -0
  355. imap_processing/tests/ultra/unit/test_lookup_utils.py +68 -0
  356. imap_processing/tests/ultra/unit/test_ultra_l1a.py +338 -0
  357. imap_processing/tests/ultra/unit/test_ultra_l1b.py +122 -0
  358. imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +57 -0
  359. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +342 -0
  360. imap_processing/tests/ultra/unit/test_ultra_l1c.py +104 -0
  361. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +35 -0
  362. imap_processing/ultra/__init__.py +1 -0
  363. imap_processing/ultra/constants.py +60 -0
  364. imap_processing/ultra/l0/__init__.py +0 -0
  365. imap_processing/ultra/l0/decom_tools.py +281 -0
  366. imap_processing/ultra/l0/decom_ultra.py +278 -0
  367. imap_processing/ultra/l0/ultra_utils.py +326 -0
  368. imap_processing/ultra/l1a/__init__.py +0 -0
  369. imap_processing/ultra/l1a/ultra_l1a.py +319 -0
  370. imap_processing/ultra/l1b/badtimes.py +26 -0
  371. imap_processing/ultra/l1b/cullingmask.py +26 -0
  372. imap_processing/ultra/l1b/de.py +59 -0
  373. imap_processing/ultra/l1b/extendedspin.py +45 -0
  374. imap_processing/ultra/l1b/lookup_utils.py +165 -0
  375. imap_processing/ultra/l1b/ultra_l1b.py +65 -0
  376. imap_processing/ultra/l1b/ultra_l1b_annotated.py +54 -0
  377. imap_processing/ultra/l1b/ultra_l1b_extended.py +764 -0
  378. imap_processing/ultra/l1c/histogram.py +36 -0
  379. imap_processing/ultra/l1c/pset.py +36 -0
  380. imap_processing/ultra/l1c/ultra_l1c.py +52 -0
  381. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +54 -0
  382. imap_processing/ultra/lookup_tables/EgyNorm.mem.csv +32769 -0
  383. imap_processing/ultra/lookup_tables/FM45_Startup1_ULTRA_IMGPARAMS_20240719.csv +2 -0
  384. imap_processing/ultra/lookup_tables/ultra45_back-pos-luts.csv +4097 -0
  385. imap_processing/ultra/lookup_tables/ultra45_tdc_norm.csv +2050 -0
  386. imap_processing/ultra/lookup_tables/ultra90_back-pos-luts.csv +4097 -0
  387. imap_processing/ultra/lookup_tables/ultra90_tdc_norm.csv +2050 -0
  388. imap_processing/ultra/lookup_tables/yadjust.csv +257 -0
  389. imap_processing/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml +547 -0
  390. imap_processing/ultra/packet_definitions/__init__.py +0 -0
  391. imap_processing/ultra/utils/__init__.py +0 -0
  392. imap_processing/ultra/utils/ultra_l1_utils.py +50 -0
  393. imap_processing/utils.py +413 -0
  394. imap_processing-0.6.0.dist-info/LICENSE +21 -0
  395. imap_processing-0.6.0.dist-info/METADATA +107 -0
  396. imap_processing-0.6.0.dist-info/RECORD +398 -0
  397. imap_processing-0.6.0.dist-info/WHEEL +4 -0
  398. imap_processing-0.6.0.dist-info/entry_points.txt +4 -0
imap_processing/cli.py ADDED
@@ -0,0 +1,863 @@
1
+ #!/usr/bin/env python3
2
+ # ruff: noqa: PLR0913
3
+ """
4
+ Run the processing for a specific instrument & data level.
5
+
6
+ This module serves as a command line utility to invoke the processing for
7
+ a user-supplied instrument and data level.
8
+
9
+ Examples
10
+ --------
11
+ imap_cli --instrument <instrument> --level <data_level>
12
+ """
13
+
14
+ import argparse
15
+ import logging
16
+ import sys
17
+ from abc import ABC, abstractmethod
18
+ from json import loads
19
+ from pathlib import Path
20
+ from typing import final
21
+ from urllib.error import HTTPError
22
+
23
+ import imap_data_access
24
+ import xarray as xr
25
+
26
+ import imap_processing
27
+ from imap_processing._version import __version__, __version_tuple__ # noqa: F401
28
+ from imap_processing.cdf.utils import load_cdf, write_cdf
29
+
30
+ # TODO: change how we import things and also folder
31
+ # structure may?
32
+ # From this:
33
+ # from imap_processing.cdf.utils import write_cdf
34
+ # To this:
35
+ # from imap_processing import cdf
36
+ # In code:
37
+ # call cdf.utils.write_cdf
38
+ from imap_processing.codice import codice_l1a, codice_l1b
39
+ from imap_processing.glows.l1a.glows_l1a import glows_l1a
40
+ from imap_processing.glows.l1b.glows_l1b import glows_l1b
41
+ from imap_processing.hi.l1a import hi_l1a
42
+ from imap_processing.hi.l1b import hi_l1b
43
+ from imap_processing.hi.l1c import hi_l1c
44
+ from imap_processing.hit.l1a.hit_l1a import hit_l1a
45
+ from imap_processing.hit.l1b.hit_l1b import hit_l1b
46
+ from imap_processing.idex.idex_l1a import PacketParser
47
+ from imap_processing.lo.l1a import lo_l1a
48
+ from imap_processing.lo.l1b import lo_l1b
49
+ from imap_processing.lo.l1c import lo_l1c
50
+ from imap_processing.mag.l1a.mag_l1a import mag_l1a
51
+ from imap_processing.mag.l1b.mag_l1b import mag_l1b
52
+ from imap_processing.mag.l1c.mag_l1c import mag_l1c
53
+ from imap_processing.swapi.l1.swapi_l1 import swapi_l1
54
+ from imap_processing.swapi.l2.swapi_l2 import swapi_l2
55
+ from imap_processing.swe.l1a.swe_l1a import swe_l1a
56
+ from imap_processing.swe.l1b.swe_l1b import swe_l1b
57
+ from imap_processing.ultra.l1a import ultra_l1a
58
+ from imap_processing.ultra.l1b import ultra_l1b
59
+ from imap_processing.ultra.l1c import ultra_l1c
60
+
61
+ logger = logging.getLogger(__name__)
62
+
63
+
64
+ def _parse_args() -> argparse.Namespace:
65
+ """
66
+ Parse the command line arguments.
67
+
68
+ The expected input format is:
69
+ --instrument "mag"
70
+ --data-level "l1a"
71
+ --descriptor "all"
72
+ --start-date "20231212"
73
+ --version "v001"
74
+ --dependency "[
75
+ {
76
+ 'instrument': 'mag',
77
+ 'data_level': 'l0',
78
+ 'descriptor': 'sci',
79
+ 'version': 'v001',
80
+ 'start_date': '20231212'
81
+ }]"
82
+ --upload-to-sdc
83
+
84
+ Returns
85
+ -------
86
+ args : argparse.Namespace
87
+ An object containing the parsed arguments and their values.
88
+ """
89
+ description = (
90
+ "This command line program invokes the processing pipeline "
91
+ "for a specific instrument and data level. Example usage: "
92
+ '"imap_cli --instrument "mag" '
93
+ '--data-level "l1a"'
94
+ '--descriptor "all"'
95
+ ' --start-date "20231212"'
96
+ '--version "v001"'
97
+ '--dependency "['
98
+ ' {"instrument": "mag",'
99
+ ' "data_level": "l0",'
100
+ ' "descriptor": "sci",'
101
+ ' "version": "v001",'
102
+ ' "start_date": "20231212"'
103
+ '}]" --upload-to-sdc"'
104
+ )
105
+ instrument_help = (
106
+ "The instrument to process. Acceptable values are: "
107
+ f"{imap_data_access.VALID_INSTRUMENTS}"
108
+ )
109
+ level_help = (
110
+ "The data level to process. Acceptable values are: "
111
+ f"{imap_processing.PROCESSING_LEVELS}"
112
+ )
113
+ descriptor_help = (
114
+ "The descriptor of the product to process. This could be 'all' or a specific "
115
+ "descriptor like 'sci-1min'. Default is 'all'."
116
+ )
117
+ dependency_help = (
118
+ "Dependency information in str format."
119
+ "Example: '[{'instrument': 'mag',"
120
+ "'data_level': 'l0',"
121
+ "'descriptor': 'sci',"
122
+ "'version': 'v001',"
123
+ "'start_date': '20231212'}]"
124
+ )
125
+
126
+ parser = argparse.ArgumentParser(prog="imap_cli", description=description)
127
+ # TODO: Add version here and change our current "version" to "data-version"?
128
+ # parser.add_argument(
129
+ # "--version",
130
+ # action="version",
131
+ # version=f"%(prog)s {imap_processing.__version__}",
132
+ # )
133
+ # Logging level
134
+ parser.add_argument(
135
+ "--debug",
136
+ help="Print lots of debugging statements",
137
+ action="store_const",
138
+ dest="loglevel",
139
+ const=logging.DEBUG,
140
+ default=logging.WARNING,
141
+ )
142
+ parser.add_argument(
143
+ "-v",
144
+ "--verbose",
145
+ help="Add verbose output",
146
+ action="store_const",
147
+ dest="loglevel",
148
+ const=logging.INFO,
149
+ )
150
+ parser.add_argument("--instrument", type=str, required=True, help=instrument_help)
151
+ parser.add_argument("--data-level", type=str, required=True, help=level_help)
152
+ # TODO: unused for now, but needed for batch job handling
153
+ # pass through of status in AWS
154
+ parser.add_argument(
155
+ "--descriptor", type=str, required=False, help=descriptor_help, default="all"
156
+ )
157
+
158
+ parser.add_argument(
159
+ "--start-date",
160
+ type=str,
161
+ required=True,
162
+ help="Start time for the output data. Format: YYYYMMDD",
163
+ )
164
+
165
+ parser.add_argument(
166
+ "--end-date",
167
+ type=str,
168
+ required=False,
169
+ help="End time for the output data. If not provided, start_time will be used "
170
+ "for end_time. Format: YYYYMMDD",
171
+ )
172
+ # TODO: Will need to add some way of including pointing numbers
173
+
174
+ parser.add_argument(
175
+ "--version",
176
+ type=str,
177
+ required=True,
178
+ help="Version of the data. Format: vXXX",
179
+ )
180
+ parser.add_argument(
181
+ "--dependency",
182
+ type=str,
183
+ required=True,
184
+ help=dependency_help,
185
+ )
186
+
187
+ parser.add_argument(
188
+ "--upload-to-sdc",
189
+ action="store_true",
190
+ required=False,
191
+ help="Upload completed output files to the IMAP SDC.",
192
+ )
193
+ args = parser.parse_args()
194
+
195
+ return args
196
+
197
+
198
+ def _validate_args(args: argparse.Namespace) -> None:
199
+ """
200
+ Ensure that the arguments are valid before kicking off the processing.
201
+
202
+ Parameters
203
+ ----------
204
+ args : argparse.Namespace
205
+ An object containing the parsed arguments and their values.
206
+ """
207
+ if args.instrument not in imap_data_access.VALID_INSTRUMENTS:
208
+ raise ValueError(
209
+ f"{args.instrument} is not in the supported instrument list: "
210
+ f"{imap_data_access.VALID_INSTRUMENTS}"
211
+ )
212
+ if args.data_level not in imap_processing.PROCESSING_LEVELS[args.instrument]:
213
+ raise ValueError(
214
+ f"{args.data_level} is not a supported data level for the {args.instrument}"
215
+ " instrument, valid levels are: "
216
+ f"{imap_processing.PROCESSING_LEVELS[args.instrument]}"
217
+ )
218
+
219
+
220
+ class ProcessInstrument(ABC):
221
+ """
222
+ An abstract base class containing a method to process an instrument.
223
+
224
+ Parameters
225
+ ----------
226
+ data_level : str
227
+ The data level to process (e.g. ``l1a``).
228
+ dependency_str : str
229
+ A string representation of the dependencies for the instrument in the
230
+ format: "[{
231
+ 'instrument': 'mag',
232
+ 'data_level': 'l0',
233
+ 'descriptor': 'sci',
234
+ 'version': 'v00-01',
235
+ 'start_date': '20231212'
236
+ }]".
237
+ start_date : str
238
+ The start date for the output data in YYYYMMDD format.
239
+ end_date : str
240
+ The end date for the output data in YYYYMMDD format.
241
+ version : str
242
+ The version of the data in vXXX format.
243
+ upload_to_sdc : bool
244
+ A flag indicating whether to upload the output file to the SDC.
245
+ """
246
+
247
+ def __init__(
248
+ self,
249
+ data_level: str,
250
+ dependency_str: str,
251
+ start_date: str,
252
+ end_date: str,
253
+ version: str,
254
+ upload_to_sdc: bool,
255
+ ) -> None:
256
+ self.data_level = data_level
257
+
258
+ # Convert string into a dictionary
259
+ self.dependencies = loads(dependency_str.replace("'", '"'))
260
+ self._dependency_list: list = []
261
+
262
+ self.start_date = start_date
263
+ self.end_date = end_date
264
+ if not end_date:
265
+ self.end_date = start_date
266
+
267
+ self.version = version
268
+ self.upload_to_sdc = upload_to_sdc
269
+
270
+ def download_dependencies(self) -> list[Path]:
271
+ """
272
+ Download the dependencies for the instrument.
273
+
274
+ Returns
275
+ -------
276
+ file_list : list[Path]
277
+ A list of file paths to the downloaded dependencies.
278
+ """
279
+ file_list = []
280
+ for dependency in self.dependencies:
281
+ try:
282
+ # TODO: Validate dep dict
283
+ # TODO: determine what dependency information is optional
284
+ return_query = imap_data_access.query(
285
+ start_date=self.start_date,
286
+ end_date=self.end_date,
287
+ instrument=dependency["instrument"],
288
+ data_level=dependency["data_level"],
289
+ version=dependency["version"],
290
+ descriptor=dependency["descriptor"],
291
+ )
292
+ except HTTPError as e:
293
+ raise ValueError(f"Unable to download files from {dependency}") from e
294
+
295
+ if not return_query:
296
+ raise FileNotFoundError(
297
+ f"File not found for required dependency "
298
+ f"{dependency} while attempting to create file."
299
+ f"This should never occur "
300
+ f"in normal processing."
301
+ )
302
+ file_list.extend(
303
+ [
304
+ imap_data_access.download(query_return["file_path"])
305
+ for query_return in return_query
306
+ ]
307
+ )
308
+ return file_list
309
+
310
+ def upload_products(self, products: list[Path]) -> None:
311
+ """
312
+ Upload data products to the IMAP SDC.
313
+
314
+ Parameters
315
+ ----------
316
+ products : list[Path]
317
+ A list of file paths to upload to the SDC.
318
+ """
319
+ if self.upload_to_sdc:
320
+ if len(products) == 0:
321
+ logger.info("No files to upload.")
322
+ for filename in products:
323
+ logger.info(f"Uploading file: {filename}")
324
+ imap_data_access.upload(filename)
325
+
326
+ @final
327
+ def process(self) -> None:
328
+ """
329
+ Run the processing workflow and cannot be overridden by subclasses.
330
+
331
+ Each IMAP processing step consists of three steps:
332
+ 1. Pre-processing actions such as downloading dependencies for processing.
333
+ 2. Do the data processing. The result of this step will usually be a list
334
+ of new products (files).
335
+ 3. Post-processing actions such as uploading files to the IMAP SDC.
336
+ """
337
+ logger.info(f"IMAP Processing Version: {imap_processing._version.__version__}")
338
+ logger.info(f"Processing {self.__class__.__name__} level {self.data_level}")
339
+ logger.info("Beginning preprocessing (download dependencies)")
340
+ dependencies = self.pre_processing()
341
+ logger.info("Beginning actual processing")
342
+ products = self.do_processing(dependencies)
343
+ logger.info("Beginning postprocessing (uploading data products)")
344
+ self.post_processing(products)
345
+ logger.info("Processing complete")
346
+
347
+ def pre_processing(self) -> list[Path]:
348
+ """
349
+ Complete pre-processing.
350
+
351
+ For this baseclass, pre-processing consists of downloading dependencies
352
+ for processing. Child classes can override this method to customize the
353
+ pre-processing actions.
354
+
355
+ Returns
356
+ -------
357
+ list[Path]
358
+ List of dependencies downloaded from the IMAP SDC.
359
+ """
360
+ self._dependency_list = self.download_dependencies()
361
+ return self._dependency_list
362
+
363
+ @abstractmethod
364
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
365
+ """
366
+ Abstract method that processes the IMAP processing steps.
367
+
368
+ All child classes must implement this method. Input and outputs are
369
+ typically lists of file paths but are free to any list.
370
+
371
+ Parameters
372
+ ----------
373
+ dependencies : list
374
+ List of dependencies to process.
375
+
376
+ Returns
377
+ -------
378
+ list[xr.Dataset]
379
+ List of products produced.
380
+ """
381
+ raise NotImplementedError
382
+
383
+ def post_processing(self, datasets: list[xr.Dataset]) -> None:
384
+ """
385
+ Complete post-processing.
386
+
387
+ Default post-processing consists of writing the datasets to local storage
388
+ and then uploading those newly generated products to the IMAP SDC.
389
+ Child classes can override this method to customize the
390
+ post-processing actions.
391
+
392
+ Parameters
393
+ ----------
394
+ datasets : list[xarray.Dataset]
395
+ A list of datasets (products) produced by do_processing method.
396
+ """
397
+ logger.info("Writing products to local storage")
398
+ products = [
399
+ write_cdf(dataset, parent_files=self._dependency_list)
400
+ for dataset in datasets
401
+ ]
402
+ self.upload_products(products)
403
+
404
+
405
+ class Codice(ProcessInstrument):
406
+ """Process CoDICE."""
407
+
408
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
409
+ """
410
+ Perform CoDICE specific processing.
411
+
412
+ Parameters
413
+ ----------
414
+ dependencies : list
415
+ List of dependencies to process.
416
+
417
+ Returns
418
+ -------
419
+ dataset : xr.Dataset
420
+ Xr.Dataset of cdf file paths.
421
+ """
422
+ print(f"Processing CoDICE {self.data_level}")
423
+ datasets: list[xr.Dataset] = []
424
+
425
+ if self.data_level == "l1a":
426
+ if len(dependencies) > 1:
427
+ raise ValueError(
428
+ f"Unexpected dependencies found for CoDICE L1a:"
429
+ f"{dependencies}. Expected only one dependency."
430
+ )
431
+ # process data
432
+ datasets = [codice_l1a.process_codice_l1a(dependencies[0], self.version)]
433
+
434
+ if self.data_level == "l1b":
435
+ if len(dependencies) > 1:
436
+ raise ValueError(
437
+ f"Unexpected dependencies found for CoDICE L1b:"
438
+ f"{dependencies}. Expected only one dependency."
439
+ )
440
+ # process data
441
+ dependency = load_cdf(dependencies[0])
442
+ datasets = [codice_l1b.process_codice_l1b(dependency, self.version)]
443
+
444
+ return datasets
445
+
446
+
447
+ class Glows(ProcessInstrument):
448
+ """Process GLOWS."""
449
+
450
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
451
+ """
452
+ Perform GLOWS specific processing.
453
+
454
+ Parameters
455
+ ----------
456
+ dependencies : list
457
+ List of dependencies to process.
458
+
459
+ Returns
460
+ -------
461
+ datasets : xr.Dataset
462
+ Xr.dataset of products.
463
+ """
464
+ print(f"Processing GLOWS {self.data_level}")
465
+ datasets: list[xr.Dataset] = []
466
+
467
+ if self.data_level == "l1a":
468
+ if len(dependencies) > 1:
469
+ raise ValueError(
470
+ f"Unexpected dependencies found for GLOWS L1A:"
471
+ f"{dependencies}. Expected only one input dependency."
472
+ )
473
+ datasets = glows_l1a(dependencies[0], self.version)
474
+
475
+ if self.data_level == "l1b":
476
+ if len(dependencies) > 1:
477
+ raise ValueError(
478
+ f"Unexpected dependencies found for GLOWS L1B:"
479
+ f"{dependencies}. Expected at least one input dependency."
480
+ )
481
+ input_dataset = load_cdf(dependencies[0])
482
+ datasets = [glows_l1b(input_dataset, self.version)]
483
+
484
+ return datasets
485
+
486
+
487
+ class Hi(ProcessInstrument):
488
+ """Process IMAP-Hi."""
489
+
490
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
491
+ """
492
+ Perform IMAP-Hi specific processing.
493
+
494
+ Parameters
495
+ ----------
496
+ dependencies : list
497
+ List of dependencies to process.
498
+
499
+ Returns
500
+ -------
501
+ datasets : xr.Dataset
502
+ Xr.Dataset of products.
503
+ """
504
+ print(f"Processing IMAP-Hi {self.data_level}")
505
+ datasets: list[xr.Dataset] = []
506
+
507
+ if self.data_level == "l1a":
508
+ # File path is expected output file path
509
+ if len(dependencies) > 1:
510
+ raise ValueError(
511
+ f"Unexpected dependencies found for Hi L1A:"
512
+ f"{dependencies}. Expected only one dependency."
513
+ )
514
+ datasets = hi_l1a.hi_l1a(dependencies[0], self.version)
515
+ elif self.data_level == "l1b":
516
+ dependencies = [load_cdf(dependency) for dependency in dependencies]
517
+ datasets = [hi_l1b.hi_l1b(dependencies[0], self.version)]
518
+ elif self.data_level == "l1c":
519
+ dependencies = [load_cdf(dependency) for dependency in dependencies]
520
+ datasets = [hi_l1c.hi_l1c(dependencies, self.version)]
521
+ else:
522
+ raise NotImplementedError(
523
+ f"Hi processing not implemented for level {self.data_level}"
524
+ )
525
+ return datasets
526
+
527
+
528
+ class Hit(ProcessInstrument):
529
+ """Process HIT."""
530
+
531
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
532
+ """
533
+ Perform HIT specific processing.
534
+
535
+ Parameters
536
+ ----------
537
+ dependencies : list
538
+ List of dependencies to process.
539
+
540
+ Returns
541
+ -------
542
+ datasets : xr.Dataset
543
+ Xr.Dataset of datasets.
544
+ """
545
+ print(f"Processing HIT {self.data_level}")
546
+ datasets: list[xr.Dataset] = []
547
+
548
+ if self.data_level == "l1a":
549
+ if len(dependencies) > 1:
550
+ raise ValueError(
551
+ f"Unexpected dependencies found for HIT L1A:"
552
+ f"{dependencies}. Expected only one dependency."
553
+ )
554
+ # process data and write all processed data to CDF files
555
+ datasets = hit_l1a(dependencies[0], self.version)
556
+
557
+ elif self.data_level == "l1b":
558
+ if len(dependencies) > 1:
559
+ raise ValueError(
560
+ f"Unexpected dependencies found for HIT L1B:"
561
+ f"{dependencies}. Expected only one dependency."
562
+ )
563
+ # process data and write all processed data to CDF files
564
+ l1a_dataset = load_cdf(dependencies[0])
565
+
566
+ datasets = hit_l1b(l1a_dataset, self.version)
567
+
568
+ return datasets
569
+
570
+
571
+ class Idex(ProcessInstrument):
572
+ """Process IDEX."""
573
+
574
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
575
+ """
576
+ Perform IDEX specific processing.
577
+
578
+ Parameters
579
+ ----------
580
+ dependencies : list
581
+ List of dependencies to process.
582
+
583
+ Returns
584
+ -------
585
+ datasets : xr.Dataset
586
+ Xr.Dataset of cdf file paths.
587
+ """
588
+ print(f"Processing IDEX {self.data_level}")
589
+ datasets: list[xr.Dataset] = []
590
+
591
+ if self.data_level == "l1":
592
+ if len(dependencies) > 1:
593
+ raise ValueError(
594
+ f"Unexpected dependencies found for IDEX L1:"
595
+ f"{dependencies}. Expected only one dependency."
596
+ )
597
+ # read CDF file
598
+
599
+ datasets = PacketParser(dependencies[0], self.version).data
600
+ return datasets
601
+
602
+
603
+ class Lo(ProcessInstrument):
604
+ """Process IMAP-Lo."""
605
+
606
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
607
+ """
608
+ Perform IMAP-Lo specific processing.
609
+
610
+ Parameters
611
+ ----------
612
+ dependencies : list
613
+ List of dependencies to process.
614
+
615
+ Returns
616
+ -------
617
+ dataset : xr.Dataset
618
+ Xr.Dataset of output files.
619
+ """
620
+ print(f"Processing IMAP-Lo {self.data_level}")
621
+ datasets: list[xr.Dataset] = []
622
+ if self.data_level == "l1a":
623
+ # L1A packet / products are 1 to 1. Should only have
624
+ # one dependency file
625
+ if len(dependencies) > 1:
626
+ raise ValueError(
627
+ f"Unexpected dependencies found for IMAP-Lo L1A:"
628
+ f"{dependencies}. Expected only one dependency."
629
+ )
630
+ datasets = lo_l1a.lo_l1a(dependencies[0], self.version)
631
+
632
+ elif self.data_level == "l1b":
633
+ data_dict = {}
634
+ for dependency in dependencies:
635
+ dataset = load_cdf(dependency)
636
+ data_dict[dataset.attrs["Logical_source"]] = dataset
637
+ datasets = lo_l1b.lo_l1b(data_dict, self.version)
638
+
639
+ elif self.data_level == "l1c":
640
+ data_dict = {}
641
+ for dependency in dependencies:
642
+ dataset = load_cdf(dependency)
643
+ data_dict[dataset.attrs["Logical_source"]] = dataset
644
+ # TODO: This is returning the wrong type
645
+ datasets = lo_l1c.lo_l1c(data_dict, self.version)
646
+
647
+ return datasets
648
+
649
+
650
+ class Mag(ProcessInstrument):
651
+ """Process MAG."""
652
+
653
+ def do_processing(self, dependencies: list[Path]) -> list[xr.Dataset]:
654
+ """
655
+ Perform MAG specific processing.
656
+
657
+ Parameters
658
+ ----------
659
+ dependencies : list[Path]
660
+ List of dependencies to process.
661
+
662
+ Returns
663
+ -------
664
+ dataset : xr.Dataset
665
+ Xr.Dataset of output files.
666
+ """
667
+ print(f"Processing MAG {self.data_level}")
668
+ datasets: list[xr.Dataset] = []
669
+
670
+ if self.data_level == "l1a":
671
+ # File path is expected output file path
672
+ if len(dependencies) > 1:
673
+ raise ValueError(
674
+ f"Unexpected dependencies found for MAG L1A:"
675
+ f"{dependencies}. Expected only one dependency."
676
+ )
677
+ # TODO: Update this type
678
+ datasets = mag_l1a(dependencies[0], data_version=self.version)
679
+
680
+ if self.data_level == "l1b":
681
+ if len(dependencies) > 1:
682
+ raise ValueError(
683
+ f"Unexpected dependencies found for MAG L1B:"
684
+ f"{dependencies}. Expected only one dependency."
685
+ )
686
+ input_data = load_cdf(dependencies[0])
687
+ datasets = [mag_l1b(input_data, self.version)]
688
+
689
+ if self.data_level == "l1c":
690
+ # L1C depends on matching norm/burst files: eg burst-magi and norm-magi or
691
+ # burst-mago and norm-mago
692
+ if len(dependencies) != 2:
693
+ raise ValueError(
694
+ f"Invalid dependencies found for MAG L1C:"
695
+ f"{dependencies}. Expected two dependencies."
696
+ )
697
+
698
+ input_data = [load_cdf(dep) for dep in dependencies]
699
+ # Input datasets can be in any order
700
+ datasets = [mag_l1c(input_data[0], input_data[1], self.version)]
701
+
702
+ return datasets
703
+
704
+
705
+ class Swapi(ProcessInstrument):
706
+ """Process SWAPI."""
707
+
708
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
709
+ """
710
+ Perform SWAPI specific processing.
711
+
712
+ Parameters
713
+ ----------
714
+ dependencies : list
715
+ List of dependencies to process.
716
+
717
+ Returns
718
+ -------
719
+ dataset : xr.Dataset
720
+ Xr.Dataset of products.
721
+ """
722
+ print(f"Processing SWAPI {self.data_level}")
723
+ datasets: list[xr.Dataset] = []
724
+
725
+ if self.data_level == "l1":
726
+ if len(dependencies) > 1:
727
+ raise ValueError(
728
+ f"Unexpected dependencies found for SWAPI L1:"
729
+ f"{dependencies}. Expected only one dependency."
730
+ )
731
+ # process data
732
+ datasets = [swapi_l1(dependencies[0], self.version)]
733
+ elif self.data_level == "l2":
734
+ if len(dependencies) > 1:
735
+ raise ValueError(
736
+ f"Unexpected dependencies found for SWAPI L2:"
737
+ f"{dependencies}. Expected only one dependency."
738
+ )
739
+ # process data
740
+ l1_dataset = load_cdf(dependencies[0])
741
+ datasets = [swapi_l2(l1_dataset, self.version)]
742
+
743
+ return datasets
744
+
745
+
746
+ class Swe(ProcessInstrument):
747
+ """Process SWE."""
748
+
749
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
750
+ """
751
+ Perform SWE specific processing.
752
+
753
+ Parameters
754
+ ----------
755
+ dependencies : list
756
+ List of dependencies to process.
757
+
758
+ Returns
759
+ -------
760
+ dataset : xr.Dataset
761
+ Path to cdf file.
762
+ """
763
+ print(f"Processing SWE {self.data_level}")
764
+ datasets: list[xr.Dataset] = []
765
+
766
+ if self.data_level == "l1a":
767
+ if len(dependencies) > 1:
768
+ raise ValueError(
769
+ f"Unexpected dependencies found for SWE L1A:"
770
+ f"{dependencies}. Expected only one dependency."
771
+ )
772
+ datasets = [swe_l1a(str(dependencies[0]), data_version=self.version)]
773
+ # Right now, we only process science data. Therefore,
774
+ # we expect only one dataset to be returned.
775
+
776
+ elif self.data_level == "l1b":
777
+ if len(dependencies) > 1:
778
+ raise ValueError(
779
+ f"Unexpected dependencies found for SWE L1B:"
780
+ f"{dependencies}. Expected only one dependency."
781
+ )
782
+ # read CDF file
783
+ l1a_dataset = load_cdf(dependencies[0])
784
+ datasets = [swe_l1b(l1a_dataset, data_version=self.version)]
785
+ else:
786
+ print("Did not recognize data level. No processing done.")
787
+
788
+ return datasets
789
+
790
+
791
+ class Ultra(ProcessInstrument):
792
+ """Process IMAP-Ultra."""
793
+
794
+ def do_processing(self, dependencies: list) -> list[xr.Dataset]:
795
+ """
796
+ Perform IMAP-Ultra specific processing.
797
+
798
+ Parameters
799
+ ----------
800
+ dependencies : list
801
+ List of dependencies to process.
802
+
803
+ Returns
804
+ -------
805
+ datasets : xr.Dataset
806
+ Xr.Dataset of products.
807
+ """
808
+ print(f"Processing IMAP-Ultra {self.data_level}")
809
+ datasets: list[xr.Dataset] = []
810
+
811
+ if self.data_level == "l1a":
812
+ # File path is expected output file path
813
+ if len(dependencies) > 1:
814
+ raise ValueError(
815
+ f"Unexpected dependencies found for ULTRA L1A:"
816
+ f"{dependencies}. Expected only one dependency."
817
+ )
818
+
819
+ datasets = ultra_l1a.ultra_l1a(dependencies[0], self.version)
820
+
821
+ elif self.data_level == "l1b":
822
+ data_dict = {}
823
+ for dependency in dependencies:
824
+ dataset = load_cdf(dependency)
825
+ data_dict[dataset.attrs["Logical_source"]] = dataset
826
+ datasets = ultra_l1b.ultra_l1b(data_dict, self.version)
827
+
828
+ elif self.data_level == "l1c":
829
+ data_dict = {}
830
+ for dependency in dependencies:
831
+ dataset = load_cdf(dependency)
832
+ data_dict[dataset.attrs["Logical_source"]] = dataset
833
+ datasets = ultra_l1c.ultra_l1c(data_dict, self.version)
834
+
835
+ return datasets
836
+
837
+
838
+ def main() -> None:
839
+ """
840
+ Run the processing for a specific instrument & data level.
841
+
842
+ Set up the command line arguments, parse them, and then invoke the
843
+ appropriate instrument processing function.
844
+ """
845
+ # NOTE: This is to allow the cli script to be installed and reference
846
+ # this function for an entrypoint.
847
+ args = _parse_args()
848
+
849
+ _validate_args(args)
850
+ cls = getattr(sys.modules[__name__], args.instrument.capitalize())
851
+ instrument = cls(
852
+ args.data_level,
853
+ args.dependency,
854
+ args.start_date,
855
+ args.end_date,
856
+ args.version,
857
+ args.upload_to_sdc,
858
+ )
859
+ instrument.process()
860
+
861
+
862
+ if __name__ == "__main__":
863
+ main()