imap-processing 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (272) hide show
  1. imap_processing/__init__.py +1 -0
  2. imap_processing/_version.py +2 -2
  3. imap_processing/ccsds/ccsds_data.py +1 -2
  4. imap_processing/ccsds/excel_to_xtce.py +1 -2
  5. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +18 -12
  6. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +569 -0
  7. imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +1846 -128
  8. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +5 -5
  9. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +20 -1
  10. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +6 -4
  11. imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +3 -3
  12. imap_processing/cdf/config/imap_mag_global_cdf_attrs.yaml +15 -0
  13. imap_processing/cdf/config/imap_swapi_variable_attrs.yaml +22 -0
  14. imap_processing/cdf/config/imap_swe_l1b_variable_attrs.yaml +16 -0
  15. imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +178 -5
  16. imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +5045 -41
  17. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +33 -19
  18. imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +8 -48
  19. imap_processing/cdf/utils.py +41 -33
  20. imap_processing/cli.py +463 -234
  21. imap_processing/codice/codice_l1a.py +260 -47
  22. imap_processing/codice/codice_l1b.py +51 -152
  23. imap_processing/codice/constants.py +38 -1
  24. imap_processing/ena_maps/ena_maps.py +658 -65
  25. imap_processing/ena_maps/utils/coordinates.py +1 -1
  26. imap_processing/ena_maps/utils/spatial_utils.py +10 -5
  27. imap_processing/glows/l1a/glows_l1a.py +28 -99
  28. imap_processing/glows/l1a/glows_l1a_data.py +2 -2
  29. imap_processing/glows/l1b/glows_l1b.py +1 -4
  30. imap_processing/glows/l1b/glows_l1b_data.py +1 -3
  31. imap_processing/glows/l2/glows_l2.py +2 -5
  32. imap_processing/hi/l1a/hi_l1a.py +31 -12
  33. imap_processing/hi/l1b/hi_l1b.py +80 -43
  34. imap_processing/hi/l1c/hi_l1c.py +12 -16
  35. imap_processing/hit/ancillary/imap_hit_l1b-to-l2-sector-dt0-factors_20250219_v002.csv +81 -0
  36. imap_processing/hit/hit_utils.py +93 -35
  37. imap_processing/hit/l0/decom_hit.py +3 -1
  38. imap_processing/hit/l1a/hit_l1a.py +30 -25
  39. imap_processing/hit/l1b/constants.py +6 -2
  40. imap_processing/hit/l1b/hit_l1b.py +279 -318
  41. imap_processing/hit/l2/constants.py +37 -0
  42. imap_processing/hit/l2/hit_l2.py +373 -264
  43. imap_processing/ialirt/l0/parse_mag.py +138 -10
  44. imap_processing/ialirt/l0/process_swapi.py +69 -0
  45. imap_processing/ialirt/l0/process_swe.py +318 -22
  46. imap_processing/ialirt/packet_definitions/ialirt.xml +216 -212
  47. imap_processing/ialirt/packet_definitions/ialirt_codicehi.xml +1 -1
  48. imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +1 -1
  49. imap_processing/ialirt/packet_definitions/ialirt_swapi.xml +14 -14
  50. imap_processing/ialirt/utils/grouping.py +1 -1
  51. imap_processing/idex/idex_constants.py +9 -1
  52. imap_processing/idex/idex_l0.py +22 -8
  53. imap_processing/idex/idex_l1a.py +75 -44
  54. imap_processing/idex/idex_l1b.py +9 -8
  55. imap_processing/idex/idex_l2a.py +79 -45
  56. imap_processing/idex/idex_l2b.py +120 -0
  57. imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -39
  58. imap_processing/idex/packet_definitions/idex_housekeeping_packet_definition.xml +9130 -0
  59. imap_processing/lo/l0/lo_science.py +1 -2
  60. imap_processing/lo/l1a/lo_l1a.py +1 -4
  61. imap_processing/lo/l1b/lo_l1b.py +527 -6
  62. imap_processing/lo/l1b/tof_conversions.py +11 -0
  63. imap_processing/lo/l1c/lo_l1c.py +1 -4
  64. imap_processing/mag/constants.py +43 -0
  65. imap_processing/mag/imap_mag_sdc_configuration_v001.py +8 -0
  66. imap_processing/mag/l1a/mag_l1a.py +2 -9
  67. imap_processing/mag/l1a/mag_l1a_data.py +10 -10
  68. imap_processing/mag/l1b/mag_l1b.py +84 -17
  69. imap_processing/mag/l1c/interpolation_methods.py +180 -3
  70. imap_processing/mag/l1c/mag_l1c.py +236 -70
  71. imap_processing/mag/l2/mag_l2.py +140 -0
  72. imap_processing/mag/l2/mag_l2_data.py +288 -0
  73. imap_processing/spacecraft/quaternions.py +1 -3
  74. imap_processing/spice/geometry.py +3 -3
  75. imap_processing/spice/kernels.py +0 -276
  76. imap_processing/spice/pointing_frame.py +257 -0
  77. imap_processing/spice/repoint.py +48 -19
  78. imap_processing/spice/spin.py +38 -33
  79. imap_processing/spice/time.py +24 -0
  80. imap_processing/swapi/l1/swapi_l1.py +16 -12
  81. imap_processing/swapi/l2/swapi_l2.py +116 -4
  82. imap_processing/swapi/swapi_utils.py +32 -0
  83. imap_processing/swe/l1a/swe_l1a.py +2 -9
  84. imap_processing/swe/l1a/swe_science.py +8 -11
  85. imap_processing/swe/l1b/swe_l1b.py +898 -23
  86. imap_processing/swe/l2/swe_l2.py +21 -77
  87. imap_processing/swe/utils/swe_constants.py +1 -0
  88. imap_processing/tests/ccsds/test_excel_to_xtce.py +1 -1
  89. imap_processing/tests/cdf/test_utils.py +14 -16
  90. imap_processing/tests/codice/conftest.py +44 -33
  91. imap_processing/tests/codice/data/validation/imap_codice_l1a_hi-pha_20241110193700_v0.0.0.cdf +0 -0
  92. imap_processing/tests/codice/data/validation/imap_codice_l1a_lo-pha_20241110193700_v0.0.0.cdf +0 -0
  93. imap_processing/tests/codice/test_codice_l1a.py +20 -11
  94. imap_processing/tests/codice/test_codice_l1b.py +6 -7
  95. imap_processing/tests/conftest.py +78 -22
  96. imap_processing/tests/ena_maps/test_ena_maps.py +462 -33
  97. imap_processing/tests/ena_maps/test_spatial_utils.py +1 -1
  98. imap_processing/tests/glows/conftest.py +10 -14
  99. imap_processing/tests/glows/test_glows_decom.py +4 -4
  100. imap_processing/tests/glows/test_glows_l1a_cdf.py +6 -27
  101. imap_processing/tests/glows/test_glows_l1a_data.py +6 -8
  102. imap_processing/tests/glows/test_glows_l1b.py +11 -11
  103. imap_processing/tests/glows/test_glows_l1b_data.py +5 -5
  104. imap_processing/tests/glows/test_glows_l2.py +2 -8
  105. imap_processing/tests/hi/conftest.py +1 -1
  106. imap_processing/tests/hi/test_hi_l1b.py +10 -12
  107. imap_processing/tests/hi/test_hi_l1c.py +27 -24
  108. imap_processing/tests/hi/test_l1a.py +7 -9
  109. imap_processing/tests/hi/test_science_direct_event.py +2 -2
  110. imap_processing/tests/hit/helpers/l1_validation.py +44 -43
  111. imap_processing/tests/hit/test_decom_hit.py +1 -1
  112. imap_processing/tests/hit/test_hit_l1a.py +9 -9
  113. imap_processing/tests/hit/test_hit_l1b.py +172 -217
  114. imap_processing/tests/hit/test_hit_l2.py +380 -118
  115. imap_processing/tests/hit/test_hit_utils.py +122 -55
  116. imap_processing/tests/hit/validation_data/hit_l1b_standard_sample2_nsrl_v4_3decimals.csv +62 -62
  117. imap_processing/tests/hit/validation_data/sci_sample_raw.csv +1 -1
  118. imap_processing/tests/ialirt/unit/test_decom_ialirt.py +16 -81
  119. imap_processing/tests/ialirt/unit/test_grouping.py +2 -2
  120. imap_processing/tests/ialirt/unit/test_parse_mag.py +71 -16
  121. imap_processing/tests/ialirt/unit/test_process_codicehi.py +3 -3
  122. imap_processing/tests/ialirt/unit/test_process_codicelo.py +3 -10
  123. imap_processing/tests/ialirt/unit/test_process_ephemeris.py +4 -4
  124. imap_processing/tests/ialirt/unit/test_process_hit.py +3 -3
  125. imap_processing/tests/ialirt/unit/test_process_swapi.py +24 -16
  126. imap_processing/tests/ialirt/unit/test_process_swe.py +115 -7
  127. imap_processing/tests/idex/conftest.py +72 -7
  128. imap_processing/tests/idex/test_data/imap_idex_l0_raw_20241206_v001.pkts +0 -0
  129. imap_processing/tests/idex/test_data/imap_idex_l0_raw_20250108_v001.pkts +0 -0
  130. imap_processing/tests/idex/test_idex_l0.py +33 -11
  131. imap_processing/tests/idex/test_idex_l1a.py +50 -23
  132. imap_processing/tests/idex/test_idex_l1b.py +104 -25
  133. imap_processing/tests/idex/test_idex_l2a.py +48 -32
  134. imap_processing/tests/idex/test_idex_l2b.py +93 -0
  135. imap_processing/tests/lo/test_lo_l1a.py +3 -3
  136. imap_processing/tests/lo/test_lo_l1b.py +371 -6
  137. imap_processing/tests/lo/test_lo_l1c.py +1 -1
  138. imap_processing/tests/lo/test_lo_science.py +6 -7
  139. imap_processing/tests/lo/test_star_sensor.py +1 -1
  140. imap_processing/tests/mag/conftest.py +58 -9
  141. imap_processing/tests/mag/test_mag_decom.py +4 -3
  142. imap_processing/tests/mag/test_mag_l1a.py +13 -7
  143. imap_processing/tests/mag/test_mag_l1b.py +9 -9
  144. imap_processing/tests/mag/test_mag_l1c.py +151 -47
  145. imap_processing/tests/mag/test_mag_l2.py +130 -0
  146. imap_processing/tests/mag/test_mag_validation.py +144 -7
  147. imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-magi-normal-in.csv +1217 -0
  148. imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-magi-normal-out.csv +1857 -0
  149. imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-mago-normal-in.csv +1217 -0
  150. imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-mago-normal-out.csv +1857 -0
  151. imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-magi-normal-in.csv +1217 -0
  152. imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-magi-normal-out.csv +1793 -0
  153. imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-mago-normal-in.csv +1217 -0
  154. imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-mago-normal-out.csv +1793 -0
  155. imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-magi-burst-in.csv +2561 -0
  156. imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-magi-normal-in.csv +961 -0
  157. imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-magi-normal-out.csv +1539 -0
  158. imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-mago-normal-in.csv +1921 -0
  159. imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-mago-normal-out.csv +2499 -0
  160. imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-magi-normal-in.csv +865 -0
  161. imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-magi-normal-out.csv +1196 -0
  162. imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-mago-normal-in.csv +1729 -0
  163. imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-mago-normal-out.csv +3053 -0
  164. imap_processing/tests/mag/validation/L2/imap_mag_l1b_norm-mago_20251017_v002.cdf +0 -0
  165. imap_processing/tests/mag/validation/calibration/imap_mag_l2-calibration-matrices_20251017_v004.cdf +0 -0
  166. imap_processing/tests/mag/validation/calibration/imap_mag_l2-offsets-norm_20251017_20251017_v001.cdf +0 -0
  167. imap_processing/tests/spacecraft/test_quaternions.py +1 -1
  168. imap_processing/tests/spice/test_data/fake_repoint_data.csv +4 -4
  169. imap_processing/tests/spice/test_data/fake_spin_data.csv +11 -11
  170. imap_processing/tests/spice/test_geometry.py +3 -3
  171. imap_processing/tests/spice/test_kernels.py +1 -200
  172. imap_processing/tests/spice/test_pointing_frame.py +185 -0
  173. imap_processing/tests/spice/test_repoint.py +20 -10
  174. imap_processing/tests/spice/test_spin.py +50 -9
  175. imap_processing/tests/spice/test_time.py +14 -0
  176. imap_processing/tests/swapi/lut/imap_swapi_esa-unit-conversion_20250211_v000.csv +73 -0
  177. imap_processing/tests/swapi/lut/imap_swapi_lut-notes_20250211_v000.csv +1025 -0
  178. imap_processing/tests/swapi/test_swapi_l1.py +7 -9
  179. imap_processing/tests/swapi/test_swapi_l2.py +180 -8
  180. imap_processing/tests/swe/lut/checker-board-indices.csv +24 -0
  181. imap_processing/tests/swe/lut/imap_swe_esa-lut_20250301_v000.csv +385 -0
  182. imap_processing/tests/swe/lut/imap_swe_l1b-in-flight-cal_20240510_20260716_v000.csv +3 -0
  183. imap_processing/tests/swe/test_swe_l1a.py +6 -6
  184. imap_processing/tests/swe/test_swe_l1a_science.py +3 -3
  185. imap_processing/tests/swe/test_swe_l1b.py +162 -24
  186. imap_processing/tests/swe/test_swe_l2.py +82 -102
  187. imap_processing/tests/test_cli.py +171 -88
  188. imap_processing/tests/test_utils.py +2 -1
  189. imap_processing/tests/ultra/data/mock_data.py +49 -21
  190. imap_processing/tests/ultra/unit/conftest.py +53 -70
  191. imap_processing/tests/ultra/unit/test_badtimes.py +2 -4
  192. imap_processing/tests/ultra/unit/test_cullingmask.py +4 -6
  193. imap_processing/tests/ultra/unit/test_de.py +3 -10
  194. imap_processing/tests/ultra/unit/test_decom_apid_880.py +27 -76
  195. imap_processing/tests/ultra/unit/test_decom_apid_881.py +15 -16
  196. imap_processing/tests/ultra/unit/test_decom_apid_883.py +12 -10
  197. imap_processing/tests/ultra/unit/test_decom_apid_896.py +202 -55
  198. imap_processing/tests/ultra/unit/test_lookup_utils.py +23 -1
  199. imap_processing/tests/ultra/unit/test_spacecraft_pset.py +3 -4
  200. imap_processing/tests/ultra/unit/test_ultra_l1a.py +84 -307
  201. imap_processing/tests/ultra/unit/test_ultra_l1b.py +30 -12
  202. imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +2 -2
  203. imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +4 -1
  204. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +163 -29
  205. imap_processing/tests/ultra/unit/test_ultra_l1c.py +5 -5
  206. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +32 -43
  207. imap_processing/tests/ultra/unit/test_ultra_l2.py +230 -0
  208. imap_processing/ultra/constants.py +1 -1
  209. imap_processing/ultra/l0/decom_tools.py +21 -34
  210. imap_processing/ultra/l0/decom_ultra.py +168 -204
  211. imap_processing/ultra/l0/ultra_utils.py +152 -136
  212. imap_processing/ultra/l1a/ultra_l1a.py +55 -243
  213. imap_processing/ultra/l1b/badtimes.py +1 -4
  214. imap_processing/ultra/l1b/cullingmask.py +2 -6
  215. imap_processing/ultra/l1b/de.py +62 -47
  216. imap_processing/ultra/l1b/extendedspin.py +8 -4
  217. imap_processing/ultra/l1b/lookup_utils.py +72 -9
  218. imap_processing/ultra/l1b/ultra_l1b.py +3 -8
  219. imap_processing/ultra/l1b/ultra_l1b_culling.py +4 -4
  220. imap_processing/ultra/l1b/ultra_l1b_extended.py +236 -78
  221. imap_processing/ultra/l1c/histogram.py +2 -6
  222. imap_processing/ultra/l1c/spacecraft_pset.py +2 -4
  223. imap_processing/ultra/l1c/ultra_l1c.py +1 -5
  224. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +107 -60
  225. imap_processing/ultra/l2/ultra_l2.py +299 -0
  226. imap_processing/ultra/lookup_tables/Angular_Profiles_FM45_LeftSlit.csv +526 -0
  227. imap_processing/ultra/lookup_tables/Angular_Profiles_FM45_RightSlit.csv +526 -0
  228. imap_processing/ultra/lookup_tables/Angular_Profiles_FM90_LeftSlit.csv +526 -0
  229. imap_processing/ultra/lookup_tables/Angular_Profiles_FM90_RightSlit.csv +526 -0
  230. imap_processing/ultra/lookup_tables/FM45_Startup1_ULTRA_IMGPARAMS_20240719.csv +2 -2
  231. imap_processing/ultra/lookup_tables/FM90_Startup1_ULTRA_IMGPARAMS_20240719.csv +2 -0
  232. imap_processing/ultra/packet_definitions/README.md +38 -0
  233. imap_processing/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml +15302 -482
  234. imap_processing/ultra/utils/ultra_l1_utils.py +13 -12
  235. imap_processing/utils.py +1 -1
  236. {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/METADATA +3 -2
  237. {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/RECORD +264 -225
  238. imap_processing/hi/l1b/hi_eng_unit_convert_table.csv +0 -154
  239. imap_processing/mag/imap_mag_sdc-configuration_v001.yaml +0 -6
  240. imap_processing/mag/l1b/__init__.py +0 -0
  241. imap_processing/swe/l1b/swe_esa_lookup_table.csv +0 -1441
  242. imap_processing/swe/l1b/swe_l1b_science.py +0 -699
  243. imap_processing/tests/swe/test_swe_l1b_science.py +0 -103
  244. imap_processing/ultra/lookup_tables/dps_sensitivity45.cdf +0 -0
  245. imap_processing/ultra/lookup_tables/ultra_90_dps_exposure_compressed.cdf +0 -0
  246. /imap_processing/idex/packet_definitions/{idex_packet_definition.xml → idex_science_packet_definition.xml} +0 -0
  247. /imap_processing/tests/ialirt/{test_data → data}/l0/20240827095047_SWE_IALIRT_packet.bin +0 -0
  248. /imap_processing/tests/ialirt/{test_data → data}/l0/461971383-404.bin +0 -0
  249. /imap_processing/tests/ialirt/{test_data → data}/l0/461971384-405.bin +0 -0
  250. /imap_processing/tests/ialirt/{test_data → data}/l0/461971385-406.bin +0 -0
  251. /imap_processing/tests/ialirt/{test_data → data}/l0/461971386-407.bin +0 -0
  252. /imap_processing/tests/ialirt/{test_data → data}/l0/461971387-408.bin +0 -0
  253. /imap_processing/tests/ialirt/{test_data → data}/l0/461971388-409.bin +0 -0
  254. /imap_processing/tests/ialirt/{test_data → data}/l0/461971389-410.bin +0 -0
  255. /imap_processing/tests/ialirt/{test_data → data}/l0/461971390-411.bin +0 -0
  256. /imap_processing/tests/ialirt/{test_data → data}/l0/461971391-412.bin +0 -0
  257. /imap_processing/tests/ialirt/{test_data → data}/l0/BinLog CCSDS_FRAG_TLM_20240826_152323Z_IALIRT_data_for_SDC.bin +0 -0
  258. /imap_processing/tests/ialirt/{test_data → data}/l0/IALiRT Raw Packet Telemetry.txt +0 -0
  259. /imap_processing/tests/ialirt/{test_data → data}/l0/apid01152.tlm +0 -0
  260. /imap_processing/tests/ialirt/{test_data → data}/l0/eu_SWP_IAL_20240826_152033.csv +0 -0
  261. /imap_processing/tests/ialirt/{test_data → data}/l0/hi_fsw_view_1_ccsds.bin +0 -0
  262. /imap_processing/tests/ialirt/{test_data → data}/l0/hit_ialirt_sample.ccsds +0 -0
  263. /imap_processing/tests/ialirt/{test_data → data}/l0/hit_ialirt_sample.csv +0 -0
  264. /imap_processing/tests/ialirt/{test_data → data}/l0/idle_export_eu.SWE_IALIRT_20240827_093852.csv +0 -0
  265. /imap_processing/tests/ialirt/{test_data → data}/l0/imap_codice_l1a_hi-ialirt_20240523200000_v0.0.0.cdf +0 -0
  266. /imap_processing/tests/ialirt/{test_data → data}/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
  267. /imap_processing/tests/ialirt/{test_data → data}/l0/sample_decoded_i-alirt_data.csv +0 -0
  268. /imap_processing/tests/mag/validation/{imap_calibration_mag_20240229_v01.cdf → calibration/imap_mag_l1b-calibration_20240229_v001.cdf} +0 -0
  269. /imap_processing/{swe/l1b/engineering_unit_convert_table.csv → tests/swe/lut/imap_swe_eu-conversion_20240510_v000.csv} +0 -0
  270. {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/LICENSE +0 -0
  271. {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/WHEEL +0 -0
  272. {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/entry_points.txt +0 -0
@@ -2,14 +2,19 @@
2
2
 
3
3
  import logging
4
4
  from pathlib import Path
5
- from typing import Union
5
+ from typing import Any, Union
6
+
7
+ from xarray import Dataset
6
8
 
7
9
  from imap_processing import decom, imap_module_directory
10
+ from imap_processing.utils import packet_file_to_datasets
8
11
 
9
12
  logger = logging.getLogger(__name__)
10
13
 
11
14
 
12
- def decom_packets(packet_file: Union[str, Path]) -> list:
15
+ def decom_packets(
16
+ packet_file: Union[str, Path],
17
+ ) -> tuple[list[Any], dict[int, Dataset]]:
13
18
  """
14
19
  Decom IDEX data packets using IDEX packet definition.
15
20
 
@@ -20,12 +25,21 @@ def decom_packets(packet_file: Union[str, Path]) -> list:
20
25
 
21
26
  Returns
22
27
  -------
23
- list
24
- All the unpacked data.
28
+ Tuple[list, dict]
29
+ Returns a list of all unpacked science data and a dictionary of datasets
30
+ indexed by their APIDs.
31
+
32
+ Notes
33
+ -----
34
+ The function 'packet_file_to_dataset' does not work with IDEX science packets due to
35
+ branching logic within the science xml file. The science data and housekeeping data
36
+ will be decommed separately and both returned from this function.
25
37
  """
26
- xtce_filename = "idex_packet_definition.xml"
27
- xtce_file = f"{imap_module_directory}/idex/packet_definitions/{xtce_filename}"
38
+ xtce_base_path = f"{imap_module_directory}/idex/packet_definitions"
39
+ science_xtce_file = f"{xtce_base_path}/idex_science_packet_definition.xml"
40
+ hk_xtce_file = f"{xtce_base_path}/idex_housekeeping_packet_definition.xml"
28
41
 
29
- decom_packet_list = decom.decom_packets(packet_file, xtce_file)
42
+ science_decom_packet_list = decom.decom_packets(packet_file, science_xtce_file)
43
+ datasets_by_apid = packet_file_to_datasets(packet_file, hk_xtce_file)
30
44
 
31
- return list(decom_packet_list)
45
+ return list(science_decom_packet_list), datasets_by_apid
@@ -10,7 +10,7 @@ Examples
10
10
  from imap_processing.idex.idex_l1a import PacketParser
11
11
 
12
12
  l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
13
- l1a_data = PacketParser(l0_file, data_version)
13
+ l1a_data = PacketParser(l0_file)
14
14
  l1a_data.write_l1a_cdf()
15
15
  """
16
16
 
@@ -26,6 +26,7 @@ import xarray as xr
26
26
 
27
27
  from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
28
28
  from imap_processing.idex.decode import rice_decode
29
+ from imap_processing.idex.idex_constants import IDEXAPID
29
30
  from imap_processing.idex.idex_l0 import decom_packets
30
31
  from imap_processing.spice.time import met_to_ttj2000ns
31
32
  from imap_processing.utils import convert_to_binary_string
@@ -57,11 +58,9 @@ class PacketParser:
57
58
  ----------
58
59
  packet_file : str
59
60
  The path and filename to the L0 file to read.
60
- data_version : str
61
- The version of the data product being created.
62
61
  """
63
62
 
64
- def __init__(self, packet_file: Union[str, Path], data_version: str) -> None:
63
+ def __init__(self, packet_file: Union[str, Path]) -> None:
65
64
  """
66
65
  Read a L0 pkts file and perform all of the decom work.
67
66
 
@@ -69,23 +68,62 @@ class PacketParser:
69
68
  ----------
70
69
  packet_file : pathlib.Path | str
71
70
  The path and filename to the L0 file to read.
72
- data_version : str
73
- The version of the data product being created.
74
71
 
75
72
  Notes
76
73
  -----
77
74
  Currently assumes one L0 file will generate exactly one L1a file.
78
75
  """
79
- decom_packet_list = decom_packets(packet_file)
76
+ self.data = []
77
+ self.idex_attrs = get_idex_attrs()
78
+ epoch_attrs = self.idex_attrs.get_variable_attributes(
79
+ "epoch", check_schema=False
80
+ )
81
+
82
+ science_packets, datset_by_apid = decom_packets(packet_file)
83
+
84
+ if science_packets:
85
+ logger.info("Processing IDEX L1A Science data.")
86
+ self.data.append(self._create_science_dataset(science_packets))
87
+
88
+ if IDEXAPID.IDEX_EVT in datset_by_apid:
89
+ logger.info("Processing IDEX L1A Event Message data.")
90
+ data = datset_by_apid[IDEXAPID.IDEX_EVT]
91
+ data.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_evt")
92
+ data["epoch"].attrs = epoch_attrs
93
+ self.data.append(data)
94
+
95
+ if IDEXAPID.IDEX_CATLST in datset_by_apid:
96
+ logger.info("Processing IDEX L1A Catalog List Summary data.")
97
+ data = datset_by_apid[IDEXAPID.IDEX_CATLST]
98
+ data.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_catlst")
99
+ data["epoch"].attrs = epoch_attrs
100
+ self.data.append(data)
101
+
102
+ logger.info("IDEX L1A data processing completed.")
103
+
104
+ def _create_science_dataset(self, science_decom_packet_list: list) -> xr.Dataset:
105
+ """
106
+ Process IDEX science packets into an xarray Dataset.
107
+
108
+ Parameters
109
+ ----------
110
+ science_decom_packet_list : list
111
+ List of decommutated science packets.
112
+
113
+ Returns
114
+ -------
115
+ xarray.Dataset
116
+ Dataset containing processed dust events.
117
+ """
80
118
  dust_events = {}
81
- for packet in decom_packet_list:
119
+ for packet in science_decom_packet_list:
82
120
  if "IDX__SCI0TYPE" in packet:
83
121
  scitype = packet["IDX__SCI0TYPE"]
84
122
  event_number = packet["IDX__SCI0EVTNUM"]
85
123
  if scitype == Scitype.FIRST_PACKET:
86
124
  # Initial packet for new dust event
87
125
  # Further packets will fill in data
88
- dust_events[event_number] = RawDustEvent(packet, data_version)
126
+ dust_events[event_number] = RawDustEvent(packet)
89
127
  elif event_number not in dust_events:
90
128
  raise KeyError(
91
129
  f"Have not receive header information from event number\
@@ -101,40 +139,43 @@ class PacketParser:
101
139
  dust_event.process() for dust_event in dust_events.values()
102
140
  ]
103
141
 
104
- self.data = xr.concat(processed_dust_impact_list, dim="epoch")
105
- idex_attrs = get_idex_attrs(data_version)
106
- self.data.attrs = idex_attrs.get_global_attributes("imap_idex_l1a_sci")
142
+ data = xr.concat(processed_dust_impact_list, dim="epoch")
143
+ data.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_sci")
107
144
 
108
145
  # Add high and low sample rate coords
109
- self.data["time_low_sample_rate_index"] = xr.DataArray(
110
- np.arange(len(self.data["time_low_sample_rate"][0])),
146
+ data["time_low_sample_rate_index"] = xr.DataArray(
147
+ np.arange(len(data["time_low_sample_rate"][0])),
111
148
  name="time_low_sample_rate_index",
112
149
  dims=["time_low_sample_rate_index"],
113
- attrs=idex_attrs.get_variable_attributes("time_low_sample_rate_index"),
150
+ attrs=self.idex_attrs.get_variable_attributes("time_low_sample_rate_index"),
114
151
  )
115
152
 
116
- self.data["time_high_sample_rate_index"] = xr.DataArray(
117
- np.arange(len(self.data["time_high_sample_rate"][0])),
153
+ data["time_high_sample_rate_index"] = xr.DataArray(
154
+ np.arange(len(data["time_high_sample_rate"][0])),
118
155
  name="time_high_sample_rate_index",
119
156
  dims=["time_high_sample_rate_index"],
120
- attrs=idex_attrs.get_variable_attributes("time_high_sample_rate_index"),
157
+ attrs=self.idex_attrs.get_variable_attributes(
158
+ "time_high_sample_rate_index"
159
+ ),
121
160
  )
122
161
  # NOTE: LABL_PTR_1 should be CDF_CHAR.
123
- self.data["time_low_sample_rate_label"] = xr.DataArray(
124
- self.data.time_low_sample_rate_index.values.astype(str),
162
+ data["time_low_sample_rate_label"] = xr.DataArray(
163
+ data.time_low_sample_rate_index.values.astype(str),
125
164
  name="time_low_sample_rate_label",
126
165
  dims=["time_low_sample_rate_index"],
127
- attrs=idex_attrs.get_variable_attributes("time_low_sample_rate_label"),
166
+ attrs=self.idex_attrs.get_variable_attributes("time_low_sample_rate_label"),
128
167
  )
129
168
 
130
- self.data["time_high_sample_rate_label"] = xr.DataArray(
131
- self.data.time_high_sample_rate_index.values.astype(str),
169
+ data["time_high_sample_rate_label"] = xr.DataArray(
170
+ data.time_high_sample_rate_index.values.astype(str),
132
171
  name="time_high_sample_rate_label",
133
172
  dims=["time_high_sample_rate_index"],
134
- attrs=idex_attrs.get_variable_attributes("time_high_sample_rate_label"),
173
+ attrs=self.idex_attrs.get_variable_attributes(
174
+ "time_high_sample_rate_label"
175
+ ),
135
176
  )
136
177
 
137
- logger.info("IDEX L1A science data processing completed.")
178
+ return data
138
179
 
139
180
 
140
181
  def _read_waveform_bits(waveform_raw: str, high_sample: bool = True) -> list[int]:
@@ -198,8 +239,6 @@ class RawDustEvent:
198
239
  ----------
199
240
  header_packet : space_packet_parser.packets.CCSDSPacket
200
241
  The FPGA metadata event header.
201
- data_version : str
202
- The version of the data product being created.
203
242
 
204
243
  Attributes
205
244
  ----------
@@ -252,9 +291,7 @@ class RawDustEvent:
252
291
  MAX_HIGH_BLOCKS = 16
253
292
  MAX_LOW_BLOCKS = 64
254
293
 
255
- def __init__(
256
- self, header_packet: space_packet_parser.packets.CCSDSPacket, data_version: str
257
- ) -> None:
294
+ def __init__(self, header_packet: space_packet_parser.packets.CCSDSPacket) -> None:
258
295
  """
259
296
  Initialize a raw dust event, with an FPGA Header Packet from IDEX.
260
297
 
@@ -268,8 +305,6 @@ class RawDustEvent:
268
305
  ----------
269
306
  header_packet : space_packet_parser.packets.CCSDSPacket
270
307
  The FPGA metadata event header.
271
- data_version : str
272
- Data version for CDF filename, in the format ``vXXX``.
273
308
  """
274
309
  # Calculate the impact time in seconds since epoch
275
310
  self.impact_time = 0
@@ -301,7 +336,7 @@ class RawDustEvent:
301
336
  self.Ion_Grid_bits = ""
302
337
 
303
338
  self.compressed = self.telemetry_items["idx__sci0comp"]
304
- self.cdf_attrs = get_idex_attrs(data_version)
339
+ self.cdf_attrs = get_idex_attrs()
305
340
 
306
341
  def _append_raw_data(self, scitype: Scitype, bits: str) -> None:
307
342
  """
@@ -445,11 +480,12 @@ class RawDustEvent:
445
480
  List of the high sample waveform.
446
481
  """
447
482
  samples = self.MAX_HIGH_BLOCKS * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK
483
+ ints: list[int] = []
448
484
  if self.compressed.raw_value == 1:
449
- ints = rice_decode(waveform_raw, nbit10=True, sample_count=samples)
485
+ ints.extend(rice_decode(waveform_raw, nbit10=True, sample_count=samples))
450
486
  ints = ints[:-3]
451
487
  else:
452
- ints = _read_waveform_bits(waveform_raw, high_sample=True)
488
+ ints.extend(_read_waveform_bits(waveform_raw, high_sample=True))
453
489
  return ints
454
490
 
455
491
  def _parse_low_sample_waveform(self, waveform_raw: str) -> list[int]:
@@ -470,10 +506,11 @@ class RawDustEvent:
470
506
  List of processed low sample waveform.
471
507
  """
472
508
  samples = self.MAX_LOW_BLOCKS * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK
509
+ ints: list[int] = []
473
510
  if self.compressed.raw_value == 1:
474
- ints = rice_decode(waveform_raw, nbit10=False, sample_count=samples)
511
+ ints.extend(rice_decode(waveform_raw, nbit10=False, sample_count=samples))
475
512
  else:
476
- ints = _read_waveform_bits(waveform_raw, high_sample=False)
513
+ ints.extend(_read_waveform_bits(waveform_raw, high_sample=False))
477
514
  return ints
478
515
 
479
516
  def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray:
@@ -651,15 +688,10 @@ class RawDustEvent:
651
688
  return dataset
652
689
 
653
690
 
654
- def get_idex_attrs(data_version: str) -> ImapCdfAttributes:
691
+ def get_idex_attrs() -> ImapCdfAttributes:
655
692
  """
656
693
  Load in CDF attributes for IDEX instrument.
657
694
 
658
- Parameters
659
- ----------
660
- data_version : str
661
- Data version for CDF filename, in the format "vXXX".
662
-
663
695
  Returns
664
696
  -------
665
697
  idex_attrs : ImapCdfAttributes
@@ -668,5 +700,4 @@ def get_idex_attrs(data_version: str) -> ImapCdfAttributes:
668
700
  idex_attrs = ImapCdfAttributes()
669
701
  idex_attrs.add_instrument_global_attrs("idex")
670
702
  idex_attrs.add_instrument_variable_attrs("idex", "l1a")
671
- idex_attrs.add_global_attribute("Data_version", data_version)
672
703
  return idex_attrs
@@ -9,8 +9,8 @@ Examples
9
9
  from imap_processing.idex.idex_l1b import idex_l1b
10
10
 
11
11
  l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
12
- l1a_data = PacketParser(l0_file, data_version)
13
- l1b_data = idex_l1b(l1a_data, data_version)
12
+ l1a_data = PacketParser(l0_file)
13
+ l1b_data = idex_l1b(l1a_data)
14
14
  write_cdf(l1b_data)
15
15
  """
16
16
 
@@ -77,7 +77,7 @@ class TriggerMode(Enum):
77
77
  return f"{channel.upper()}{TriggerMode(mode).name}"
78
78
 
79
79
 
80
- def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
80
+ def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset:
81
81
  """
82
82
  Will process IDEX l1a data to create l1b data products.
83
83
 
@@ -85,8 +85,6 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
85
85
  ----------
86
86
  l1a_dataset : xarray.Dataset
87
87
  IDEX L1a dataset to process.
88
- data_version : str
89
- Version of the data product being created.
90
88
 
91
89
  Returns
92
90
  -------
@@ -101,7 +99,6 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
101
99
  idex_attrs = ImapCdfAttributes()
102
100
  idex_attrs.add_instrument_global_attrs(instrument="idex")
103
101
  idex_attrs.add_instrument_variable_attrs(instrument="idex", level="l1b")
104
- idex_attrs.add_global_attribute("Data_version", data_version)
105
102
 
106
103
  var_information_path = (
107
104
  f"{imap_module_directory}/idex/idex_variable_unpacking_and_eu_conversion.csv"
@@ -295,10 +292,14 @@ def get_trigger_mode_and_level(
295
292
  # Bit-shift right 22 places and use a 10-bit mask to extract the level value.
296
293
  threshold_level = float((trigger_controls >> 22) & mask)
297
294
 
298
- # If it is the high gain channel multiply the level by the conversion factor.
299
- # TODO: determine why the idex team is only doing this for the high gain channel
295
+ # multiply the threshold level by the conversion factor.
300
296
  if gain_channel == "hg":
301
297
  threshold_level *= ConversionFactors["TOF_High"]
298
+ elif gain_channel == "mg":
299
+ threshold_level *= ConversionFactors["TOF_Mid"]
300
+ elif gain_channel == "lg":
301
+ threshold_level *= ConversionFactors["TOF_Low"]
302
+
302
303
  return mode_label, threshold_level
303
304
 
304
305
  for channel in channels:
@@ -10,16 +10,14 @@ Examples
10
10
  from imap_processing.idex.idex_l2a import idex_l2a
11
11
 
12
12
  l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
13
- l1a_data = PacketParser(l0_file, data_version)
14
- l1b_data = idex_l1b(l1a_data, data_version)
15
- l2a_data = idex_l2a(l1b_data, data_version)
13
+ l1a_data = PacketParser(l0_file)
14
+ l1b_data = idex_l1b(l1a_data)
15
+ l2a_data = idex_l2a(l1b_data)
16
16
  write_cdf(l2a_data)
17
17
  """
18
18
 
19
- # ruff: noqa: PLR0913
20
19
  import logging
21
20
  from enum import IntEnum
22
- from typing import Union
23
21
 
24
22
  import numpy as np
25
23
  import pandas as pd
@@ -32,7 +30,6 @@ from scipy.stats import exponnorm
32
30
 
33
31
  from imap_processing import imap_module_directory
34
32
  from imap_processing.idex import idex_constants
35
- from imap_processing.idex.idex_constants import ConversionFactors
36
33
  from imap_processing.idex.idex_l1a import get_idex_attrs
37
34
 
38
35
  logger = logging.getLogger(__name__)
@@ -54,7 +51,7 @@ class BaselineNoiseTime(IntEnum):
54
51
  STOP = -5
55
52
 
56
53
 
57
- def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
54
+ def idex_l2a(l1b_dataset: xr.Dataset) -> xr.Dataset:
58
55
  """
59
56
  Will process IDEX l1b data to create l2a data products.
60
57
 
@@ -70,8 +67,6 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
70
67
  ----------
71
68
  l1b_dataset : xarray.Dataset
72
69
  IDEX L1a dataset to process.
73
- data_version : str
74
- Version of the data product being created.
75
70
 
76
71
  Returns
77
72
  -------
@@ -104,7 +99,7 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
104
99
  kappa = calculate_kappa(mass_scales, peaks_2d)
105
100
 
106
101
  # Analyze peaks for estimating dust composition
107
- peak_fits, area_under_fits = xr.apply_ufunc(
102
+ peak_fits_params, area_under_fits, fit_chisqr, fit_redchi = xr.apply_ufunc(
108
103
  analyze_peaks,
109
104
  tof_high,
110
105
  hs_time,
@@ -117,10 +112,11 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
117
112
  ["time_high_sample_rate_index"],
118
113
  [],
119
114
  ],
120
- # TODO: Determine dimension name
121
115
  output_core_dims=[
122
- ["time_of_flight", "peak_fit_parameters"],
123
- ["time_of_flight"],
116
+ ["mass", "peak_fit_parameters"],
117
+ ["mass"],
118
+ [],
119
+ [],
124
120
  ],
125
121
  vectorize=True,
126
122
  )
@@ -128,13 +124,11 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
128
124
  l2a_dataset = l1b_dataset.copy()
129
125
 
130
126
  for waveform in ["Target_Low", "Target_High", "Ion_Grid"]:
131
- # Convert back to raw DNs for more accurate fits
132
- waveform_dn = l1b_dataset[waveform] / ConversionFactors[waveform]
133
127
  # Get the dust mass estimates and fit results
134
128
  fit_results = xr.apply_ufunc(
135
129
  estimate_dust_mass,
136
130
  ls_time,
137
- waveform_dn,
131
+ l1b_dataset[waveform],
138
132
  input_core_dims=[
139
133
  ["time_low_sample_rate_index"],
140
134
  ["time_low_sample_rate_index"],
@@ -152,20 +146,23 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
152
146
  waveform_name = waveform.lower()
153
147
  # Add variables
154
148
  l2a_dataset[f"{waveform_name}_fit_parameters"] = fit_results[0]
155
- l2a_dataset[f"{waveform_name}_fit_imapct_charge"] = fit_results[1]
149
+ l2a_dataset[f"{waveform_name}_fit_impact_charge"] = fit_results[1]
156
150
  # TODO: convert charge to mass
157
- l2a_dataset[f"{waveform_name}_fit_imapct_mass_estimate"] = fit_results[1]
151
+ l2a_dataset[f"{waveform_name}_fit_impact_mass_estimate"] = fit_results[1]
158
152
  l2a_dataset[f"{waveform_name}_chi_squared"] = fit_results[2]
159
153
  l2a_dataset[f"{waveform_name}_reduced_chi_squared"] = fit_results[3]
160
154
  l2a_dataset[f"{waveform_name}_fit_results"] = fit_results[4]
161
155
 
162
- l2a_dataset["tof_peak_fit_parameters"] = peak_fits
156
+ l2a_dataset["tof_peak_fit_parameters"] = peak_fits_params
163
157
  l2a_dataset["tof_peak_area_under_fit"] = area_under_fits
158
+ l2a_dataset["tof_peak_chi_square"] = fit_chisqr
159
+ l2a_dataset["tof_peak_reduced_chi_square"] = fit_redchi
160
+
164
161
  l2a_dataset["tof_peak_kappa"] = xr.DataArray(kappa, dims=["epoch"])
165
162
  l2a_dataset["tof_snr"] = xr.DataArray(snr, dims=["epoch"])
166
163
  l2a_dataset["mass"] = mass_scales_da
167
164
  # Update global attributes
168
- idex_attrs = get_idex_attrs(data_version)
165
+ idex_attrs = get_idex_attrs()
169
166
  l2a_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l2a_sci")
170
167
 
171
168
  logger.info("IDEX L2A science data processing completed.")
@@ -361,7 +358,7 @@ def analyze_peaks(
361
358
  mass_scale: xr.DataArray,
362
359
  event_num: int,
363
360
  peaks_2d: np.ndarray,
364
- ) -> tuple[NDArray, NDArray]:
361
+ ) -> tuple[NDArray, NDArray, float, float]:
365
362
  """
366
363
  Fit an EMG curve to the Time of Flight data around each peak.
367
364
 
@@ -404,8 +401,8 @@ def analyze_peaks(
404
401
  time_slice = high_sampling_time[start:end]
405
402
  tof_slice = tof_high[start:end]
406
403
 
407
- param = fit_emg(time_slice, tof_slice, event_num)
408
- if param is None:
404
+ param, chisqr, redchi = fit_emg(time_slice, tof_slice, event_num)
405
+ if np.all(np.isnan(param)):
409
406
  continue
410
407
 
411
408
  area = calculate_area_under_emg(time_slice, param)
@@ -427,7 +424,6 @@ def analyze_peaks(
427
424
  mass = max(0, round(mass))
428
425
  # Find the first index with non-zero fit parameters, starting from current mass
429
426
  non_zero_idxs = np.nonzero(np.all(fit_params[mass:] != 0, axis=-1))[0]
430
-
431
427
  # Determine index to use
432
428
  # If no non-zero parameters found, use current mass index
433
429
  # Otherwise, use the current mass plus offset to first non-zero index
@@ -437,16 +433,14 @@ def analyze_peaks(
437
433
  fit_params[idx] = np.array([mu, sigma, lam])
438
434
  area_under_emg[idx] = area
439
435
  else:
440
- logger.warning(
441
- f"Unable to find a slot for mass: {mass}. Discarding " f"value."
442
- )
436
+ logger.warning(f"Unable to find a slot for mass: {mass}. Discarding value.")
443
437
 
444
- return fit_params, area_under_emg
438
+ return fit_params, area_under_emg, chisqr, redchi
445
439
 
446
440
 
447
441
  def fit_emg(
448
442
  peak_time: np.ndarray, peak_signal: np.ndarray, event_num: int
449
- ) -> Union[NDArray, None]:
443
+ ) -> tuple[NDArray, float, float]:
450
444
  """
451
445
  Fit an exponentially modified gaussian function to the peak signal.
452
446
 
@@ -465,9 +459,13 @@ def fit_emg(
465
459
 
466
460
  Returns
467
461
  -------
468
- param : numpy.ndarray or None
462
+ param : numpy.ndarray
469
463
  Fitted EMG optimal values for the parameters (popt) [k (shape parameter), mu,
470
- sigma] if fit successful, None otherwise.
464
+ sigma] if fit is successful, array of np.nans otherwise.
465
+ chisqr : float
466
+ Chi-square value if fit is successful, np.nan otherwise.
467
+ redchi : float
468
+ Reduced chi-square value if fit is successful, np.nan otherwise.
471
469
  """
472
470
  # Initial Guess for the parameters of the emg fit:
473
471
  # center of gaussian
@@ -490,11 +488,14 @@ def fit_emg(
490
488
  f"Time range: {peak_time[0]:.2f} to {peak_time[-1]:.2f}\n"
491
489
  f"Signal range: {min(peak_signal):.2f} to {max(peak_signal):.2f}\n"
492
490
  f"Event number: {event_num}\n"
493
- "Returning None."
491
+ "Returning np.nan values."
494
492
  )
495
- return None
493
+ return np.full(len(p0), np.nan), np.nan, np.nan
494
+
495
+ emg_fit = exponnorm.pdf(peak_time, *param)
496
+ chisqr, redchi = chi_square(peak_signal, emg_fit, len(p0))
496
497
 
497
- return param
498
+ return param, chisqr, redchi
498
499
 
499
500
 
500
501
  def calculate_area_under_emg(time_slice: np.ndarray, param: np.ndarray) -> float:
@@ -555,8 +556,6 @@ def estimate_dust_mass(
555
556
  result : numpy.ndarray
556
557
  The model values evaluated at each time point.
557
558
  """
558
- # TODO: The IDEX team is iterating on this Function and will provide more
559
- # information soon.
560
559
  signal = np.array(target_signal.data)
561
560
  time = np.array(low_sampling_time.data)
562
561
  good_mask = np.logical_and(
@@ -615,11 +614,7 @@ def estimate_dust_mass(
615
614
  impact_fit = fit_impact(time, *param)
616
615
  # Calculate the resulting signal amplitude after removing baseline noise
617
616
  sig_amp = max(impact_fit) - np.mean(signal_baseline)
618
-
619
- # Calculate chi square and reduced chi square
620
- chisqr = float(np.sum((signal - impact_fit) ** 2))
621
- # To get reduced chi square divide by dof (number of points - number of params)
622
- redchi = chisqr / (len(signal) - len(p0))
617
+ chisqr, redchi = chi_square(signal, impact_fit, len(p0))
623
618
 
624
619
  return param, float(sig_amp), chisqr, redchi, impact_fit
625
620
 
@@ -729,7 +724,7 @@ def remove_signal_noise(
729
724
 
730
725
  def sine_fit(time: np.ndarray, a: float, f: float, p: float) -> NDArray:
731
726
  """
732
- Generate a sine wave with given amplitude, frequency, and phase.
727
+ Generate a sine wave with given amplitude, angular frequency, and phase.
733
728
 
734
729
  Parameters
735
730
  ----------
@@ -738,7 +733,7 @@ def sine_fit(time: np.ndarray, a: float, f: float, p: float) -> NDArray:
738
733
  a : float
739
734
  Amplitude of the sine wave.
740
735
  f : float
741
- Frequency of the sine wave in Hz.
736
+ Angular frequency of the sine wave.
742
737
  p : float
743
738
  Phase shift of the sine wave in radians.
744
739
 
@@ -747,7 +742,7 @@ def sine_fit(time: np.ndarray, a: float, f: float, p: float) -> NDArray:
747
742
  numpy.ndarray
748
743
  Sine wave values calculated at the input time points.
749
744
  """
750
- return a * np.sin(2 * np.pi * f * time + p)
745
+ return a * np.sin(f * time + p)
751
746
 
752
747
 
753
748
  def butter_lowpass_filter(
@@ -772,7 +767,6 @@ def butter_lowpass_filter(
772
767
  numpy.ndarray
773
768
  Filtered signal.
774
769
  """
775
- # TODO: The IDEX team might be switching this function out for a different filter.
776
770
  sample_period = time[1] - time[0]
777
771
  # sampling frequency
778
772
  fs = (time[-1] - time[0]) / sample_period # Hz
@@ -787,3 +781,43 @@ def butter_lowpass_filter(
787
781
  b, a = butter(order, normal_cutoff, btype="low", analog=False)
788
782
  y = filtfilt(b, a, signal)
789
783
  return y
784
+
785
+
786
+ def chi_square(
787
+ observed: np.ndarray, expected: np.ndarray, num_params: int
788
+ ) -> tuple[float, float]:
789
+ """
790
+ Calculate the chi-square and reduced chi-square statistics.
791
+
792
+ This implementation follows the approach used in lmfit.minimize()'s
793
+ _calculate_statistics() method, which calculates chi-square as the sum of squared
794
+ residuals:
795
+
796
+ chisqr = (residual**2).sum()
797
+
798
+ And reduced chi-square as the chi-square divided by degrees of freedom:
799
+
800
+ ndata = len(residual)
801
+ nfree = ndata - number_of_parameters
802
+ redchi = chisqr / max(1, nfree)
803
+
804
+ Parameters
805
+ ----------
806
+ observed : numpy.ndarray
807
+ The observed signal.
808
+ expected : numpy.ndarray
809
+ The expected signal calculated with the fit parameters.
810
+ num_params : int
811
+ The number of parameters used in the fit.
812
+
813
+ Returns
814
+ -------
815
+ chisqr : float
816
+ The chi-square value.
817
+ redchi : float
818
+ The reduced chi-square value.
819
+ """
820
+ residuals = observed - expected
821
+ chisqr = float(np.sum(residuals**2))
822
+ redchi = chisqr / max(1, (len(observed) - num_params))
823
+ return chisqr, redchi