imap-processing 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imap-processing might be problematic. Click here for more details.
- imap_processing/__init__.py +1 -0
- imap_processing/_version.py +2 -2
- imap_processing/ccsds/ccsds_data.py +1 -2
- imap_processing/ccsds/excel_to_xtce.py +1 -2
- imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +18 -12
- imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +569 -0
- imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +1846 -128
- imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +5 -5
- imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +20 -1
- imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +6 -4
- imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +3 -3
- imap_processing/cdf/config/imap_mag_global_cdf_attrs.yaml +15 -0
- imap_processing/cdf/config/imap_swapi_variable_attrs.yaml +22 -0
- imap_processing/cdf/config/imap_swe_l1b_variable_attrs.yaml +16 -0
- imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +178 -5
- imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +5045 -41
- imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +33 -19
- imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +8 -48
- imap_processing/cdf/utils.py +41 -33
- imap_processing/cli.py +463 -234
- imap_processing/codice/codice_l1a.py +260 -47
- imap_processing/codice/codice_l1b.py +51 -152
- imap_processing/codice/constants.py +38 -1
- imap_processing/ena_maps/ena_maps.py +658 -65
- imap_processing/ena_maps/utils/coordinates.py +1 -1
- imap_processing/ena_maps/utils/spatial_utils.py +10 -5
- imap_processing/glows/l1a/glows_l1a.py +28 -99
- imap_processing/glows/l1a/glows_l1a_data.py +2 -2
- imap_processing/glows/l1b/glows_l1b.py +1 -4
- imap_processing/glows/l1b/glows_l1b_data.py +1 -3
- imap_processing/glows/l2/glows_l2.py +2 -5
- imap_processing/hi/l1a/hi_l1a.py +31 -12
- imap_processing/hi/l1b/hi_l1b.py +80 -43
- imap_processing/hi/l1c/hi_l1c.py +12 -16
- imap_processing/hit/ancillary/imap_hit_l1b-to-l2-sector-dt0-factors_20250219_v002.csv +81 -0
- imap_processing/hit/hit_utils.py +93 -35
- imap_processing/hit/l0/decom_hit.py +3 -1
- imap_processing/hit/l1a/hit_l1a.py +30 -25
- imap_processing/hit/l1b/constants.py +6 -2
- imap_processing/hit/l1b/hit_l1b.py +279 -318
- imap_processing/hit/l2/constants.py +37 -0
- imap_processing/hit/l2/hit_l2.py +373 -264
- imap_processing/ialirt/l0/parse_mag.py +138 -10
- imap_processing/ialirt/l0/process_swapi.py +69 -0
- imap_processing/ialirt/l0/process_swe.py +318 -22
- imap_processing/ialirt/packet_definitions/ialirt.xml +216 -212
- imap_processing/ialirt/packet_definitions/ialirt_codicehi.xml +1 -1
- imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +1 -1
- imap_processing/ialirt/packet_definitions/ialirt_swapi.xml +14 -14
- imap_processing/ialirt/utils/grouping.py +1 -1
- imap_processing/idex/idex_constants.py +9 -1
- imap_processing/idex/idex_l0.py +22 -8
- imap_processing/idex/idex_l1a.py +75 -44
- imap_processing/idex/idex_l1b.py +9 -8
- imap_processing/idex/idex_l2a.py +79 -45
- imap_processing/idex/idex_l2b.py +120 -0
- imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -39
- imap_processing/idex/packet_definitions/idex_housekeeping_packet_definition.xml +9130 -0
- imap_processing/lo/l0/lo_science.py +1 -2
- imap_processing/lo/l1a/lo_l1a.py +1 -4
- imap_processing/lo/l1b/lo_l1b.py +527 -6
- imap_processing/lo/l1b/tof_conversions.py +11 -0
- imap_processing/lo/l1c/lo_l1c.py +1 -4
- imap_processing/mag/constants.py +43 -0
- imap_processing/mag/imap_mag_sdc_configuration_v001.py +8 -0
- imap_processing/mag/l1a/mag_l1a.py +2 -9
- imap_processing/mag/l1a/mag_l1a_data.py +10 -10
- imap_processing/mag/l1b/mag_l1b.py +84 -17
- imap_processing/mag/l1c/interpolation_methods.py +180 -3
- imap_processing/mag/l1c/mag_l1c.py +236 -70
- imap_processing/mag/l2/mag_l2.py +140 -0
- imap_processing/mag/l2/mag_l2_data.py +288 -0
- imap_processing/spacecraft/quaternions.py +1 -3
- imap_processing/spice/geometry.py +3 -3
- imap_processing/spice/kernels.py +0 -276
- imap_processing/spice/pointing_frame.py +257 -0
- imap_processing/spice/repoint.py +48 -19
- imap_processing/spice/spin.py +38 -33
- imap_processing/spice/time.py +24 -0
- imap_processing/swapi/l1/swapi_l1.py +16 -12
- imap_processing/swapi/l2/swapi_l2.py +116 -4
- imap_processing/swapi/swapi_utils.py +32 -0
- imap_processing/swe/l1a/swe_l1a.py +2 -9
- imap_processing/swe/l1a/swe_science.py +8 -11
- imap_processing/swe/l1b/swe_l1b.py +898 -23
- imap_processing/swe/l2/swe_l2.py +21 -77
- imap_processing/swe/utils/swe_constants.py +1 -0
- imap_processing/tests/ccsds/test_excel_to_xtce.py +1 -1
- imap_processing/tests/cdf/test_utils.py +14 -16
- imap_processing/tests/codice/conftest.py +44 -33
- imap_processing/tests/codice/data/validation/imap_codice_l1a_hi-pha_20241110193700_v0.0.0.cdf +0 -0
- imap_processing/tests/codice/data/validation/imap_codice_l1a_lo-pha_20241110193700_v0.0.0.cdf +0 -0
- imap_processing/tests/codice/test_codice_l1a.py +20 -11
- imap_processing/tests/codice/test_codice_l1b.py +6 -7
- imap_processing/tests/conftest.py +78 -22
- imap_processing/tests/ena_maps/test_ena_maps.py +462 -33
- imap_processing/tests/ena_maps/test_spatial_utils.py +1 -1
- imap_processing/tests/glows/conftest.py +10 -14
- imap_processing/tests/glows/test_glows_decom.py +4 -4
- imap_processing/tests/glows/test_glows_l1a_cdf.py +6 -27
- imap_processing/tests/glows/test_glows_l1a_data.py +6 -8
- imap_processing/tests/glows/test_glows_l1b.py +11 -11
- imap_processing/tests/glows/test_glows_l1b_data.py +5 -5
- imap_processing/tests/glows/test_glows_l2.py +2 -8
- imap_processing/tests/hi/conftest.py +1 -1
- imap_processing/tests/hi/test_hi_l1b.py +10 -12
- imap_processing/tests/hi/test_hi_l1c.py +27 -24
- imap_processing/tests/hi/test_l1a.py +7 -9
- imap_processing/tests/hi/test_science_direct_event.py +2 -2
- imap_processing/tests/hit/helpers/l1_validation.py +44 -43
- imap_processing/tests/hit/test_decom_hit.py +1 -1
- imap_processing/tests/hit/test_hit_l1a.py +9 -9
- imap_processing/tests/hit/test_hit_l1b.py +172 -217
- imap_processing/tests/hit/test_hit_l2.py +380 -118
- imap_processing/tests/hit/test_hit_utils.py +122 -55
- imap_processing/tests/hit/validation_data/hit_l1b_standard_sample2_nsrl_v4_3decimals.csv +62 -62
- imap_processing/tests/hit/validation_data/sci_sample_raw.csv +1 -1
- imap_processing/tests/ialirt/unit/test_decom_ialirt.py +16 -81
- imap_processing/tests/ialirt/unit/test_grouping.py +2 -2
- imap_processing/tests/ialirt/unit/test_parse_mag.py +71 -16
- imap_processing/tests/ialirt/unit/test_process_codicehi.py +3 -3
- imap_processing/tests/ialirt/unit/test_process_codicelo.py +3 -10
- imap_processing/tests/ialirt/unit/test_process_ephemeris.py +4 -4
- imap_processing/tests/ialirt/unit/test_process_hit.py +3 -3
- imap_processing/tests/ialirt/unit/test_process_swapi.py +24 -16
- imap_processing/tests/ialirt/unit/test_process_swe.py +115 -7
- imap_processing/tests/idex/conftest.py +72 -7
- imap_processing/tests/idex/test_data/imap_idex_l0_raw_20241206_v001.pkts +0 -0
- imap_processing/tests/idex/test_data/imap_idex_l0_raw_20250108_v001.pkts +0 -0
- imap_processing/tests/idex/test_idex_l0.py +33 -11
- imap_processing/tests/idex/test_idex_l1a.py +50 -23
- imap_processing/tests/idex/test_idex_l1b.py +104 -25
- imap_processing/tests/idex/test_idex_l2a.py +48 -32
- imap_processing/tests/idex/test_idex_l2b.py +93 -0
- imap_processing/tests/lo/test_lo_l1a.py +3 -3
- imap_processing/tests/lo/test_lo_l1b.py +371 -6
- imap_processing/tests/lo/test_lo_l1c.py +1 -1
- imap_processing/tests/lo/test_lo_science.py +6 -7
- imap_processing/tests/lo/test_star_sensor.py +1 -1
- imap_processing/tests/mag/conftest.py +58 -9
- imap_processing/tests/mag/test_mag_decom.py +4 -3
- imap_processing/tests/mag/test_mag_l1a.py +13 -7
- imap_processing/tests/mag/test_mag_l1b.py +9 -9
- imap_processing/tests/mag/test_mag_l1c.py +151 -47
- imap_processing/tests/mag/test_mag_l2.py +130 -0
- imap_processing/tests/mag/test_mag_validation.py +144 -7
- imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-magi-normal-in.csv +1217 -0
- imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-magi-normal-out.csv +1857 -0
- imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-mago-normal-in.csv +1217 -0
- imap_processing/tests/mag/validation/L1c/T013/mag-l1b-l1c-t013-mago-normal-out.csv +1857 -0
- imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-magi-normal-in.csv +1217 -0
- imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-magi-normal-out.csv +1793 -0
- imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-mago-normal-in.csv +1217 -0
- imap_processing/tests/mag/validation/L1c/T014/mag-l1b-l1c-t014-mago-normal-out.csv +1793 -0
- imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-magi-burst-in.csv +2561 -0
- imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-magi-normal-in.csv +961 -0
- imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-magi-normal-out.csv +1539 -0
- imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-mago-normal-in.csv +1921 -0
- imap_processing/tests/mag/validation/L1c/T015/mag-l1b-l1c-t015-mago-normal-out.csv +2499 -0
- imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-magi-normal-in.csv +865 -0
- imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-magi-normal-out.csv +1196 -0
- imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-mago-normal-in.csv +1729 -0
- imap_processing/tests/mag/validation/L1c/T016/mag-l1b-l1c-t016-mago-normal-out.csv +3053 -0
- imap_processing/tests/mag/validation/L2/imap_mag_l1b_norm-mago_20251017_v002.cdf +0 -0
- imap_processing/tests/mag/validation/calibration/imap_mag_l2-calibration-matrices_20251017_v004.cdf +0 -0
- imap_processing/tests/mag/validation/calibration/imap_mag_l2-offsets-norm_20251017_20251017_v001.cdf +0 -0
- imap_processing/tests/spacecraft/test_quaternions.py +1 -1
- imap_processing/tests/spice/test_data/fake_repoint_data.csv +4 -4
- imap_processing/tests/spice/test_data/fake_spin_data.csv +11 -11
- imap_processing/tests/spice/test_geometry.py +3 -3
- imap_processing/tests/spice/test_kernels.py +1 -200
- imap_processing/tests/spice/test_pointing_frame.py +185 -0
- imap_processing/tests/spice/test_repoint.py +20 -10
- imap_processing/tests/spice/test_spin.py +50 -9
- imap_processing/tests/spice/test_time.py +14 -0
- imap_processing/tests/swapi/lut/imap_swapi_esa-unit-conversion_20250211_v000.csv +73 -0
- imap_processing/tests/swapi/lut/imap_swapi_lut-notes_20250211_v000.csv +1025 -0
- imap_processing/tests/swapi/test_swapi_l1.py +7 -9
- imap_processing/tests/swapi/test_swapi_l2.py +180 -8
- imap_processing/tests/swe/lut/checker-board-indices.csv +24 -0
- imap_processing/tests/swe/lut/imap_swe_esa-lut_20250301_v000.csv +385 -0
- imap_processing/tests/swe/lut/imap_swe_l1b-in-flight-cal_20240510_20260716_v000.csv +3 -0
- imap_processing/tests/swe/test_swe_l1a.py +6 -6
- imap_processing/tests/swe/test_swe_l1a_science.py +3 -3
- imap_processing/tests/swe/test_swe_l1b.py +162 -24
- imap_processing/tests/swe/test_swe_l2.py +82 -102
- imap_processing/tests/test_cli.py +171 -88
- imap_processing/tests/test_utils.py +2 -1
- imap_processing/tests/ultra/data/mock_data.py +49 -21
- imap_processing/tests/ultra/unit/conftest.py +53 -70
- imap_processing/tests/ultra/unit/test_badtimes.py +2 -4
- imap_processing/tests/ultra/unit/test_cullingmask.py +4 -6
- imap_processing/tests/ultra/unit/test_de.py +3 -10
- imap_processing/tests/ultra/unit/test_decom_apid_880.py +27 -76
- imap_processing/tests/ultra/unit/test_decom_apid_881.py +15 -16
- imap_processing/tests/ultra/unit/test_decom_apid_883.py +12 -10
- imap_processing/tests/ultra/unit/test_decom_apid_896.py +202 -55
- imap_processing/tests/ultra/unit/test_lookup_utils.py +23 -1
- imap_processing/tests/ultra/unit/test_spacecraft_pset.py +3 -4
- imap_processing/tests/ultra/unit/test_ultra_l1a.py +84 -307
- imap_processing/tests/ultra/unit/test_ultra_l1b.py +30 -12
- imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +2 -2
- imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +4 -1
- imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +163 -29
- imap_processing/tests/ultra/unit/test_ultra_l1c.py +5 -5
- imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +32 -43
- imap_processing/tests/ultra/unit/test_ultra_l2.py +230 -0
- imap_processing/ultra/constants.py +1 -1
- imap_processing/ultra/l0/decom_tools.py +21 -34
- imap_processing/ultra/l0/decom_ultra.py +168 -204
- imap_processing/ultra/l0/ultra_utils.py +152 -136
- imap_processing/ultra/l1a/ultra_l1a.py +55 -243
- imap_processing/ultra/l1b/badtimes.py +1 -4
- imap_processing/ultra/l1b/cullingmask.py +2 -6
- imap_processing/ultra/l1b/de.py +62 -47
- imap_processing/ultra/l1b/extendedspin.py +8 -4
- imap_processing/ultra/l1b/lookup_utils.py +72 -9
- imap_processing/ultra/l1b/ultra_l1b.py +3 -8
- imap_processing/ultra/l1b/ultra_l1b_culling.py +4 -4
- imap_processing/ultra/l1b/ultra_l1b_extended.py +236 -78
- imap_processing/ultra/l1c/histogram.py +2 -6
- imap_processing/ultra/l1c/spacecraft_pset.py +2 -4
- imap_processing/ultra/l1c/ultra_l1c.py +1 -5
- imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +107 -60
- imap_processing/ultra/l2/ultra_l2.py +299 -0
- imap_processing/ultra/lookup_tables/Angular_Profiles_FM45_LeftSlit.csv +526 -0
- imap_processing/ultra/lookup_tables/Angular_Profiles_FM45_RightSlit.csv +526 -0
- imap_processing/ultra/lookup_tables/Angular_Profiles_FM90_LeftSlit.csv +526 -0
- imap_processing/ultra/lookup_tables/Angular_Profiles_FM90_RightSlit.csv +526 -0
- imap_processing/ultra/lookup_tables/FM45_Startup1_ULTRA_IMGPARAMS_20240719.csv +2 -2
- imap_processing/ultra/lookup_tables/FM90_Startup1_ULTRA_IMGPARAMS_20240719.csv +2 -0
- imap_processing/ultra/packet_definitions/README.md +38 -0
- imap_processing/ultra/packet_definitions/ULTRA_SCI_COMBINED.xml +15302 -482
- imap_processing/ultra/utils/ultra_l1_utils.py +13 -12
- imap_processing/utils.py +1 -1
- {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/METADATA +3 -2
- {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/RECORD +264 -225
- imap_processing/hi/l1b/hi_eng_unit_convert_table.csv +0 -154
- imap_processing/mag/imap_mag_sdc-configuration_v001.yaml +0 -6
- imap_processing/mag/l1b/__init__.py +0 -0
- imap_processing/swe/l1b/swe_esa_lookup_table.csv +0 -1441
- imap_processing/swe/l1b/swe_l1b_science.py +0 -699
- imap_processing/tests/swe/test_swe_l1b_science.py +0 -103
- imap_processing/ultra/lookup_tables/dps_sensitivity45.cdf +0 -0
- imap_processing/ultra/lookup_tables/ultra_90_dps_exposure_compressed.cdf +0 -0
- /imap_processing/idex/packet_definitions/{idex_packet_definition.xml → idex_science_packet_definition.xml} +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/20240827095047_SWE_IALIRT_packet.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971383-404.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971384-405.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971385-406.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971386-407.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971387-408.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971388-409.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971389-410.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971390-411.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/461971391-412.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/BinLog CCSDS_FRAG_TLM_20240826_152323Z_IALIRT_data_for_SDC.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/IALiRT Raw Packet Telemetry.txt +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/apid01152.tlm +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/eu_SWP_IAL_20240826_152033.csv +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/hi_fsw_view_1_ccsds.bin +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/hit_ialirt_sample.ccsds +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/hit_ialirt_sample.csv +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/idle_export_eu.SWE_IALIRT_20240827_093852.csv +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/imap_codice_l1a_hi-ialirt_20240523200000_v0.0.0.cdf +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
- /imap_processing/tests/ialirt/{test_data → data}/l0/sample_decoded_i-alirt_data.csv +0 -0
- /imap_processing/tests/mag/validation/{imap_calibration_mag_20240229_v01.cdf → calibration/imap_mag_l1b-calibration_20240229_v001.cdf} +0 -0
- /imap_processing/{swe/l1b/engineering_unit_convert_table.csv → tests/swe/lut/imap_swe_eu-conversion_20240510_v000.csv} +0 -0
- {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/LICENSE +0 -0
- {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/WHEEL +0 -0
- {imap_processing-0.12.0.dist-info → imap_processing-0.13.0.dist-info}/entry_points.txt +0 -0
imap_processing/idex/idex_l0.py
CHANGED
|
@@ -2,14 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Union
|
|
5
|
+
from typing import Any, Union
|
|
6
|
+
|
|
7
|
+
from xarray import Dataset
|
|
6
8
|
|
|
7
9
|
from imap_processing import decom, imap_module_directory
|
|
10
|
+
from imap_processing.utils import packet_file_to_datasets
|
|
8
11
|
|
|
9
12
|
logger = logging.getLogger(__name__)
|
|
10
13
|
|
|
11
14
|
|
|
12
|
-
def decom_packets(
|
|
15
|
+
def decom_packets(
|
|
16
|
+
packet_file: Union[str, Path],
|
|
17
|
+
) -> tuple[list[Any], dict[int, Dataset]]:
|
|
13
18
|
"""
|
|
14
19
|
Decom IDEX data packets using IDEX packet definition.
|
|
15
20
|
|
|
@@ -20,12 +25,21 @@ def decom_packets(packet_file: Union[str, Path]) -> list:
|
|
|
20
25
|
|
|
21
26
|
Returns
|
|
22
27
|
-------
|
|
23
|
-
list
|
|
24
|
-
|
|
28
|
+
Tuple[list, dict]
|
|
29
|
+
Returns a list of all unpacked science data and a dictionary of datasets
|
|
30
|
+
indexed by their APIDs.
|
|
31
|
+
|
|
32
|
+
Notes
|
|
33
|
+
-----
|
|
34
|
+
The function 'packet_file_to_dataset' does not work with IDEX science packets due to
|
|
35
|
+
branching logic within the science xml file. The science data and housekeeping data
|
|
36
|
+
will be decommed separately and both returned from this function.
|
|
25
37
|
"""
|
|
26
|
-
|
|
27
|
-
|
|
38
|
+
xtce_base_path = f"{imap_module_directory}/idex/packet_definitions"
|
|
39
|
+
science_xtce_file = f"{xtce_base_path}/idex_science_packet_definition.xml"
|
|
40
|
+
hk_xtce_file = f"{xtce_base_path}/idex_housekeeping_packet_definition.xml"
|
|
28
41
|
|
|
29
|
-
|
|
42
|
+
science_decom_packet_list = decom.decom_packets(packet_file, science_xtce_file)
|
|
43
|
+
datasets_by_apid = packet_file_to_datasets(packet_file, hk_xtce_file)
|
|
30
44
|
|
|
31
|
-
return list(
|
|
45
|
+
return list(science_decom_packet_list), datasets_by_apid
|
imap_processing/idex/idex_l1a.py
CHANGED
|
@@ -10,7 +10,7 @@ Examples
|
|
|
10
10
|
from imap_processing.idex.idex_l1a import PacketParser
|
|
11
11
|
|
|
12
12
|
l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
|
|
13
|
-
l1a_data = PacketParser(l0_file
|
|
13
|
+
l1a_data = PacketParser(l0_file)
|
|
14
14
|
l1a_data.write_l1a_cdf()
|
|
15
15
|
"""
|
|
16
16
|
|
|
@@ -26,6 +26,7 @@ import xarray as xr
|
|
|
26
26
|
|
|
27
27
|
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
|
|
28
28
|
from imap_processing.idex.decode import rice_decode
|
|
29
|
+
from imap_processing.idex.idex_constants import IDEXAPID
|
|
29
30
|
from imap_processing.idex.idex_l0 import decom_packets
|
|
30
31
|
from imap_processing.spice.time import met_to_ttj2000ns
|
|
31
32
|
from imap_processing.utils import convert_to_binary_string
|
|
@@ -57,11 +58,9 @@ class PacketParser:
|
|
|
57
58
|
----------
|
|
58
59
|
packet_file : str
|
|
59
60
|
The path and filename to the L0 file to read.
|
|
60
|
-
data_version : str
|
|
61
|
-
The version of the data product being created.
|
|
62
61
|
"""
|
|
63
62
|
|
|
64
|
-
def __init__(self, packet_file: Union[str, Path]
|
|
63
|
+
def __init__(self, packet_file: Union[str, Path]) -> None:
|
|
65
64
|
"""
|
|
66
65
|
Read a L0 pkts file and perform all of the decom work.
|
|
67
66
|
|
|
@@ -69,23 +68,62 @@ class PacketParser:
|
|
|
69
68
|
----------
|
|
70
69
|
packet_file : pathlib.Path | str
|
|
71
70
|
The path and filename to the L0 file to read.
|
|
72
|
-
data_version : str
|
|
73
|
-
The version of the data product being created.
|
|
74
71
|
|
|
75
72
|
Notes
|
|
76
73
|
-----
|
|
77
74
|
Currently assumes one L0 file will generate exactly one L1a file.
|
|
78
75
|
"""
|
|
79
|
-
|
|
76
|
+
self.data = []
|
|
77
|
+
self.idex_attrs = get_idex_attrs()
|
|
78
|
+
epoch_attrs = self.idex_attrs.get_variable_attributes(
|
|
79
|
+
"epoch", check_schema=False
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
science_packets, datset_by_apid = decom_packets(packet_file)
|
|
83
|
+
|
|
84
|
+
if science_packets:
|
|
85
|
+
logger.info("Processing IDEX L1A Science data.")
|
|
86
|
+
self.data.append(self._create_science_dataset(science_packets))
|
|
87
|
+
|
|
88
|
+
if IDEXAPID.IDEX_EVT in datset_by_apid:
|
|
89
|
+
logger.info("Processing IDEX L1A Event Message data.")
|
|
90
|
+
data = datset_by_apid[IDEXAPID.IDEX_EVT]
|
|
91
|
+
data.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_evt")
|
|
92
|
+
data["epoch"].attrs = epoch_attrs
|
|
93
|
+
self.data.append(data)
|
|
94
|
+
|
|
95
|
+
if IDEXAPID.IDEX_CATLST in datset_by_apid:
|
|
96
|
+
logger.info("Processing IDEX L1A Catalog List Summary data.")
|
|
97
|
+
data = datset_by_apid[IDEXAPID.IDEX_CATLST]
|
|
98
|
+
data.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_catlst")
|
|
99
|
+
data["epoch"].attrs = epoch_attrs
|
|
100
|
+
self.data.append(data)
|
|
101
|
+
|
|
102
|
+
logger.info("IDEX L1A data processing completed.")
|
|
103
|
+
|
|
104
|
+
def _create_science_dataset(self, science_decom_packet_list: list) -> xr.Dataset:
|
|
105
|
+
"""
|
|
106
|
+
Process IDEX science packets into an xarray Dataset.
|
|
107
|
+
|
|
108
|
+
Parameters
|
|
109
|
+
----------
|
|
110
|
+
science_decom_packet_list : list
|
|
111
|
+
List of decommutated science packets.
|
|
112
|
+
|
|
113
|
+
Returns
|
|
114
|
+
-------
|
|
115
|
+
xarray.Dataset
|
|
116
|
+
Dataset containing processed dust events.
|
|
117
|
+
"""
|
|
80
118
|
dust_events = {}
|
|
81
|
-
for packet in
|
|
119
|
+
for packet in science_decom_packet_list:
|
|
82
120
|
if "IDX__SCI0TYPE" in packet:
|
|
83
121
|
scitype = packet["IDX__SCI0TYPE"]
|
|
84
122
|
event_number = packet["IDX__SCI0EVTNUM"]
|
|
85
123
|
if scitype == Scitype.FIRST_PACKET:
|
|
86
124
|
# Initial packet for new dust event
|
|
87
125
|
# Further packets will fill in data
|
|
88
|
-
dust_events[event_number] = RawDustEvent(packet
|
|
126
|
+
dust_events[event_number] = RawDustEvent(packet)
|
|
89
127
|
elif event_number not in dust_events:
|
|
90
128
|
raise KeyError(
|
|
91
129
|
f"Have not receive header information from event number\
|
|
@@ -101,40 +139,43 @@ class PacketParser:
|
|
|
101
139
|
dust_event.process() for dust_event in dust_events.values()
|
|
102
140
|
]
|
|
103
141
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
self.data.attrs = idex_attrs.get_global_attributes("imap_idex_l1a_sci")
|
|
142
|
+
data = xr.concat(processed_dust_impact_list, dim="epoch")
|
|
143
|
+
data.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_sci")
|
|
107
144
|
|
|
108
145
|
# Add high and low sample rate coords
|
|
109
|
-
|
|
110
|
-
np.arange(len(
|
|
146
|
+
data["time_low_sample_rate_index"] = xr.DataArray(
|
|
147
|
+
np.arange(len(data["time_low_sample_rate"][0])),
|
|
111
148
|
name="time_low_sample_rate_index",
|
|
112
149
|
dims=["time_low_sample_rate_index"],
|
|
113
|
-
attrs=idex_attrs.get_variable_attributes("time_low_sample_rate_index"),
|
|
150
|
+
attrs=self.idex_attrs.get_variable_attributes("time_low_sample_rate_index"),
|
|
114
151
|
)
|
|
115
152
|
|
|
116
|
-
|
|
117
|
-
np.arange(len(
|
|
153
|
+
data["time_high_sample_rate_index"] = xr.DataArray(
|
|
154
|
+
np.arange(len(data["time_high_sample_rate"][0])),
|
|
118
155
|
name="time_high_sample_rate_index",
|
|
119
156
|
dims=["time_high_sample_rate_index"],
|
|
120
|
-
attrs=idex_attrs.get_variable_attributes(
|
|
157
|
+
attrs=self.idex_attrs.get_variable_attributes(
|
|
158
|
+
"time_high_sample_rate_index"
|
|
159
|
+
),
|
|
121
160
|
)
|
|
122
161
|
# NOTE: LABL_PTR_1 should be CDF_CHAR.
|
|
123
|
-
|
|
124
|
-
|
|
162
|
+
data["time_low_sample_rate_label"] = xr.DataArray(
|
|
163
|
+
data.time_low_sample_rate_index.values.astype(str),
|
|
125
164
|
name="time_low_sample_rate_label",
|
|
126
165
|
dims=["time_low_sample_rate_index"],
|
|
127
|
-
attrs=idex_attrs.get_variable_attributes("time_low_sample_rate_label"),
|
|
166
|
+
attrs=self.idex_attrs.get_variable_attributes("time_low_sample_rate_label"),
|
|
128
167
|
)
|
|
129
168
|
|
|
130
|
-
|
|
131
|
-
|
|
169
|
+
data["time_high_sample_rate_label"] = xr.DataArray(
|
|
170
|
+
data.time_high_sample_rate_index.values.astype(str),
|
|
132
171
|
name="time_high_sample_rate_label",
|
|
133
172
|
dims=["time_high_sample_rate_index"],
|
|
134
|
-
attrs=idex_attrs.get_variable_attributes(
|
|
173
|
+
attrs=self.idex_attrs.get_variable_attributes(
|
|
174
|
+
"time_high_sample_rate_label"
|
|
175
|
+
),
|
|
135
176
|
)
|
|
136
177
|
|
|
137
|
-
|
|
178
|
+
return data
|
|
138
179
|
|
|
139
180
|
|
|
140
181
|
def _read_waveform_bits(waveform_raw: str, high_sample: bool = True) -> list[int]:
|
|
@@ -198,8 +239,6 @@ class RawDustEvent:
|
|
|
198
239
|
----------
|
|
199
240
|
header_packet : space_packet_parser.packets.CCSDSPacket
|
|
200
241
|
The FPGA metadata event header.
|
|
201
|
-
data_version : str
|
|
202
|
-
The version of the data product being created.
|
|
203
242
|
|
|
204
243
|
Attributes
|
|
205
244
|
----------
|
|
@@ -252,9 +291,7 @@ class RawDustEvent:
|
|
|
252
291
|
MAX_HIGH_BLOCKS = 16
|
|
253
292
|
MAX_LOW_BLOCKS = 64
|
|
254
293
|
|
|
255
|
-
def __init__(
|
|
256
|
-
self, header_packet: space_packet_parser.packets.CCSDSPacket, data_version: str
|
|
257
|
-
) -> None:
|
|
294
|
+
def __init__(self, header_packet: space_packet_parser.packets.CCSDSPacket) -> None:
|
|
258
295
|
"""
|
|
259
296
|
Initialize a raw dust event, with an FPGA Header Packet from IDEX.
|
|
260
297
|
|
|
@@ -268,8 +305,6 @@ class RawDustEvent:
|
|
|
268
305
|
----------
|
|
269
306
|
header_packet : space_packet_parser.packets.CCSDSPacket
|
|
270
307
|
The FPGA metadata event header.
|
|
271
|
-
data_version : str
|
|
272
|
-
Data version for CDF filename, in the format ``vXXX``.
|
|
273
308
|
"""
|
|
274
309
|
# Calculate the impact time in seconds since epoch
|
|
275
310
|
self.impact_time = 0
|
|
@@ -301,7 +336,7 @@ class RawDustEvent:
|
|
|
301
336
|
self.Ion_Grid_bits = ""
|
|
302
337
|
|
|
303
338
|
self.compressed = self.telemetry_items["idx__sci0comp"]
|
|
304
|
-
self.cdf_attrs = get_idex_attrs(
|
|
339
|
+
self.cdf_attrs = get_idex_attrs()
|
|
305
340
|
|
|
306
341
|
def _append_raw_data(self, scitype: Scitype, bits: str) -> None:
|
|
307
342
|
"""
|
|
@@ -445,11 +480,12 @@ class RawDustEvent:
|
|
|
445
480
|
List of the high sample waveform.
|
|
446
481
|
"""
|
|
447
482
|
samples = self.MAX_HIGH_BLOCKS * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK
|
|
483
|
+
ints: list[int] = []
|
|
448
484
|
if self.compressed.raw_value == 1:
|
|
449
|
-
ints
|
|
485
|
+
ints.extend(rice_decode(waveform_raw, nbit10=True, sample_count=samples))
|
|
450
486
|
ints = ints[:-3]
|
|
451
487
|
else:
|
|
452
|
-
ints
|
|
488
|
+
ints.extend(_read_waveform_bits(waveform_raw, high_sample=True))
|
|
453
489
|
return ints
|
|
454
490
|
|
|
455
491
|
def _parse_low_sample_waveform(self, waveform_raw: str) -> list[int]:
|
|
@@ -470,10 +506,11 @@ class RawDustEvent:
|
|
|
470
506
|
List of processed low sample waveform.
|
|
471
507
|
"""
|
|
472
508
|
samples = self.MAX_LOW_BLOCKS * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK
|
|
509
|
+
ints: list[int] = []
|
|
473
510
|
if self.compressed.raw_value == 1:
|
|
474
|
-
ints
|
|
511
|
+
ints.extend(rice_decode(waveform_raw, nbit10=False, sample_count=samples))
|
|
475
512
|
else:
|
|
476
|
-
ints
|
|
513
|
+
ints.extend(_read_waveform_bits(waveform_raw, high_sample=False))
|
|
477
514
|
return ints
|
|
478
515
|
|
|
479
516
|
def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray:
|
|
@@ -651,15 +688,10 @@ class RawDustEvent:
|
|
|
651
688
|
return dataset
|
|
652
689
|
|
|
653
690
|
|
|
654
|
-
def get_idex_attrs(
|
|
691
|
+
def get_idex_attrs() -> ImapCdfAttributes:
|
|
655
692
|
"""
|
|
656
693
|
Load in CDF attributes for IDEX instrument.
|
|
657
694
|
|
|
658
|
-
Parameters
|
|
659
|
-
----------
|
|
660
|
-
data_version : str
|
|
661
|
-
Data version for CDF filename, in the format "vXXX".
|
|
662
|
-
|
|
663
695
|
Returns
|
|
664
696
|
-------
|
|
665
697
|
idex_attrs : ImapCdfAttributes
|
|
@@ -668,5 +700,4 @@ def get_idex_attrs(data_version: str) -> ImapCdfAttributes:
|
|
|
668
700
|
idex_attrs = ImapCdfAttributes()
|
|
669
701
|
idex_attrs.add_instrument_global_attrs("idex")
|
|
670
702
|
idex_attrs.add_instrument_variable_attrs("idex", "l1a")
|
|
671
|
-
idex_attrs.add_global_attribute("Data_version", data_version)
|
|
672
703
|
return idex_attrs
|
imap_processing/idex/idex_l1b.py
CHANGED
|
@@ -9,8 +9,8 @@ Examples
|
|
|
9
9
|
from imap_processing.idex.idex_l1b import idex_l1b
|
|
10
10
|
|
|
11
11
|
l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
|
|
12
|
-
l1a_data = PacketParser(l0_file
|
|
13
|
-
l1b_data = idex_l1b(l1a_data
|
|
12
|
+
l1a_data = PacketParser(l0_file)
|
|
13
|
+
l1b_data = idex_l1b(l1a_data)
|
|
14
14
|
write_cdf(l1b_data)
|
|
15
15
|
"""
|
|
16
16
|
|
|
@@ -77,7 +77,7 @@ class TriggerMode(Enum):
|
|
|
77
77
|
return f"{channel.upper()}{TriggerMode(mode).name}"
|
|
78
78
|
|
|
79
79
|
|
|
80
|
-
def idex_l1b(l1a_dataset: xr.Dataset
|
|
80
|
+
def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset:
|
|
81
81
|
"""
|
|
82
82
|
Will process IDEX l1a data to create l1b data products.
|
|
83
83
|
|
|
@@ -85,8 +85,6 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
85
85
|
----------
|
|
86
86
|
l1a_dataset : xarray.Dataset
|
|
87
87
|
IDEX L1a dataset to process.
|
|
88
|
-
data_version : str
|
|
89
|
-
Version of the data product being created.
|
|
90
88
|
|
|
91
89
|
Returns
|
|
92
90
|
-------
|
|
@@ -101,7 +99,6 @@ def idex_l1b(l1a_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
101
99
|
idex_attrs = ImapCdfAttributes()
|
|
102
100
|
idex_attrs.add_instrument_global_attrs(instrument="idex")
|
|
103
101
|
idex_attrs.add_instrument_variable_attrs(instrument="idex", level="l1b")
|
|
104
|
-
idex_attrs.add_global_attribute("Data_version", data_version)
|
|
105
102
|
|
|
106
103
|
var_information_path = (
|
|
107
104
|
f"{imap_module_directory}/idex/idex_variable_unpacking_and_eu_conversion.csv"
|
|
@@ -295,10 +292,14 @@ def get_trigger_mode_and_level(
|
|
|
295
292
|
# Bit-shift right 22 places and use a 10-bit mask to extract the level value.
|
|
296
293
|
threshold_level = float((trigger_controls >> 22) & mask)
|
|
297
294
|
|
|
298
|
-
#
|
|
299
|
-
# TODO: determine why the idex team is only doing this for the high gain channel
|
|
295
|
+
# multiply the threshold level by the conversion factor.
|
|
300
296
|
if gain_channel == "hg":
|
|
301
297
|
threshold_level *= ConversionFactors["TOF_High"]
|
|
298
|
+
elif gain_channel == "mg":
|
|
299
|
+
threshold_level *= ConversionFactors["TOF_Mid"]
|
|
300
|
+
elif gain_channel == "lg":
|
|
301
|
+
threshold_level *= ConversionFactors["TOF_Low"]
|
|
302
|
+
|
|
302
303
|
return mode_label, threshold_level
|
|
303
304
|
|
|
304
305
|
for channel in channels:
|
imap_processing/idex/idex_l2a.py
CHANGED
|
@@ -10,16 +10,14 @@ Examples
|
|
|
10
10
|
from imap_processing.idex.idex_l2a import idex_l2a
|
|
11
11
|
|
|
12
12
|
l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20231214_v001.pkts"
|
|
13
|
-
l1a_data = PacketParser(l0_file
|
|
14
|
-
l1b_data = idex_l1b(l1a_data
|
|
15
|
-
l2a_data = idex_l2a(l1b_data
|
|
13
|
+
l1a_data = PacketParser(l0_file)
|
|
14
|
+
l1b_data = idex_l1b(l1a_data)
|
|
15
|
+
l2a_data = idex_l2a(l1b_data)
|
|
16
16
|
write_cdf(l2a_data)
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
|
-
# ruff: noqa: PLR0913
|
|
20
19
|
import logging
|
|
21
20
|
from enum import IntEnum
|
|
22
|
-
from typing import Union
|
|
23
21
|
|
|
24
22
|
import numpy as np
|
|
25
23
|
import pandas as pd
|
|
@@ -32,7 +30,6 @@ from scipy.stats import exponnorm
|
|
|
32
30
|
|
|
33
31
|
from imap_processing import imap_module_directory
|
|
34
32
|
from imap_processing.idex import idex_constants
|
|
35
|
-
from imap_processing.idex.idex_constants import ConversionFactors
|
|
36
33
|
from imap_processing.idex.idex_l1a import get_idex_attrs
|
|
37
34
|
|
|
38
35
|
logger = logging.getLogger(__name__)
|
|
@@ -54,7 +51,7 @@ class BaselineNoiseTime(IntEnum):
|
|
|
54
51
|
STOP = -5
|
|
55
52
|
|
|
56
53
|
|
|
57
|
-
def idex_l2a(l1b_dataset: xr.Dataset
|
|
54
|
+
def idex_l2a(l1b_dataset: xr.Dataset) -> xr.Dataset:
|
|
58
55
|
"""
|
|
59
56
|
Will process IDEX l1b data to create l2a data products.
|
|
60
57
|
|
|
@@ -70,8 +67,6 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
70
67
|
----------
|
|
71
68
|
l1b_dataset : xarray.Dataset
|
|
72
69
|
IDEX L1a dataset to process.
|
|
73
|
-
data_version : str
|
|
74
|
-
Version of the data product being created.
|
|
75
70
|
|
|
76
71
|
Returns
|
|
77
72
|
-------
|
|
@@ -104,7 +99,7 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
104
99
|
kappa = calculate_kappa(mass_scales, peaks_2d)
|
|
105
100
|
|
|
106
101
|
# Analyze peaks for estimating dust composition
|
|
107
|
-
|
|
102
|
+
peak_fits_params, area_under_fits, fit_chisqr, fit_redchi = xr.apply_ufunc(
|
|
108
103
|
analyze_peaks,
|
|
109
104
|
tof_high,
|
|
110
105
|
hs_time,
|
|
@@ -117,10 +112,11 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
117
112
|
["time_high_sample_rate_index"],
|
|
118
113
|
[],
|
|
119
114
|
],
|
|
120
|
-
# TODO: Determine dimension name
|
|
121
115
|
output_core_dims=[
|
|
122
|
-
["
|
|
123
|
-
["
|
|
116
|
+
["mass", "peak_fit_parameters"],
|
|
117
|
+
["mass"],
|
|
118
|
+
[],
|
|
119
|
+
[],
|
|
124
120
|
],
|
|
125
121
|
vectorize=True,
|
|
126
122
|
)
|
|
@@ -128,13 +124,11 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
128
124
|
l2a_dataset = l1b_dataset.copy()
|
|
129
125
|
|
|
130
126
|
for waveform in ["Target_Low", "Target_High", "Ion_Grid"]:
|
|
131
|
-
# Convert back to raw DNs for more accurate fits
|
|
132
|
-
waveform_dn = l1b_dataset[waveform] / ConversionFactors[waveform]
|
|
133
127
|
# Get the dust mass estimates and fit results
|
|
134
128
|
fit_results = xr.apply_ufunc(
|
|
135
129
|
estimate_dust_mass,
|
|
136
130
|
ls_time,
|
|
137
|
-
|
|
131
|
+
l1b_dataset[waveform],
|
|
138
132
|
input_core_dims=[
|
|
139
133
|
["time_low_sample_rate_index"],
|
|
140
134
|
["time_low_sample_rate_index"],
|
|
@@ -152,20 +146,23 @@ def idex_l2a(l1b_dataset: xr.Dataset, data_version: str) -> xr.Dataset:
|
|
|
152
146
|
waveform_name = waveform.lower()
|
|
153
147
|
# Add variables
|
|
154
148
|
l2a_dataset[f"{waveform_name}_fit_parameters"] = fit_results[0]
|
|
155
|
-
l2a_dataset[f"{waveform_name}
|
|
149
|
+
l2a_dataset[f"{waveform_name}_fit_impact_charge"] = fit_results[1]
|
|
156
150
|
# TODO: convert charge to mass
|
|
157
|
-
l2a_dataset[f"{waveform_name}
|
|
151
|
+
l2a_dataset[f"{waveform_name}_fit_impact_mass_estimate"] = fit_results[1]
|
|
158
152
|
l2a_dataset[f"{waveform_name}_chi_squared"] = fit_results[2]
|
|
159
153
|
l2a_dataset[f"{waveform_name}_reduced_chi_squared"] = fit_results[3]
|
|
160
154
|
l2a_dataset[f"{waveform_name}_fit_results"] = fit_results[4]
|
|
161
155
|
|
|
162
|
-
l2a_dataset["tof_peak_fit_parameters"] =
|
|
156
|
+
l2a_dataset["tof_peak_fit_parameters"] = peak_fits_params
|
|
163
157
|
l2a_dataset["tof_peak_area_under_fit"] = area_under_fits
|
|
158
|
+
l2a_dataset["tof_peak_chi_square"] = fit_chisqr
|
|
159
|
+
l2a_dataset["tof_peak_reduced_chi_square"] = fit_redchi
|
|
160
|
+
|
|
164
161
|
l2a_dataset["tof_peak_kappa"] = xr.DataArray(kappa, dims=["epoch"])
|
|
165
162
|
l2a_dataset["tof_snr"] = xr.DataArray(snr, dims=["epoch"])
|
|
166
163
|
l2a_dataset["mass"] = mass_scales_da
|
|
167
164
|
# Update global attributes
|
|
168
|
-
idex_attrs = get_idex_attrs(
|
|
165
|
+
idex_attrs = get_idex_attrs()
|
|
169
166
|
l2a_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l2a_sci")
|
|
170
167
|
|
|
171
168
|
logger.info("IDEX L2A science data processing completed.")
|
|
@@ -361,7 +358,7 @@ def analyze_peaks(
|
|
|
361
358
|
mass_scale: xr.DataArray,
|
|
362
359
|
event_num: int,
|
|
363
360
|
peaks_2d: np.ndarray,
|
|
364
|
-
) -> tuple[NDArray, NDArray]:
|
|
361
|
+
) -> tuple[NDArray, NDArray, float, float]:
|
|
365
362
|
"""
|
|
366
363
|
Fit an EMG curve to the Time of Flight data around each peak.
|
|
367
364
|
|
|
@@ -404,8 +401,8 @@ def analyze_peaks(
|
|
|
404
401
|
time_slice = high_sampling_time[start:end]
|
|
405
402
|
tof_slice = tof_high[start:end]
|
|
406
403
|
|
|
407
|
-
param = fit_emg(time_slice, tof_slice, event_num)
|
|
408
|
-
if param
|
|
404
|
+
param, chisqr, redchi = fit_emg(time_slice, tof_slice, event_num)
|
|
405
|
+
if np.all(np.isnan(param)):
|
|
409
406
|
continue
|
|
410
407
|
|
|
411
408
|
area = calculate_area_under_emg(time_slice, param)
|
|
@@ -427,7 +424,6 @@ def analyze_peaks(
|
|
|
427
424
|
mass = max(0, round(mass))
|
|
428
425
|
# Find the first index with non-zero fit parameters, starting from current mass
|
|
429
426
|
non_zero_idxs = np.nonzero(np.all(fit_params[mass:] != 0, axis=-1))[0]
|
|
430
|
-
|
|
431
427
|
# Determine index to use
|
|
432
428
|
# If no non-zero parameters found, use current mass index
|
|
433
429
|
# Otherwise, use the current mass plus offset to first non-zero index
|
|
@@ -437,16 +433,14 @@ def analyze_peaks(
|
|
|
437
433
|
fit_params[idx] = np.array([mu, sigma, lam])
|
|
438
434
|
area_under_emg[idx] = area
|
|
439
435
|
else:
|
|
440
|
-
logger.warning(
|
|
441
|
-
f"Unable to find a slot for mass: {mass}. Discarding " f"value."
|
|
442
|
-
)
|
|
436
|
+
logger.warning(f"Unable to find a slot for mass: {mass}. Discarding value.")
|
|
443
437
|
|
|
444
|
-
return fit_params, area_under_emg
|
|
438
|
+
return fit_params, area_under_emg, chisqr, redchi
|
|
445
439
|
|
|
446
440
|
|
|
447
441
|
def fit_emg(
|
|
448
442
|
peak_time: np.ndarray, peak_signal: np.ndarray, event_num: int
|
|
449
|
-
) ->
|
|
443
|
+
) -> tuple[NDArray, float, float]:
|
|
450
444
|
"""
|
|
451
445
|
Fit an exponentially modified gaussian function to the peak signal.
|
|
452
446
|
|
|
@@ -465,9 +459,13 @@ def fit_emg(
|
|
|
465
459
|
|
|
466
460
|
Returns
|
|
467
461
|
-------
|
|
468
|
-
param : numpy.ndarray
|
|
462
|
+
param : numpy.ndarray
|
|
469
463
|
Fitted EMG optimal values for the parameters (popt) [k (shape parameter), mu,
|
|
470
|
-
sigma] if fit successful,
|
|
464
|
+
sigma] if fit is successful, array of np.nans otherwise.
|
|
465
|
+
chisqr : float
|
|
466
|
+
Chi-square value if fit is successful, np.nan otherwise.
|
|
467
|
+
redchi : float
|
|
468
|
+
Reduced chi-square value if fit is successful, np.nan otherwise.
|
|
471
469
|
"""
|
|
472
470
|
# Initial Guess for the parameters of the emg fit:
|
|
473
471
|
# center of gaussian
|
|
@@ -490,11 +488,14 @@ def fit_emg(
|
|
|
490
488
|
f"Time range: {peak_time[0]:.2f} to {peak_time[-1]:.2f}\n"
|
|
491
489
|
f"Signal range: {min(peak_signal):.2f} to {max(peak_signal):.2f}\n"
|
|
492
490
|
f"Event number: {event_num}\n"
|
|
493
|
-
"Returning
|
|
491
|
+
"Returning np.nan values."
|
|
494
492
|
)
|
|
495
|
-
return
|
|
493
|
+
return np.full(len(p0), np.nan), np.nan, np.nan
|
|
494
|
+
|
|
495
|
+
emg_fit = exponnorm.pdf(peak_time, *param)
|
|
496
|
+
chisqr, redchi = chi_square(peak_signal, emg_fit, len(p0))
|
|
496
497
|
|
|
497
|
-
return param
|
|
498
|
+
return param, chisqr, redchi
|
|
498
499
|
|
|
499
500
|
|
|
500
501
|
def calculate_area_under_emg(time_slice: np.ndarray, param: np.ndarray) -> float:
|
|
@@ -555,8 +556,6 @@ def estimate_dust_mass(
|
|
|
555
556
|
result : numpy.ndarray
|
|
556
557
|
The model values evaluated at each time point.
|
|
557
558
|
"""
|
|
558
|
-
# TODO: The IDEX team is iterating on this Function and will provide more
|
|
559
|
-
# information soon.
|
|
560
559
|
signal = np.array(target_signal.data)
|
|
561
560
|
time = np.array(low_sampling_time.data)
|
|
562
561
|
good_mask = np.logical_and(
|
|
@@ -615,11 +614,7 @@ def estimate_dust_mass(
|
|
|
615
614
|
impact_fit = fit_impact(time, *param)
|
|
616
615
|
# Calculate the resulting signal amplitude after removing baseline noise
|
|
617
616
|
sig_amp = max(impact_fit) - np.mean(signal_baseline)
|
|
618
|
-
|
|
619
|
-
# Calculate chi square and reduced chi square
|
|
620
|
-
chisqr = float(np.sum((signal - impact_fit) ** 2))
|
|
621
|
-
# To get reduced chi square divide by dof (number of points - number of params)
|
|
622
|
-
redchi = chisqr / (len(signal) - len(p0))
|
|
617
|
+
chisqr, redchi = chi_square(signal, impact_fit, len(p0))
|
|
623
618
|
|
|
624
619
|
return param, float(sig_amp), chisqr, redchi, impact_fit
|
|
625
620
|
|
|
@@ -729,7 +724,7 @@ def remove_signal_noise(
|
|
|
729
724
|
|
|
730
725
|
def sine_fit(time: np.ndarray, a: float, f: float, p: float) -> NDArray:
|
|
731
726
|
"""
|
|
732
|
-
Generate a sine wave with given amplitude, frequency, and phase.
|
|
727
|
+
Generate a sine wave with given amplitude, angular frequency, and phase.
|
|
733
728
|
|
|
734
729
|
Parameters
|
|
735
730
|
----------
|
|
@@ -738,7 +733,7 @@ def sine_fit(time: np.ndarray, a: float, f: float, p: float) -> NDArray:
|
|
|
738
733
|
a : float
|
|
739
734
|
Amplitude of the sine wave.
|
|
740
735
|
f : float
|
|
741
|
-
|
|
736
|
+
Angular frequency of the sine wave.
|
|
742
737
|
p : float
|
|
743
738
|
Phase shift of the sine wave in radians.
|
|
744
739
|
|
|
@@ -747,7 +742,7 @@ def sine_fit(time: np.ndarray, a: float, f: float, p: float) -> NDArray:
|
|
|
747
742
|
numpy.ndarray
|
|
748
743
|
Sine wave values calculated at the input time points.
|
|
749
744
|
"""
|
|
750
|
-
return a * np.sin(
|
|
745
|
+
return a * np.sin(f * time + p)
|
|
751
746
|
|
|
752
747
|
|
|
753
748
|
def butter_lowpass_filter(
|
|
@@ -772,7 +767,6 @@ def butter_lowpass_filter(
|
|
|
772
767
|
numpy.ndarray
|
|
773
768
|
Filtered signal.
|
|
774
769
|
"""
|
|
775
|
-
# TODO: The IDEX team might be switching this function out for a different filter.
|
|
776
770
|
sample_period = time[1] - time[0]
|
|
777
771
|
# sampling frequency
|
|
778
772
|
fs = (time[-1] - time[0]) / sample_period # Hz
|
|
@@ -787,3 +781,43 @@ def butter_lowpass_filter(
|
|
|
787
781
|
b, a = butter(order, normal_cutoff, btype="low", analog=False)
|
|
788
782
|
y = filtfilt(b, a, signal)
|
|
789
783
|
return y
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def chi_square(
|
|
787
|
+
observed: np.ndarray, expected: np.ndarray, num_params: int
|
|
788
|
+
) -> tuple[float, float]:
|
|
789
|
+
"""
|
|
790
|
+
Calculate the chi-square and reduced chi-square statistics.
|
|
791
|
+
|
|
792
|
+
This implementation follows the approach used in lmfit.minimize()'s
|
|
793
|
+
_calculate_statistics() method, which calculates chi-square as the sum of squared
|
|
794
|
+
residuals:
|
|
795
|
+
|
|
796
|
+
chisqr = (residual**2).sum()
|
|
797
|
+
|
|
798
|
+
And reduced chi-square as the chi-square divided by degrees of freedom:
|
|
799
|
+
|
|
800
|
+
ndata = len(residual)
|
|
801
|
+
nfree = ndata - number_of_parameters
|
|
802
|
+
redchi = chisqr / max(1, nfree)
|
|
803
|
+
|
|
804
|
+
Parameters
|
|
805
|
+
----------
|
|
806
|
+
observed : numpy.ndarray
|
|
807
|
+
The observed signal.
|
|
808
|
+
expected : numpy.ndarray
|
|
809
|
+
The expected signal calculated with the fit parameters.
|
|
810
|
+
num_params : int
|
|
811
|
+
The number of parameters used in the fit.
|
|
812
|
+
|
|
813
|
+
Returns
|
|
814
|
+
-------
|
|
815
|
+
chisqr : float
|
|
816
|
+
The chi-square value.
|
|
817
|
+
redchi : float
|
|
818
|
+
The reduced chi-square value.
|
|
819
|
+
"""
|
|
820
|
+
residuals = observed - expected
|
|
821
|
+
chisqr = float(np.sum(residuals**2))
|
|
822
|
+
redchi = chisqr / max(1, (len(observed) - num_params))
|
|
823
|
+
return chisqr, redchi
|