imap-processing 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (172) hide show
  1. imap_processing/__init__.py +1 -1
  2. imap_processing/_version.py +2 -2
  3. imap_processing/ccsds/excel_to_xtce.py +36 -2
  4. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
  5. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
  6. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
  7. imap_processing/cdf/config/imap_hi_variable_attrs.yaml +136 -9
  8. imap_processing/cdf/config/imap_hit_global_cdf_attrs.yaml +14 -0
  9. imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +63 -1
  10. imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
  11. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +14 -7
  12. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +577 -235
  13. imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +326 -0
  14. imap_processing/cdf/config/imap_lo_l1a_variable_attrs.yaml +33 -23
  15. imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
  16. imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
  17. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +137 -79
  18. imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
  19. imap_processing/cdf/imap_cdf_manager.py +31 -27
  20. imap_processing/cdf/utils.py +3 -5
  21. imap_processing/cli.py +25 -14
  22. imap_processing/codice/codice_l1a.py +153 -63
  23. imap_processing/codice/constants.py +10 -10
  24. imap_processing/codice/decompress.py +10 -11
  25. imap_processing/codice/utils.py +1 -0
  26. imap_processing/glows/l1a/glows_l1a.py +1 -2
  27. imap_processing/glows/l1b/glows_l1b.py +3 -3
  28. imap_processing/glows/l1b/glows_l1b_data.py +59 -37
  29. imap_processing/glows/l2/glows_l2_data.py +123 -0
  30. imap_processing/hi/l1a/hi_l1a.py +4 -4
  31. imap_processing/hi/l1a/histogram.py +107 -109
  32. imap_processing/hi/l1a/science_direct_event.py +92 -225
  33. imap_processing/hi/l1b/hi_l1b.py +85 -11
  34. imap_processing/hi/l1c/hi_l1c.py +23 -1
  35. imap_processing/hi/packet_definitions/TLM_HI_COMBINED_SCI.xml +3994 -0
  36. imap_processing/hi/utils.py +1 -1
  37. imap_processing/hit/hit_utils.py +221 -0
  38. imap_processing/hit/l0/constants.py +118 -0
  39. imap_processing/hit/l0/decom_hit.py +100 -156
  40. imap_processing/hit/l1a/hit_l1a.py +170 -184
  41. imap_processing/hit/l1b/hit_l1b.py +33 -153
  42. imap_processing/ialirt/l0/process_codicelo.py +153 -0
  43. imap_processing/ialirt/l0/process_hit.py +5 -5
  44. imap_processing/ialirt/packet_definitions/ialirt_codicelo.xml +281 -0
  45. imap_processing/ialirt/process_ephemeris.py +212 -0
  46. imap_processing/idex/idex_l1a.py +65 -84
  47. imap_processing/idex/idex_l1b.py +192 -0
  48. imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +33 -0
  49. imap_processing/idex/packet_definitions/idex_packet_definition.xml +97 -595
  50. imap_processing/lo/l0/decompression_tables/decompression_tables.py +17 -1
  51. imap_processing/lo/l0/lo_science.py +45 -13
  52. imap_processing/lo/l1a/lo_l1a.py +76 -8
  53. imap_processing/lo/packet_definitions/lo_xtce.xml +8344 -1849
  54. imap_processing/mag/l0/decom_mag.py +4 -3
  55. imap_processing/mag/l1a/mag_l1a.py +12 -13
  56. imap_processing/mag/l1a/mag_l1a_data.py +1 -2
  57. imap_processing/mag/l1b/mag_l1b.py +90 -7
  58. imap_processing/spice/geometry.py +156 -16
  59. imap_processing/spice/time.py +144 -2
  60. imap_processing/swapi/l1/swapi_l1.py +4 -4
  61. imap_processing/swapi/l2/swapi_l2.py +1 -1
  62. imap_processing/swapi/packet_definitions/swapi_packet_definition.xml +1535 -446
  63. imap_processing/swe/l1b/swe_l1b_science.py +8 -8
  64. imap_processing/swe/l2/swe_l2.py +134 -17
  65. imap_processing/tests/ccsds/test_data/expected_output.xml +2 -1
  66. imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
  67. imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
  68. imap_processing/tests/codice/conftest.py +1 -17
  69. imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
  70. imap_processing/tests/codice/test_codice_l0.py +8 -2
  71. imap_processing/tests/codice/test_codice_l1a.py +127 -107
  72. imap_processing/tests/codice/test_codice_l1b.py +1 -0
  73. imap_processing/tests/codice/test_decompress.py +7 -7
  74. imap_processing/tests/conftest.py +100 -58
  75. imap_processing/tests/glows/conftest.py +6 -0
  76. imap_processing/tests/glows/test_glows_l1b.py +9 -9
  77. imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
  78. imap_processing/tests/hi/test_data/l0/H90_NHK_20241104.bin +0 -0
  79. imap_processing/tests/hi/test_data/l0/H90_sci_cnt_20241104.bin +0 -0
  80. imap_processing/tests/hi/test_data/l0/H90_sci_de_20241104.bin +0 -0
  81. imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
  82. imap_processing/tests/hi/test_hi_l1b.py +73 -3
  83. imap_processing/tests/hi/test_hi_l1c.py +10 -2
  84. imap_processing/tests/hi/test_l1a.py +31 -58
  85. imap_processing/tests/hi/test_science_direct_event.py +58 -0
  86. imap_processing/tests/hi/test_utils.py +4 -3
  87. imap_processing/tests/hit/test_data/sci_sample1.ccsds +0 -0
  88. imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +95 -36
  89. imap_processing/tests/hit/test_hit_l1a.py +299 -179
  90. imap_processing/tests/hit/test_hit_l1b.py +231 -24
  91. imap_processing/tests/hit/test_hit_utils.py +218 -0
  92. imap_processing/tests/hit/validation_data/hskp_sample_eu.csv +89 -0
  93. imap_processing/tests/hit/validation_data/sci_sample_raw1.csv +29 -0
  94. imap_processing/tests/ialirt/test_data/l0/apid01152.tlm +0 -0
  95. imap_processing/tests/ialirt/test_data/l0/imap_codice_l1a_lo-ialirt_20241110193700_v0.0.0.cdf +0 -0
  96. imap_processing/tests/ialirt/unit/test_process_codicelo.py +106 -0
  97. imap_processing/tests/ialirt/unit/test_process_ephemeris.py +109 -0
  98. imap_processing/tests/ialirt/unit/test_process_hit.py +9 -6
  99. imap_processing/tests/idex/conftest.py +2 -2
  100. imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
  101. imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
  102. imap_processing/tests/idex/test_idex_l0.py +4 -4
  103. imap_processing/tests/idex/test_idex_l1a.py +8 -2
  104. imap_processing/tests/idex/test_idex_l1b.py +126 -0
  105. imap_processing/tests/lo/test_lo_l1a.py +7 -16
  106. imap_processing/tests/lo/test_lo_science.py +69 -5
  107. imap_processing/tests/lo/test_pkts/imap_lo_l0_raw_20240803_v002.pkts +0 -0
  108. imap_processing/tests/lo/validation_data/Instrument_FM1_T104_R129_20240803_ILO_SCI_DE_dec_DN_with_fills.csv +1999 -0
  109. imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
  110. imap_processing/tests/mag/test_mag_l1b.py +97 -7
  111. imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
  112. imap_processing/tests/spice/test_geometry.py +115 -9
  113. imap_processing/tests/spice/test_time.py +135 -6
  114. imap_processing/tests/swapi/test_swapi_decom.py +75 -69
  115. imap_processing/tests/swapi/test_swapi_l1.py +4 -4
  116. imap_processing/tests/swe/conftest.py +33 -0
  117. imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
  118. imap_processing/tests/swe/test_swe_l1b.py +29 -8
  119. imap_processing/tests/swe/test_swe_l2.py +64 -8
  120. imap_processing/tests/test_utils.py +2 -2
  121. imap_processing/tests/ultra/test_data/l0/ultra45_raw_sc_ultrarawimg_withFSWcalcs_FM45_40P_Phi28p5_BeamCal_LinearScan_phi2850_theta-000_20240207T102740.csv +3314 -3314
  122. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
  123. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
  124. imap_processing/tests/ultra/unit/test_de.py +113 -0
  125. imap_processing/tests/ultra/unit/test_spatial_utils.py +125 -0
  126. imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
  127. imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
  128. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +55 -35
  129. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +10 -68
  130. imap_processing/ultra/constants.py +12 -3
  131. imap_processing/ultra/l1b/de.py +168 -30
  132. imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
  133. imap_processing/ultra/l1b/ultra_l1b_extended.py +46 -80
  134. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +60 -144
  135. imap_processing/ultra/utils/spatial_utils.py +221 -0
  136. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/METADATA +15 -14
  137. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/RECORD +142 -139
  138. imap_processing/cdf/cdf_attribute_manager.py +0 -322
  139. imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
  140. imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  141. imap_processing/hi/l0/decom_hi.py +0 -24
  142. imap_processing/hi/packet_definitions/hi_packet_definition.xml +0 -482
  143. imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
  144. imap_processing/hit/l0/data_classes/science_packet.py +0 -259
  145. imap_processing/hit/l0/utils/hit_base.py +0 -57
  146. imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
  147. imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  148. imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
  149. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
  150. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
  151. imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
  152. imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
  153. imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
  154. imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
  155. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
  156. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
  157. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
  158. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
  159. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
  160. imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
  161. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
  162. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
  163. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
  164. imap_processing/tests/hi/test_decom.py +0 -55
  165. imap_processing/tests/hi/test_l1a_sci_de.py +0 -72
  166. imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
  167. imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
  168. /imap_processing/{hi/l0/__init__.py → tests/glows/test_glows_l2_data.py} +0 -0
  169. /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
  170. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/LICENSE +0 -0
  171. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/WHEEL +0 -0
  172. {imap_processing-0.7.0.dist-info → imap_processing-0.9.0.dist-info}/entry_points.txt +0 -0
@@ -417,19 +417,19 @@ class HistogramL1B:
417
417
  IMAP spin angle ψ for bin centers
418
418
  filter_temperature_average
419
419
  block-averaged value, decoded to Celsius degrees using Eq. (47)
420
- filter_temperature_variance
420
+ filter_temperature_std_dev
421
421
  standard deviation (1 sigma), decoded to Celsius degrees using Eq. (51)
422
422
  hv_voltage_average
423
423
  block-averaged value, decoded to volts using Eq. (47)
424
- hv_voltage_variance
424
+ hv_voltage_std_dev
425
425
  standard deviation (1 sigma), decoded to volts using Eq. (51)
426
426
  spin_period_average
427
427
  block-averaged onboard value, decoded to seconds using Eq. (47)
428
- spin_period_variance
428
+ spin_period_std_dev
429
429
  standard deviation (1 sigma), decoded to seconds using Eq. (51)
430
430
  pulse_length_average
431
431
  block-averaged value, decoded to μs using Eq. (47)
432
- pulse_length_variance
432
+ pulse_length_std_dev
433
433
  standard deviation (1 sigma), decoded to μs using Eq. (51)
434
434
  glows_start_time
435
435
  GLOWS clock, subseconds as decimal part of float
@@ -444,24 +444,24 @@ class HistogramL1B:
444
444
  is_inside_excluded_region, is_excluded_by_instr_team, is_suspected_transient]
445
445
  spin_period_ground_average
446
446
  block-averaged value computed on ground
447
- spin_period_ground_variance
447
+ spin_period_ground_std_dev
448
448
  standard deviation (1 sigma)
449
449
  position_angle_offset_average
450
450
  block-averaged value in degrees
451
- position_angle_offset_variance
451
+ position_angle_offset_std_dev
452
452
  standard deviation (1 sigma)
453
- spin_axis_orientation_variance
453
+ spin_axis_orientation_std_dev
454
454
  standard deviation( 1 sigma): ∆λ, ∆φ for ⟨λ⟩, ⟨φ⟩
455
455
  spin_axis_orientation_average
456
456
  block-averaged spin-axis ecliptic longitude ⟨λ⟩ and latitude ⟨φ⟩ in degrees
457
457
  spacecraft_location_average
458
458
  block-averaged Cartesian ecliptic coordinates ⟨X⟩, ⟨Y ⟩, ⟨Z⟩ [km] of IMAP
459
- spacecraft_location_variance
459
+ spacecraft_location_std_dev
460
460
  standard deviations (1 sigma) ∆X, ∆Y , ∆Z for ⟨X⟩, ⟨Y ⟩, ⟨Z⟩
461
461
  spacecraft_velocity_average
462
462
  block-averaged values ⟨VX⟩, ⟨VY⟩, ⟨VZ⟩ [km/s] of IMAP velocity components
463
463
  (Cartesian ecliptic frame)
464
- spacecraft_velocity_variance
464
+ spacecraft_velocity_std_dev
465
465
  standard deviations (1 sigma) ∆VX , ∆VY , ∆VZ for ⟨VX ⟩, ⟨VY ⟩, ⟨VZ ⟩
466
466
  flags
467
467
  flags for extra information, per histogram. This should be a human-readable
@@ -470,9 +470,7 @@ class HistogramL1B:
470
470
 
471
471
  histogram: np.ndarray
472
472
  flight_software_version: str
473
- # pkts_file_name: str TODO: add this in L0
474
473
  seq_count_in_pkts_file: int
475
- # l1a_file_name: str TODO: add this
476
474
  # ancillary_data_files: np.ndarray TODO Add this
477
475
  first_spin_id: int
478
476
  last_spin_id: int
@@ -482,13 +480,17 @@ class HistogramL1B:
482
480
  number_of_bins_per_histogram: int
483
481
  number_of_events: int
484
482
  filter_temperature_average: np.double
485
- filter_temperature_variance: np.double
483
+ filter_temperature_variance: InitVar[np.double]
484
+ filter_temperature_std_dev: np.double = field(init=False)
486
485
  hv_voltage_average: np.double
487
- hv_voltage_variance: np.double
486
+ hv_voltage_variance: InitVar[np.double]
487
+ hv_voltage_std_dev: np.double = field(init=False)
488
488
  spin_period_average: np.double
489
- spin_period_variance: np.double
489
+ spin_period_variance: InitVar[np.double]
490
+ spin_period_std_dev: np.double = field(init=False)
490
491
  pulse_length_average: np.double
491
- pulse_length_variance: np.double
492
+ pulse_length_variance: InitVar[np.double]
493
+ pulse_length_std_dev: np.double = field(init=False)
492
494
  imap_start_time: np.double # No conversion needed from l1a->l1b
493
495
  imap_time_offset: np.double # No conversion needed from l1a->l1b
494
496
  glows_start_time: np.double # No conversion needed from l1a->l1b
@@ -499,40 +501,60 @@ class HistogramL1B:
499
501
  imap_spin_angle_bin_cntr: np.ndarray = field(init=False) # Same size as bins
500
502
  histogram_flag_array: np.ndarray = field(init=False)
501
503
  spin_period_ground_average: np.double = field(init=False) # retrieved from SPICE?
502
- spin_period_ground_variance: np.double = field(init=False) # retrieved from SPICE?
504
+ spin_period_ground_std_dev: np.double = field(init=False) # retrieved from SPICE?
503
505
  position_angle_offset_average: np.double = field(init=False) # retrieved from SPICE
504
- position_angle_offset_variance: np.double = field(init=False) # from SPICE
505
- spin_axis_orientation_variance: np.double = field(init=False) # from SPICE
506
+ position_angle_offset_std_dev: np.double = field(init=False) # from SPICE
507
+ spin_axis_orientation_std_dev: np.double = field(init=False) # from SPICE
506
508
  spin_axis_orientation_average: np.double = field(init=False) # retrieved from SPICE
507
509
  spacecraft_location_average: np.ndarray = field(init=False) # retrieved from SPIC
508
- spacecraft_location_variance: np.ndarray = field(init=False) # retrieved from SPIC
510
+ spacecraft_location_std_dev: np.ndarray = field(init=False) # retrieved from SPIC
509
511
  spacecraft_velocity_average: np.ndarray = field(init=False) # retrieved from SPIC
510
- spacecraft_velocity_variance: np.ndarray = field(init=False) # retrieved from SPIC
512
+ spacecraft_velocity_std_dev: np.ndarray = field(init=False) # retrieved from SPIC
511
513
  flags: np.ndarray = field(init=False)
512
514
  # TODO:
513
515
  # - Determine a good way to output flags as "human readable"
514
516
  # - Add spice pieces
515
- # - add in the filenames for the input files - should they be global attributes?
516
517
  # - also unique identifiers
517
518
  # - Bad angle algorithm using SPICE locations
518
519
  # - Move ancillary file to AWS
519
520
 
520
- def __post_init__(self) -> None:
521
- """Will process data."""
521
+ def __post_init__(
522
+ self,
523
+ filter_temperature_variance: np.double,
524
+ hv_voltage_variance: np.double,
525
+ spin_period_variance: np.double,
526
+ pulse_length_variance: np.double,
527
+ ) -> None:
528
+ """
529
+ Will process data.
530
+
531
+ The input variance values are used to calculate the output standard deviation.
532
+
533
+ Parameters
534
+ ----------
535
+ filter_temperature_variance : numpy.double
536
+ Encoded filter temperature variance.
537
+ hv_voltage_variance : numpy.double
538
+ Encoded HV voltage variance.
539
+ spin_period_variance : numpy.double
540
+ Encoded spin period variance.
541
+ pulse_length_variance : numpy.double
542
+ Encoded pulse length variance.
543
+ """
522
544
  # self.histogram_flag_array = np.zeros((2,))
523
545
 
524
546
  # TODO: These pieces will need to be filled in from SPICE kernels. For now,
525
547
  # they are placeholders. GLOWS example code has better placeholders if needed.
526
548
  self.spin_period_ground_average = np.double(-999.9)
527
- self.spin_period_ground_variance = np.double(-999.9)
549
+ self.spin_period_ground_std_dev = np.double(-999.9)
528
550
  self.position_angle_offset_average = np.double(-999.9)
529
- self.position_angle_offset_variance = np.double(-999.9)
530
- self.spin_axis_orientation_variance = np.double(-999.9)
551
+ self.position_angle_offset_std_dev = np.double(-999.9)
552
+ self.spin_axis_orientation_std_dev = np.double(-999.9)
531
553
  self.spin_axis_orientation_average = np.double(-999.9)
532
554
  self.spacecraft_location_average = np.array([-999.9, -999.9, -999.9])
533
- self.spacecraft_location_variance = np.array([-999.9, -999.9, -999.9])
555
+ self.spacecraft_location_std_dev = np.array([-999.9, -999.9, -999.9])
534
556
  self.spacecraft_velocity_average = np.array([-999.9, -999.9, -999.9])
535
- self.spacecraft_velocity_variance = np.array([-999.9, -999.9, -999.9])
557
+ self.spacecraft_velocity_std_dev = np.array([-999.9, -999.9, -999.9])
536
558
  # Will require some additional inputs
537
559
  self.imap_spin_angle_bin_cntr = np.zeros((3600,))
538
560
 
@@ -546,34 +568,34 @@ class HistogramL1B:
546
568
  self.filter_temperature_average = self.ancillary_parameters.decode(
547
569
  "filter_temperature", self.filter_temperature_average
548
570
  )
549
- self.filter_temperature_variance = self.ancillary_parameters.decode_std_dev(
550
- "filter_temperature", self.filter_temperature_variance
571
+ self.filter_temperature_std_dev = self.ancillary_parameters.decode_std_dev(
572
+ "filter_temperature", filter_temperature_variance
551
573
  )
552
574
 
553
575
  self.hv_voltage_average = self.ancillary_parameters.decode(
554
576
  "hv_voltage", self.hv_voltage_average
555
577
  )
556
- self.hv_voltage_variance = self.ancillary_parameters.decode_std_dev(
557
- "hv_voltage", self.hv_voltage_variance
578
+ self.hv_voltage_std_dev = self.ancillary_parameters.decode_std_dev(
579
+ "hv_voltage", hv_voltage_variance
558
580
  )
559
581
  self.spin_period_average = self.ancillary_parameters.decode(
560
582
  "spin_period", self.spin_period_average
561
583
  )
562
- self.spin_period_variance = self.ancillary_parameters.decode_std_dev(
563
- "spin_period", self.spin_period_variance
584
+ self.spin_period_std_dev = self.ancillary_parameters.decode_std_dev(
585
+ "spin_period", spin_period_variance
564
586
  )
565
587
  self.pulse_length_average = self.ancillary_parameters.decode(
566
588
  "pulse_length", self.pulse_length_average
567
589
  )
568
- self.pulse_length_variance = self.ancillary_parameters.decode_std_dev(
569
- "pulse_length", self.pulse_length_variance
590
+ self.pulse_length_std_dev = self.ancillary_parameters.decode_std_dev(
591
+ "pulse_length", pulse_length_variance
570
592
  )
571
593
 
572
594
  self.histogram_flag_array = np.zeros((4, 3600), dtype=np.uint8)
573
595
  # self.unique_block_identifier = np.datetime_as_string(
574
596
  # np.datetime64(int(self.imap_start_time), "ns"), "s"
575
597
  # )
576
- self.flags = np.zeros((17, 3600), dtype=np.uint8)
598
+ self.flags = np.ones((17,), dtype=np.uint8)
577
599
 
578
600
  def output_data(self) -> tuple:
579
601
  """
@@ -0,0 +1,123 @@
1
+ """Module containing the class definition for the HistogramL2 class."""
2
+
3
+ from dataclasses import InitVar, dataclass, field
4
+
5
+ import numpy as np
6
+
7
+
8
+ @dataclass
9
+ class HistogramL2:
10
+ """
11
+ Dataclass describing Histogram L2 data variables and methods.
12
+
13
+ This class collects multiple HistogramL1B classes into one L2 per observational day.
14
+
15
+ flight_software_version : str
16
+ number_of_good_l1b_inputs : int
17
+ number of good-time Level-1B times used for generation of Level-2 data
18
+ total_l1b_inputs : int
19
+ number of all Level-1B times for observational day
20
+ identifier : int
21
+ unique Level-2 histogram identifier
22
+ start_time : numpy.double
23
+ UTC start time of a given observational day
24
+ end_time : numpy.double
25
+ UTC end time of a given observational day
26
+ daily_lightcurve : numpy.ndarray
27
+ arrays for observational-day-accumulated lightcurve
28
+ filter_temperature_average : numpy.ndarray
29
+ observational-day-averaged filter temperature [Celsius deg]
30
+ filter_temperature_variance : numpy.ndarray
31
+ standard deviation for filter temperature [Celsius deg]
32
+ hv_voltage_average : numpy.ndarray
33
+ observational-day-averaged channeltron voltage [volt]
34
+ hv_voltage_variance : numpy.ndarray
35
+ standard deviation for channeltron voltage [volt]
36
+ spin_period_average : numpy.ndarray
37
+ observational-day-averaged spin period [s] (onboard value)
38
+ spin_period_variance : numpy.ndarray
39
+ a standard deviation for spin period [s]
40
+ pulse_length_average : numpy.ndarray
41
+ observational-day-averaged pulse length [μs]
42
+ pulse_length_variance : numpy.ndarray
43
+ standard deviation for pulse length [μs]
44
+ spin_period_ground_average : numpy.ndarray
45
+ observational-day-averaged spin period [s] (ground value)
46
+ spin_period_ground_variance : numpy.ndarray
47
+ a standard deviation for spin period [s]
48
+ position_angle_offset_average : numpy.ndarray
49
+ observational-day-averaged GLOWS angular offset [deg]
50
+ position_angle_offset_variance : numpy.ndarray
51
+ standard deviation for GLOWS angular offset [seg]
52
+ spin_axis_orientation_variance : numpy.ndarray
53
+ standard deviation for spin-axis longitude and latitude [deg]
54
+ spacecraft_location_variance : numpy.ndarray
55
+ standard deviation for ecliptic coordinates [km] of IMAP
56
+ spacecraft_velocity_variance : numpy.ndarray
57
+ standard deviation for IMAP velocity components [km/s]
58
+ spin_axis_orientation_average : numpy.ndarray
59
+ observational-day-averaged spin-axis ecliptic longitude and latitude [deg]
60
+ spacecraft_location_average : numpy.ndarray
61
+ observational-day-averaged Cartesian ecliptic coordinates ⟨X⟩, ⟨Y ⟩, ⟨Z⟩ [km]
62
+ of IMAP
63
+ spacecraft_velocity_average : numpy.ndarray
64
+ observational-day-averaged values ⟨VX ⟩, ⟨VY ⟩, ⟨VZ ⟩ of IMAP velocity
65
+ components [km/s] (Cartesian ecliptic frame)
66
+ bad_time_flag_occurrences : numpy.ndarray
67
+ numbers of occurrences of blocks for each bad-time flag during observational day
68
+ """
69
+
70
+ flight_software_version: str
71
+ number_of_good_l1b_inputs: int
72
+ total_l1b_inputs: int
73
+ # identifier: int # comes from unique_block_identifier
74
+ start_time: np.double
75
+ end_time: np.double
76
+ daily_lightcurve: np.ndarray = field(init=False)
77
+ filter_temperature_average: np.ndarray[np.double]
78
+ filter_temperature_std_dev: np.ndarray[np.double]
79
+ hv_voltage_average: np.ndarray[np.double]
80
+ hv_voltage_std_dev: np.ndarray[np.double]
81
+ spin_period_average: np.ndarray[np.double]
82
+ spin_period_std_dev: np.ndarray[np.double]
83
+ pulse_length_average: np.ndarray[np.double]
84
+ pulse_length_std_dev: np.ndarray[np.double]
85
+ spin_period_ground_average: np.ndarray[np.double]
86
+ spin_period_ground_std_dev: np.ndarray[np.double]
87
+ position_angle_offset_average: np.ndarray[np.double]
88
+ position_angle_offset_std_dev: np.ndarray[np.double]
89
+ spin_axis_orientation_std_dev: np.ndarray[np.double]
90
+ spacecraft_location_std_dev: np.ndarray[np.double]
91
+ spacecraft_velocity_std_dev: np.ndarray[np.double]
92
+ spin_axis_orientation_average: np.ndarray[np.double]
93
+ spacecraft_location_average: np.ndarray[np.double]
94
+ spacecraft_velocity_average: np.ndarray[np.double]
95
+ bad_time_flag_occurrences: np.ndarray
96
+ histogram: InitVar[np.ndarray]
97
+
98
+ def __post_init__(self, histogram: np.ndarray) -> None:
99
+ """
100
+ Post-initialization method to generate the daily light curve from one histogram.
101
+
102
+ Parameters
103
+ ----------
104
+ histogram : numpy.ndarray
105
+ Histogram data from L1B, of shape (bins,) where bins is nominally 3600.
106
+ """
107
+ self.daily_lightcurve = self.generate_lightcurve(histogram)
108
+
109
+ def generate_lightcurve(self, histogram: np.ndarray) -> np.ndarray:
110
+ """
111
+ Given an array of (n, bins) histograms, generate one lightcurve of size (bins).
112
+
113
+ Parameters
114
+ ----------
115
+ histogram : numpy.ndarray
116
+ Histogram data from L1B, of shape (bins,) where bins is nominally 3600.
117
+
118
+ Returns
119
+ -------
120
+ numpy.ndarray
121
+ Lightcurve of size (bins).
122
+ """
123
+ return np.zeros(3600) # type: ignore[no-any-return]
@@ -33,7 +33,7 @@ def hi_l1a(packet_file_path: Union[str, Path], data_version: str) -> list[xr.Dat
33
33
  List of processed xarray dataset.
34
34
  """
35
35
  packet_def_file = (
36
- imap_module_directory / "hi/packet_definitions/hi_packet_definition.xml"
36
+ imap_module_directory / "hi/packet_definitions/TLM_HI_COMBINED_SCI.xml"
37
37
  )
38
38
  datasets_by_apid = packet_file_to_datasets(
39
39
  packet_file=packet_file_path, xtce_packet_definition=packet_def_file
@@ -42,18 +42,18 @@ def hi_l1a(packet_file_path: Union[str, Path], data_version: str) -> list[xr.Dat
42
42
  # Process science to l1a.
43
43
  processed_data = []
44
44
  for apid in datasets_by_apid:
45
- if apid == HIAPID.H45_SCI_CNT:
45
+ if apid in [HIAPID.H45_SCI_CNT, HIAPID.H90_SCI_CNT]:
46
46
  logger.info(
47
47
  "Processing histogram data for [%s] packets", HIAPID.H45_SCI_CNT.name
48
48
  )
49
49
  data = hist_create_dataset(datasets_by_apid[apid])
50
- elif apid == HIAPID.H45_SCI_DE:
50
+ elif apid in [HIAPID.H45_SCI_DE, HIAPID.H90_SCI_DE]:
51
51
  logger.info(
52
52
  "Processing direct event data for [%s] packets", HIAPID.H45_SCI_DE.name
53
53
  )
54
54
 
55
55
  data = science_direct_event(datasets_by_apid[apid])
56
- elif apid == HIAPID.H45_APP_NHK:
56
+ elif apid in [HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK]:
57
57
  logger.info(
58
58
  "Processing housekeeping data for [%s] packets", HIAPID.H45_APP_NHK.name
59
59
  )
@@ -2,36 +2,36 @@
2
2
 
3
3
  import numpy as np
4
4
  import xarray as xr
5
+ from numpy._typing import NDArray
5
6
 
6
7
  from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
7
- from imap_processing.utils import convert_to_binary_string
8
8
 
9
9
  # define the names of the 24 counter arrays
10
10
  # contained in the histogram packet
11
11
  QUALIFIED_COUNTERS = (
12
- "qual_ab",
13
- "qual_c1c2",
14
- "qual_ac1",
15
- "qual_bc1",
16
- "qual_abc1",
17
- "qual_ac1c2",
18
- "qual_bc1c2",
19
- "qual_abc1c2",
12
+ "ab_qualified",
13
+ "c1c2_qualified",
14
+ "ac1_qualified",
15
+ "bc1_qualified",
16
+ "abc1_qualified",
17
+ "ac1c2_qualified",
18
+ "bc1c2_qualified",
19
+ "abc1c2_qualified",
20
20
  )
21
21
  LONG_COUNTERS = (
22
- "long_a",
23
- "long_b",
24
- "long_c",
25
- "long_ab",
26
- "long_c1c2",
27
- "long_ac1",
28
- "long_bc1",
29
- "long_abc1",
30
- "long_ac1c2",
31
- "long_bc1c2",
32
- "long_abc1c2",
22
+ "a_first_only",
23
+ "b_first_only",
24
+ "c_first_only",
25
+ "ab_long",
26
+ "c1c2_long",
27
+ "ac1_long",
28
+ "bc1_long",
29
+ "abc1_long",
30
+ "ac1c2_long",
31
+ "bc1c2_long",
32
+ "abc1c2_long",
33
33
  )
34
- TOTAL_COUNTERS = ("total_a", "total_b", "total_c", "fee_de_sent", "fee_de_recd")
34
+ TOTAL_COUNTERS = ("a_total", "b_total", "c_total", "fee_de_recd", "fee_de_sent")
35
35
 
36
36
 
37
37
  def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
@@ -41,99 +41,56 @@ def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
41
41
  Parameters
42
42
  ----------
43
43
  input_ds : xarray.Dataset
44
- Dataset of packets.
44
+ Dataset of packets generated using the
45
+ `imap_processing.utils.packet_file_to_datasets` function.
45
46
 
46
47
  Returns
47
48
  -------
48
49
  dataset : xarray.Dataset
49
50
  Dataset with all metadata field data in xr.DataArray.
50
51
  """
51
- dataset = allocate_histogram_dataset(len(input_ds.epoch))
52
-
53
- # TODO: Move into the allocate dataset function?
54
- dataset["epoch"].data[:] = input_ds["epoch"].data
55
- dataset["ccsds_met"].data = input_ds["ccsds_met"].data
56
- dataset["esa_stepping_num"].data = input_ds["esa_step"].data
57
-
58
- # unpack the packets data into the Dataset
59
- # (npackets, 24 * 90 * 12)
60
- # TODO: Look into avoiding the for-loops below
61
- # It seems like we could try to reshape the arrays and do some numpy
62
- # broadcasting rather than for-loops directly here
63
- for i_epoch, counters_bytes_data in enumerate(input_ds["counters"].data):
64
- binary_str_val = convert_to_binary_string(counters_bytes_data)
65
- # unpack 24 arrays of 90 12-bit unsigned integers
66
- counter_ints = [
67
- int(binary_str_val[i * 12 : (i + 1) * 12], 2) for i in range(90 * 24)
68
- ]
69
- # populate the dataset with the unpacked integers
70
- for i_counter, counter in enumerate(
71
- (*QUALIFIED_COUNTERS, *LONG_COUNTERS, *TOTAL_COUNTERS)
72
- ):
73
- dataset[counter][i_epoch] = counter_ints[
74
- i_counter * 90 : (i_counter + 1) * 90
75
- ]
76
-
77
- return dataset
78
-
79
-
80
- def allocate_histogram_dataset(num_packets: int) -> xr.Dataset:
81
- """
82
- Allocate empty xarray.Dataset for specified number of Hi Histogram packets.
83
-
84
- Parameters
85
- ----------
86
- num_packets : int
87
- The number of Hi Histogram packets to allocate space for
88
- in the xarray.Dataset.
89
-
90
- Returns
91
- -------
92
- dataset : xarray.Dataset
93
- Empty xarray.Dataset ready to be filled with packet data.
94
- """
95
52
  attr_mgr = ImapCdfAttributes()
96
53
  attr_mgr.add_instrument_global_attrs(instrument="hi")
97
- attr_mgr.load_variable_attributes("imap_hi_variable_attrs.yaml")
98
- # preallocate the xr.DataArrays for all CDF attributes based on number of packets
99
- coords = dict()
100
- coords["epoch"] = xr.DataArray(
101
- np.empty(num_packets, dtype="datetime64[ns]"),
102
- name="epoch",
103
- dims=["epoch"],
104
- attrs=attr_mgr.get_variable_attributes("epoch"),
105
- )
106
- # Histogram data is binned in 90, 4-degree bins
107
- coords["angle"] = xr.DataArray(
108
- np.arange(2, 360, 4),
109
- name="angle",
110
- dims=["angle"],
111
- attrs=attr_mgr.get_variable_attributes("hi_hist_angle"),
112
- )
54
+ attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
113
55
 
114
- data_vars = dict()
115
- # Generate label variables
116
- data_vars["angle_label"] = xr.DataArray(
117
- coords["angle"].values.astype(str),
118
- name="angle_label",
119
- dims=["angle"],
120
- attrs=attr_mgr.get_variable_attributes(
121
- "hi_hist_angle_label", check_schema=False
122
- ),
123
- )
124
- # Other data variables
125
- data_vars["ccsds_met"] = xr.DataArray(
126
- np.empty(num_packets, dtype=np.uint32),
127
- dims=["epoch"],
128
- attrs=attr_mgr.get_variable_attributes("hi_hist_ccsds_met"),
56
+ # Rename shcoarse variable (do this first since it copies the input_ds)
57
+ dataset = input_ds.rename_vars({"shcoarse": "ccsds_met"})
58
+
59
+ dataset.epoch.attrs.update(
60
+ attr_mgr.get_variable_attributes("epoch"),
129
61
  )
130
- data_vars["esa_stepping_num"] = xr.DataArray(
131
- np.empty(num_packets, dtype=np.uint8),
132
- dims=["epoch"],
133
- attrs=attr_mgr.get_variable_attributes("hi_hist_esa_step"),
62
+ # Add the hist_angle coordinate
63
+ # Histogram data is binned in 90, 4-degree bins
64
+ attrs = attr_mgr.get_variable_attributes("hi_hist_angle")
65
+ dataset.coords.update(
66
+ {
67
+ "angle": xr.DataArray(
68
+ np.arange(2, 360, 4),
69
+ name="angle",
70
+ dims=["angle"],
71
+ attrs=attrs,
72
+ )
73
+ }
134
74
  )
135
-
136
- # Allocate xarray.DataArray objects for the 24 90-element histogram counters
75
+ # Update existing variable attributes
76
+ for var_name in [
77
+ "version",
78
+ "type",
79
+ "sec_hdr_flg",
80
+ "pkt_apid",
81
+ "seq_flgs",
82
+ "src_seq_ctr",
83
+ "pkt_len",
84
+ "ccsds_met",
85
+ "esa_step",
86
+ "num_of_spins",
87
+ "cksum",
88
+ ]:
89
+ attrs = attr_mgr.get_variable_attributes(f"hi_hist_{var_name}")
90
+ dataset.data_vars[var_name].attrs.update(attrs)
91
+
92
+ new_vars = dict()
93
+ # Populate 90-element histogram counters
137
94
  default_counter_attrs = attr_mgr.get_variable_attributes("hi_hist_counters")
138
95
  for counter_name in (*QUALIFIED_COUNTERS, *LONG_COUNTERS, *TOTAL_COUNTERS):
139
96
  # Inject counter name into generic counter attributes
@@ -141,15 +98,56 @@ def allocate_histogram_dataset(num_packets: int) -> xr.Dataset:
141
98
  for key, val in counter_attrs.items():
142
99
  if isinstance(val, str) and "{counter_name}" in val:
143
100
  counter_attrs[key] = val.format(counter_name=counter_name)
144
- data_vars[counter_name] = xr.DataArray(
145
- data=np.empty((num_packets, len(coords["angle"])), np.uint16),
101
+ # Instantiate the counter DataArray
102
+ new_vars[counter_name] = xr.DataArray(
103
+ data=unpack_hist_counter(input_ds[counter_name].data.sum()),
146
104
  dims=["epoch", "angle"],
147
105
  attrs=counter_attrs,
148
106
  )
149
107
 
150
- dataset = xr.Dataset(
151
- data_vars=data_vars,
152
- coords=coords,
153
- attrs=attr_mgr.get_global_attributes("imap_hi_l1a_hist_attrs"),
108
+ # Generate label variable for angle coordinate
109
+ new_vars["angle_label"] = xr.DataArray(
110
+ dataset.coords["angle"].values.astype(str),
111
+ name="angle_label",
112
+ dims=["angle"],
113
+ attrs=attr_mgr.get_variable_attributes(
114
+ "hi_hist_angle_label", check_schema=False
115
+ ),
154
116
  )
117
+
118
+ dataset.update(new_vars)
119
+ dataset.attrs.update(attr_mgr.get_global_attributes("imap_hi_l1a_hist_attrs"))
120
+
155
121
  return dataset
122
+
123
+
124
+ def unpack_hist_counter(counter_bytes: bytes) -> NDArray[np.uint16]:
125
+ """
126
+ Unpack Hi SCI_CNT counter data for a single counter.
127
+
128
+ Parameters
129
+ ----------
130
+ counter_bytes : bytes
131
+ Sum individual bytes for all epochs of a Hi SCI_CNT counter.
132
+
133
+ Returns
134
+ -------
135
+ output_array : numpy.ndarray[numpy.uint16]
136
+ The unpacked 12-bit unsigned integers for the input bytes. The
137
+ output array has a shape of (n, 90) where n is the number of SCI_CNT
138
+ packets in the input dataset.
139
+ """
140
+ # Interpret bytes for all epochs of current counter as uint8 array
141
+ counter_uint8 = np.frombuffer(counter_bytes, dtype=np.uint8)
142
+ # Split into triplets of upper-byte, split-byte and lower-byte arrays
143
+ upper_uint8, split_unit8, lower_uint8 = np.reshape(
144
+ counter_uint8, (3, -1), order="F"
145
+ ).astype(np.uint16)
146
+ # Compute even indexed uint12 values from upper-byte and first 4-bits of
147
+ # split-byte
148
+ even_uint12 = (upper_uint8 << 4) + (split_unit8 >> 4)
149
+ # Compute odd indexed uint12 values from lower 4-bits of split-byte and
150
+ # lower-byte
151
+ odd_uint12 = ((split_unit8 & (2**4 - 1)) << 8) + lower_uint8
152
+ output_array = np.column_stack((even_uint12, odd_uint12)).reshape(-1, 90)
153
+ return output_array