imap-processing 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (124) hide show
  1. imap_processing/__init__.py +1 -1
  2. imap_processing/_version.py +2 -2
  3. imap_processing/ccsds/excel_to_xtce.py +34 -2
  4. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
  5. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
  6. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
  7. imap_processing/cdf/config/imap_hi_variable_attrs.yaml +36 -8
  8. imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
  9. imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +7 -7
  10. imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +32 -33
  11. imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
  12. imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
  13. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +133 -78
  14. imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
  15. imap_processing/cdf/imap_cdf_manager.py +31 -27
  16. imap_processing/cli.py +12 -10
  17. imap_processing/codice/codice_l1a.py +151 -61
  18. imap_processing/codice/constants.py +1 -1
  19. imap_processing/codice/decompress.py +4 -9
  20. imap_processing/codice/utils.py +1 -0
  21. imap_processing/glows/l1b/glows_l1b.py +3 -3
  22. imap_processing/glows/l1b/glows_l1b_data.py +59 -37
  23. imap_processing/glows/l2/glows_l2_data.py +123 -0
  24. imap_processing/hi/l1a/histogram.py +1 -1
  25. imap_processing/hi/l1a/science_direct_event.py +1 -1
  26. imap_processing/hi/l1b/hi_l1b.py +85 -11
  27. imap_processing/hi/l1c/hi_l1c.py +23 -1
  28. imap_processing/hi/utils.py +1 -1
  29. imap_processing/hit/hit_utils.py +221 -0
  30. imap_processing/hit/l0/constants.py +118 -0
  31. imap_processing/hit/l0/decom_hit.py +186 -153
  32. imap_processing/hit/l1a/hit_l1a.py +20 -175
  33. imap_processing/hit/l1b/hit_l1b.py +33 -153
  34. imap_processing/idex/idex_l1a.py +10 -9
  35. imap_processing/lo/l0/decompression_tables/decompression_tables.py +1 -1
  36. imap_processing/lo/l0/lo_science.py +1 -1
  37. imap_processing/lo/packet_definitions/lo_xtce.xml +1 -3296
  38. imap_processing/mag/l0/decom_mag.py +4 -3
  39. imap_processing/mag/l1a/mag_l1a.py +11 -11
  40. imap_processing/mag/l1b/mag_l1b.py +89 -7
  41. imap_processing/spice/geometry.py +126 -4
  42. imap_processing/swapi/l1/swapi_l1.py +1 -1
  43. imap_processing/swapi/l2/swapi_l2.py +1 -1
  44. imap_processing/swe/l1b/swe_l1b_science.py +8 -8
  45. imap_processing/tests/ccsds/test_data/expected_output.xml +1 -0
  46. imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
  47. imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
  48. imap_processing/tests/codice/conftest.py +1 -17
  49. imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
  50. imap_processing/tests/codice/test_codice_l0.py +8 -2
  51. imap_processing/tests/codice/test_codice_l1a.py +127 -107
  52. imap_processing/tests/codice/test_codice_l1b.py +1 -0
  53. imap_processing/tests/codice/test_decompress.py +7 -7
  54. imap_processing/tests/conftest.py +54 -15
  55. imap_processing/tests/glows/conftest.py +6 -0
  56. imap_processing/tests/glows/test_glows_l1b.py +9 -9
  57. imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
  58. imap_processing/tests/glows/test_glows_l2_data.py +0 -0
  59. imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
  60. imap_processing/tests/hi/test_hi_l1b.py +71 -1
  61. imap_processing/tests/hi/test_hi_l1c.py +10 -2
  62. imap_processing/tests/hi/test_utils.py +4 -3
  63. imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +84 -35
  64. imap_processing/tests/hit/test_hit_l1a.py +2 -197
  65. imap_processing/tests/hit/test_hit_l1b.py +156 -25
  66. imap_processing/tests/hit/test_hit_utils.py +218 -0
  67. imap_processing/tests/idex/conftest.py +1 -1
  68. imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
  69. imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
  70. imap_processing/tests/idex/test_idex_l0.py +3 -3
  71. imap_processing/tests/idex/test_idex_l1a.py +1 -1
  72. imap_processing/tests/lo/test_lo_science.py +2 -2
  73. imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
  74. imap_processing/tests/mag/test_mag_l1b.py +59 -3
  75. imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
  76. imap_processing/tests/spice/test_geometry.py +84 -4
  77. imap_processing/tests/swe/conftest.py +33 -0
  78. imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
  79. imap_processing/tests/swe/test_swe_l1b.py +29 -8
  80. imap_processing/tests/test_utils.py +1 -1
  81. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
  82. imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
  83. imap_processing/tests/ultra/unit/test_de.py +108 -0
  84. imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
  85. imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
  86. imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +21 -11
  87. imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +9 -44
  88. imap_processing/ultra/constants.py +8 -3
  89. imap_processing/ultra/l1b/de.py +174 -30
  90. imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
  91. imap_processing/ultra/l1b/ultra_l1b_extended.py +21 -14
  92. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +70 -119
  93. {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/METADATA +15 -14
  94. {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/RECORD +98 -113
  95. imap_processing/cdf/cdf_attribute_manager.py +0 -322
  96. imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
  97. imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  98. imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
  99. imap_processing/hit/l0/data_classes/science_packet.py +0 -259
  100. imap_processing/hit/l0/utils/hit_base.py +0 -57
  101. imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
  102. imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
  103. imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
  104. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
  105. imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
  106. imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
  107. imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
  108. imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
  109. imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
  110. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
  111. imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
  112. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
  113. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
  114. imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
  115. imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
  116. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
  117. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
  118. imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
  119. imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
  120. imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
  121. /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
  122. {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/LICENSE +0 -0
  123. {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/WHEEL +0 -0
  124. {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/entry_points.txt +0 -0
@@ -1,18 +1,17 @@
1
1
  """
2
2
  Perform CoDICE l1a processing.
3
3
 
4
- This module processes decommutated CoDICE packets and creates L1a data products.
4
+ This module processes CoDICE L0 files and creates L1a data products.
5
5
 
6
6
  Notes
7
7
  -----
8
- from imap_processing.codice.codice_l0 import decom_packets
9
8
  from imap_processing.codice.codice_l1a import process_codice_l1a
10
- packets = decom_packets(packet_file)
11
- dataset = process_codice_l1a(packets)
9
+ processed_datasets = process_codice_l1a(path_to_l0_file)
12
10
  """
13
11
 
14
12
  from __future__ import annotations
15
13
 
14
+ import ast
16
15
  import logging
17
16
  from pathlib import Path
18
17
  from typing import Any
@@ -28,12 +27,10 @@ from imap_processing.codice import constants
28
27
  from imap_processing.codice.codice_l0 import decom_packets
29
28
  from imap_processing.codice.decompress import decompress
30
29
  from imap_processing.codice.utils import CODICEAPID
31
- from imap_processing.utils import convert_to_binary_string
32
30
 
33
31
  logger = logging.getLogger(__name__)
34
32
  logger.setLevel(logging.INFO)
35
33
 
36
- # TODO: Add support for decomming multiple APIDs from a single file
37
34
  # TODO: Determine what should go in event data CDF and how it should be
38
35
  # structured.
39
36
 
@@ -87,23 +84,39 @@ class CoDICEL1aPipeline:
87
84
  self.plan_step = plan_step
88
85
  self.view_id = view_id
89
86
 
90
- def decompress_data(self, science_values: str) -> None:
87
+ def decompress_data(self, science_values: list[str]) -> None:
91
88
  """
92
89
  Perform decompression on the data.
93
90
 
94
- The science data within the packet is a compressed, binary string of
91
+ The science data within the packet is a compressed byte string of
95
92
  values. Apply the appropriate decompression algorithm to get an array
96
93
  of decompressed values.
97
94
 
98
95
  Parameters
99
96
  ----------
100
- science_values : str
101
- A string of binary data representing the science values of the data.
97
+ science_values : list[str]
98
+ A list of byte strings representing the science values of the data
99
+ for each packet.
102
100
  """
103
- compression_algorithm = constants.LO_COMPRESSION_ID_LOOKUP[self.view_id]
101
+ # The compression algorithm depends on the instrument and view ID
102
+ if self.config["instrument"] == "lo":
103
+ compression_algorithm = constants.LO_COMPRESSION_ID_LOOKUP[self.view_id]
104
+ elif self.config["instrument"] == "hi":
105
+ compression_algorithm = constants.HI_COMPRESSION_ID_LOOKUP[self.view_id]
106
+
107
+ self.raw_data = []
108
+ for packet_data, byte_count in zip(
109
+ science_values, self.dataset.byte_count.data
110
+ ):
111
+ # Convert from numpy array to byte object
112
+ values = ast.literal_eval(str(packet_data))
104
113
 
105
- # Decompress the binary string into a list of integers
106
- self.data = decompress(science_values, compression_algorithm)
114
+ # Only use the values up to the byte count. Bytes after this are
115
+ # used as padding and are not needed
116
+ values = values[:byte_count]
117
+
118
+ decompressed_values = decompress(values, compression_algorithm)
119
+ self.raw_data.append(decompressed_values)
107
120
 
108
121
  def define_coordinates(self) -> None:
109
122
  """
@@ -115,7 +128,7 @@ class CoDICEL1aPipeline:
115
128
 
116
129
  for name in self.config["coords"]:
117
130
  if name == "epoch":
118
- values = self.packet_dataset.epoch
131
+ values = self.dataset.epoch.data
119
132
  elif name == "inst_az":
120
133
  values = np.arange(self.config["num_positions"])
121
134
  elif name == "spin_sector":
@@ -145,7 +158,7 @@ class CoDICEL1aPipeline:
145
158
 
146
159
  Returns
147
160
  -------
148
- dataset : xarray.Dataset
161
+ processed_dataset : xarray.Dataset
149
162
  The 'final' ``xarray`` dataset.
150
163
  """
151
164
  # Create the main dataset to hold all the variables
@@ -154,12 +167,18 @@ class CoDICEL1aPipeline:
154
167
  attrs=self.cdf_attrs.get_global_attributes(self.config["dataset_name"]),
155
168
  )
156
169
 
157
- # Create a data variable for each counter
158
- for variable_data, variable_name in zip(
159
- self.data, self.config["variable_names"]
170
+ # Stack the data so that it is easier to reshape and iterate over
171
+ all_data = np.stack(self.data)
172
+
173
+ # The dimension of all data is (epoch, num_counters, num_positions,
174
+ # num_spin_sectors, num_energy_steps) (or may be slightly different
175
+ # depending on the data product). In any case, iterate over the
176
+ # num_counters dimension to isolate the data for each counter so
177
+ # that it can be placed in a CDF data variable.
178
+ for counter, variable_name in zip(
179
+ range(all_data.shape[1]), self.config["variable_names"]
160
180
  ):
161
- # Reshape to 4 dimensions to allow for epoch dimension
162
- reshaped_variable_data = np.expand_dims(variable_data, axis=0)
181
+ counter_data = all_data[:, counter, :, :, :]
163
182
 
164
183
  # Get the CDF attributes
165
184
  descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1]
@@ -168,7 +187,7 @@ class CoDICEL1aPipeline:
168
187
 
169
188
  # Create the CDF data variable
170
189
  dataset[variable_name] = xr.DataArray(
171
- reshaped_variable_data,
190
+ counter_data,
172
191
  name=variable_name,
173
192
  dims=self.config["dims"],
174
193
  attrs=attrs,
@@ -322,32 +341,51 @@ class CoDICEL1aPipeline:
322
341
  3D arrays representing dimensions such as spin sectors, positions, and
323
342
  energies (depending on the data product).
324
343
  """
344
+ self.data = []
345
+
325
346
  # For CoDICE-lo, data are a 3D arrays with a shape representing
326
347
  # [<num_positions>,<num_spin_sectors>,<num_energy_steps>]
327
348
  if self.config["instrument"] == "lo":
328
- self.data = np.array(self.data, dtype=np.uint32).reshape(
329
- (
330
- self.config["num_counters"],
331
- self.config["num_positions"],
332
- self.config["num_spin_sectors"],
333
- self.config["num_energy_steps"],
334
- )
335
- )
349
+ for packet_data in self.raw_data:
350
+ if packet_data:
351
+ reshaped_packet_data = np.array(
352
+ packet_data, dtype=np.uint32
353
+ ).reshape(
354
+ (
355
+ self.config["num_counters"],
356
+ self.config["num_positions"],
357
+ self.config["num_spin_sectors"],
358
+ self.config["num_energy_steps"],
359
+ )
360
+ )
361
+ self.data.append(reshaped_packet_data)
362
+ else:
363
+ self.data.append(None)
336
364
 
337
365
  # For CoDICE-hi, data are a 3D array with a shape representing
338
366
  # [<num_energy_steps>,<num_positions>,<num_spin_sectors>]
339
367
  elif self.config["instrument"] == "hi":
340
- self.data = np.array(self.data, dtype=np.uint32).reshape(
341
- (
342
- self.config["num_counters"],
343
- self.config["num_energy_steps"],
344
- self.config["num_positions"],
345
- self.config["num_spin_sectors"],
346
- )
347
- )
368
+ for packet_data in self.raw_data:
369
+ if packet_data:
370
+ reshaped_packet_data = np.array(
371
+ packet_data, dtype=np.uint32
372
+ ).reshape(
373
+ (
374
+ self.config["num_counters"],
375
+ self.config["num_energy_steps"],
376
+ self.config["num_positions"],
377
+ self.config["num_spin_sectors"],
378
+ )
379
+ )
380
+ self.data.append(reshaped_packet_data)
381
+ else:
382
+ self.data.append(None)
383
+
384
+ # No longer need to keep the raw data around
385
+ del self.raw_data
348
386
 
349
387
  def set_data_product_config(
350
- self, apid: int, packet: xr.Dataset, data_version: str
388
+ self, apid: int, dataset: xr.Dataset, data_version: str
351
389
  ) -> None:
352
390
  """
353
391
  Set the various settings for defining the data products.
@@ -356,14 +394,14 @@ class CoDICEL1aPipeline:
356
394
  ----------
357
395
  apid : int
358
396
  The APID of interest.
359
- packet : xarray.Dataset
360
- A packet for the APID of interest.
397
+ dataset : xarray.Dataset
398
+ The dataset for the APID of interest.
361
399
  data_version : str
362
400
  Version of the data product being created.
363
401
  """
364
402
  # Set the packet dataset so that it can be easily called from various
365
403
  # methods
366
- self.packet_dataset = packet
404
+ self.dataset = dataset
367
405
 
368
406
  # Set various configurations of the data product
369
407
  self.config: dict[str, Any] = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore
@@ -473,7 +511,7 @@ def create_hskp_dataset(
473
511
  return dataset
474
512
 
475
513
 
476
- def get_params(packet: xr.Dataset) -> tuple[int, int, int, int]:
514
+ def get_params(dataset: xr.Dataset) -> tuple[int, int, int, int]:
477
515
  """
478
516
  Return the four 'main' parameters used for l1a processing.
479
517
 
@@ -483,8 +521,10 @@ def get_params(packet: xr.Dataset) -> tuple[int, int, int, int]:
483
521
 
484
522
  Parameters
485
523
  ----------
486
- packet : xarray.Dataset
487
- A packet for the APID of interest.
524
+ dataset : xarray.Dataset
525
+ The dataset for the APID of interest. We expect each packet in the
526
+ dataset to have the same values for the four main parameters, so the
527
+ first index of the dataset can be used to determine them.
488
528
 
489
529
  Returns
490
530
  -------
@@ -502,15 +542,37 @@ def get_params(packet: xr.Dataset) -> tuple[int, int, int, int]:
502
542
  view_id : int
503
543
  Provides information about how data was collapsed and/or compressed.
504
544
  """
505
- table_id = int(packet.table_id.data)
506
- plan_id = int(packet.plan_id.data)
507
- plan_step = int(packet.plan_step.data)
508
- view_id = int(packet.view_id.data)
545
+ table_id = int(dataset.table_id.data[0])
546
+ plan_id = int(dataset.plan_id.data[0])
547
+ plan_step = int(dataset.plan_step.data[0])
548
+ view_id = int(dataset.view_id.data[0])
509
549
 
510
550
  return table_id, plan_id, plan_step, view_id
511
551
 
512
552
 
513
- def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset:
553
+ def log_dataset_info(datasets: dict[int, xr.Dataset]) -> None:
554
+ """
555
+ Log info about the input data to help with tracking and/or debugging.
556
+
557
+ Parameters
558
+ ----------
559
+ datasets : dict[int, xarray.Dataset]
560
+ Mapping from apid to ``xarray`` dataset, one dataset per apid.
561
+ """
562
+ launch_time = np.datetime64("2010-01-01T00:01:06.184", "ns")
563
+ logger.info("\nThis input file contains the following APIDs:\n")
564
+ for apid in datasets:
565
+ num_packets = len(datasets[apid].epoch.data)
566
+ time_deltas = [np.timedelta64(item, "ns") for item in datasets[apid].epoch.data]
567
+ times = [launch_time + delta for delta in time_deltas]
568
+ start = np.datetime_as_string(times[0])
569
+ end = np.datetime_as_string(times[-1])
570
+ logger.info(
571
+ f"{CODICEAPID(apid).name}: {num_packets} packets spanning {start} to {end}"
572
+ )
573
+
574
+
575
+ def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
514
576
  """
515
577
  Will process CoDICE l0 data to create l1a data products.
516
578
 
@@ -523,38 +585,66 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset:
523
585
 
524
586
  Returns
525
587
  -------
526
- dataset : xarray.Dataset
527
- The ``xarray`` dataset containing the science data and supporting metadata.
588
+ processed_datasets : list[xarray.Dataset]
589
+ A list of the ``xarray`` datasets containing the science data and
590
+ supporting metadata.
528
591
  """
529
592
  # Decom the packets, group data by APID, and sort by time
530
593
  datasets = decom_packets(file_path)
531
594
 
595
+ # Log some information about the contents of the data
596
+ log_dataset_info(datasets)
597
+
598
+ # Placeholder to hold the final, processed datasets
599
+ processed_datasets = []
600
+
601
+ # Process each APID separately
532
602
  for apid in datasets:
533
- packet_dataset = datasets[apid]
603
+ dataset = datasets[apid]
534
604
  logger.info(f"\nProcessing {CODICEAPID(apid).name} packet")
535
605
 
606
+ # Housekeeping data
536
607
  if apid == CODICEAPID.COD_NHK:
537
- dataset = create_hskp_dataset(packet_dataset, data_version)
608
+ processed_dataset = create_hskp_dataset(dataset, data_version)
609
+ logger.info(f"\nFinal data product:\n{processed_dataset}\n")
538
610
 
611
+ # Event data
539
612
  elif apid in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]:
540
- dataset = create_event_dataset(apid, packet_dataset, data_version)
613
+ processed_dataset = create_event_dataset(apid, dataset, data_version)
614
+ logger.info(f"\nFinal data product:\n{processed_dataset}\n")
541
615
 
616
+ # Everything else
542
617
  elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING:
543
618
  # Extract the data
544
- science_values = packet_dataset.data.data[0]
545
- science_values = convert_to_binary_string(science_values)
619
+ science_values = [packet.data for packet in dataset.data]
546
620
 
547
621
  # Get the four "main" parameters for processing
548
- table_id, plan_id, plan_step, view_id = get_params(packet_dataset)
622
+ table_id, plan_id, plan_step, view_id = get_params(dataset)
549
623
 
550
624
  # Run the pipeline to create a dataset for the product
551
625
  pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id)
552
- pipeline.set_data_product_config(apid, packet_dataset, data_version)
626
+ pipeline.set_data_product_config(apid, dataset, data_version)
553
627
  pipeline.decompress_data(science_values)
554
628
  pipeline.reshape_data()
555
629
  pipeline.define_coordinates()
556
- dataset = pipeline.define_data_variables()
630
+ processed_dataset = pipeline.define_data_variables()
557
631
 
558
- logger.info(f"\nFinal data product:\n{dataset}\n")
632
+ logger.info(f"\nFinal data product:\n{processed_dataset}\n")
559
633
 
560
- return dataset
634
+ # TODO: Still need to implement I-ALiRT and hi-priorities data products
635
+ elif apid in [
636
+ CODICEAPID.COD_HI_INST_COUNTS_PRIORITIES,
637
+ CODICEAPID.COD_HI_IAL,
638
+ CODICEAPID.COD_LO_IAL,
639
+ ]:
640
+ logger.info("\tStill need to properly implement")
641
+ processed_dataset = None
642
+
643
+ # For APIDs that don't require processing
644
+ else:
645
+ logger.info(f"\t{apid} does not require processing")
646
+ continue
647
+
648
+ processed_datasets.append(processed_dataset)
649
+
650
+ return processed_datasets
@@ -124,7 +124,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
124
124
  "instrument": "hi",
125
125
  "num_counters": 3,
126
126
  "num_energy_steps": 1, # TODO: Double check with Joey
127
- "num_positions": 16, # TODO: Double check with Joey
127
+ "num_positions": 12, # TODO: Double check with Joey
128
128
  "num_spin_sectors": 1,
129
129
  "support_variables": [], # No support variables for this one
130
130
  "variable_names": HI_INST_COUNTS_SINGLES_VARIABLE_NAMES,
@@ -94,9 +94,9 @@ def _apply_lzma_lossless(compressed_bytes: bytes) -> bytes:
94
94
  return lzma_decompressed_values
95
95
 
96
96
 
97
- def decompress(compressed_binary: str, algorithm: IntEnum) -> list[int]:
97
+ def decompress(compressed_bytes: bytes, algorithm: IntEnum) -> list[int]:
98
98
  """
99
- Perform decompression on a binary string into a list of integers.
99
+ Perform decompression on a byte stream into a list of integers.
100
100
 
101
101
  Apply the appropriate decompression algorithm(s) based on the value
102
102
  of the ``algorithm`` attribute. One or more individual algorithms may be
@@ -104,8 +104,8 @@ def decompress(compressed_binary: str, algorithm: IntEnum) -> list[int]:
104
104
 
105
105
  Parameters
106
106
  ----------
107
- compressed_binary : str
108
- The compressed binary string.
107
+ compressed_bytes : bytes
108
+ The compressed byte stream.
109
109
  algorithm : int
110
110
  The algorithm to apply. Supported algorithms are provided in the
111
111
  ``codice_utils.CoDICECompression`` class.
@@ -115,11 +115,6 @@ def decompress(compressed_binary: str, algorithm: IntEnum) -> list[int]:
115
115
  decompressed_values : list[int]
116
116
  The 24- or 32-bit decompressed values.
117
117
  """
118
- # Convert the binary string to a byte stream
119
- compressed_bytes = int(compressed_binary, 2).to_bytes(
120
- (len(compressed_binary) + 7) // 8, byteorder="big"
121
- )
122
-
123
118
  # Apply the appropriate decompression algorithm
124
119
  if algorithm == CoDICECompression.NO_COMPRESSION:
125
120
  decompressed_values = list(compressed_bytes)
@@ -43,6 +43,7 @@ class CODICEAPID(IntEnum):
43
43
  COD_HI_INST_COUNTS_SINGLES = 1171
44
44
  COD_HI_OMNI_SPECIES_COUNTS = 1172
45
45
  COD_HI_SECT_SPECIES_COUNTS = 1173
46
+ COD_HI_INST_COUNTS_PRIORITIES = 1174
46
47
  COD_CSTOL_CONFIG = 2457
47
48
 
48
49
 
@@ -255,10 +255,10 @@ def process_histogram(l1a: xr.Dataset) -> xr.Dataset:
255
255
  "imap_spin_angle_bin_cntr": ["bins"],
256
256
  "histogram_flag_array": ["bad_angle_flags", "bins"],
257
257
  "spacecraft_location_average": ["ecliptic"],
258
- "spacecraft_location_variance": ["ecliptic"],
258
+ "spacecraft_location_std_dev": ["ecliptic"],
259
259
  "spacecraft_velocity_average": ["ecliptic"],
260
- "spacecraft_velocity_variance": ["ecliptic"],
261
- "flags": ["flag_dim", "bins"],
260
+ "spacecraft_velocity_std_dev": ["ecliptic"],
261
+ "flags": ["flag_dim"],
262
262
  }
263
263
 
264
264
  # For each attribute, retrieve the dims from output_dimension_mapping or use an
@@ -417,19 +417,19 @@ class HistogramL1B:
417
417
  IMAP spin angle ψ for bin centers
418
418
  filter_temperature_average
419
419
  block-averaged value, decoded to Celsius degrees using Eq. (47)
420
- filter_temperature_variance
420
+ filter_temperature_std_dev
421
421
  standard deviation (1 sigma), decoded to Celsius degrees using Eq. (51)
422
422
  hv_voltage_average
423
423
  block-averaged value, decoded to volts using Eq. (47)
424
- hv_voltage_variance
424
+ hv_voltage_std_dev
425
425
  standard deviation (1 sigma), decoded to volts using Eq. (51)
426
426
  spin_period_average
427
427
  block-averaged onboard value, decoded to seconds using Eq. (47)
428
- spin_period_variance
428
+ spin_period_std_dev
429
429
  standard deviation (1 sigma), decoded to seconds using Eq. (51)
430
430
  pulse_length_average
431
431
  block-averaged value, decoded to μs using Eq. (47)
432
- pulse_length_variance
432
+ pulse_length_std_dev
433
433
  standard deviation (1 sigma), decoded to μs using Eq. (51)
434
434
  glows_start_time
435
435
  GLOWS clock, subseconds as decimal part of float
@@ -444,24 +444,24 @@ class HistogramL1B:
444
444
  is_inside_excluded_region, is_excluded_by_instr_team, is_suspected_transient]
445
445
  spin_period_ground_average
446
446
  block-averaged value computed on ground
447
- spin_period_ground_variance
447
+ spin_period_ground_std_dev
448
448
  standard deviation (1 sigma)
449
449
  position_angle_offset_average
450
450
  block-averaged value in degrees
451
- position_angle_offset_variance
451
+ position_angle_offset_std_dev
452
452
  standard deviation (1 sigma)
453
- spin_axis_orientation_variance
453
+ spin_axis_orientation_std_dev
454
454
  standard deviation( 1 sigma): ∆λ, ∆φ for ⟨λ⟩, ⟨φ⟩
455
455
  spin_axis_orientation_average
456
456
  block-averaged spin-axis ecliptic longitude ⟨λ⟩ and latitude ⟨φ⟩ in degrees
457
457
  spacecraft_location_average
458
458
  block-averaged Cartesian ecliptic coordinates ⟨X⟩, ⟨Y ⟩, ⟨Z⟩ [km] of IMAP
459
- spacecraft_location_variance
459
+ spacecraft_location_std_dev
460
460
  standard deviations (1 sigma) ∆X, ∆Y , ∆Z for ⟨X⟩, ⟨Y ⟩, ⟨Z⟩
461
461
  spacecraft_velocity_average
462
462
  block-averaged values ⟨VX⟩, ⟨VY⟩, ⟨VZ⟩ [km/s] of IMAP velocity components
463
463
  (Cartesian ecliptic frame)
464
- spacecraft_velocity_variance
464
+ spacecraft_velocity_std_dev
465
465
  standard deviations (1 sigma) ∆VX , ∆VY , ∆VZ for ⟨VX ⟩, ⟨VY ⟩, ⟨VZ ⟩
466
466
  flags
467
467
  flags for extra information, per histogram. This should be a human-readable
@@ -470,9 +470,7 @@ class HistogramL1B:
470
470
 
471
471
  histogram: np.ndarray
472
472
  flight_software_version: str
473
- # pkts_file_name: str TODO: add this in L0
474
473
  seq_count_in_pkts_file: int
475
- # l1a_file_name: str TODO: add this
476
474
  # ancillary_data_files: np.ndarray TODO Add this
477
475
  first_spin_id: int
478
476
  last_spin_id: int
@@ -482,13 +480,17 @@ class HistogramL1B:
482
480
  number_of_bins_per_histogram: int
483
481
  number_of_events: int
484
482
  filter_temperature_average: np.double
485
- filter_temperature_variance: np.double
483
+ filter_temperature_variance: InitVar[np.double]
484
+ filter_temperature_std_dev: np.double = field(init=False)
486
485
  hv_voltage_average: np.double
487
- hv_voltage_variance: np.double
486
+ hv_voltage_variance: InitVar[np.double]
487
+ hv_voltage_std_dev: np.double = field(init=False)
488
488
  spin_period_average: np.double
489
- spin_period_variance: np.double
489
+ spin_period_variance: InitVar[np.double]
490
+ spin_period_std_dev: np.double = field(init=False)
490
491
  pulse_length_average: np.double
491
- pulse_length_variance: np.double
492
+ pulse_length_variance: InitVar[np.double]
493
+ pulse_length_std_dev: np.double = field(init=False)
492
494
  imap_start_time: np.double # No conversion needed from l1a->l1b
493
495
  imap_time_offset: np.double # No conversion needed from l1a->l1b
494
496
  glows_start_time: np.double # No conversion needed from l1a->l1b
@@ -499,40 +501,60 @@ class HistogramL1B:
499
501
  imap_spin_angle_bin_cntr: np.ndarray = field(init=False) # Same size as bins
500
502
  histogram_flag_array: np.ndarray = field(init=False)
501
503
  spin_period_ground_average: np.double = field(init=False) # retrieved from SPICE?
502
- spin_period_ground_variance: np.double = field(init=False) # retrieved from SPICE?
504
+ spin_period_ground_std_dev: np.double = field(init=False) # retrieved from SPICE?
503
505
  position_angle_offset_average: np.double = field(init=False) # retrieved from SPICE
504
- position_angle_offset_variance: np.double = field(init=False) # from SPICE
505
- spin_axis_orientation_variance: np.double = field(init=False) # from SPICE
506
+ position_angle_offset_std_dev: np.double = field(init=False) # from SPICE
507
+ spin_axis_orientation_std_dev: np.double = field(init=False) # from SPICE
506
508
  spin_axis_orientation_average: np.double = field(init=False) # retrieved from SPICE
507
509
  spacecraft_location_average: np.ndarray = field(init=False) # retrieved from SPIC
508
- spacecraft_location_variance: np.ndarray = field(init=False) # retrieved from SPIC
510
+ spacecraft_location_std_dev: np.ndarray = field(init=False) # retrieved from SPIC
509
511
  spacecraft_velocity_average: np.ndarray = field(init=False) # retrieved from SPIC
510
- spacecraft_velocity_variance: np.ndarray = field(init=False) # retrieved from SPIC
512
+ spacecraft_velocity_std_dev: np.ndarray = field(init=False) # retrieved from SPIC
511
513
  flags: np.ndarray = field(init=False)
512
514
  # TODO:
513
515
  # - Determine a good way to output flags as "human readable"
514
516
  # - Add spice pieces
515
- # - add in the filenames for the input files - should they be global attributes?
516
517
  # - also unique identifiers
517
518
  # - Bad angle algorithm using SPICE locations
518
519
  # - Move ancillary file to AWS
519
520
 
520
- def __post_init__(self) -> None:
521
- """Will process data."""
521
+ def __post_init__(
522
+ self,
523
+ filter_temperature_variance: np.double,
524
+ hv_voltage_variance: np.double,
525
+ spin_period_variance: np.double,
526
+ pulse_length_variance: np.double,
527
+ ) -> None:
528
+ """
529
+ Will process data.
530
+
531
+ The input variance values are used to calculate the output standard deviation.
532
+
533
+ Parameters
534
+ ----------
535
+ filter_temperature_variance : numpy.double
536
+ Encoded filter temperature variance.
537
+ hv_voltage_variance : numpy.double
538
+ Encoded HV voltage variance.
539
+ spin_period_variance : numpy.double
540
+ Encoded spin period variance.
541
+ pulse_length_variance : numpy.double
542
+ Encoded pulse length variance.
543
+ """
522
544
  # self.histogram_flag_array = np.zeros((2,))
523
545
 
524
546
  # TODO: These pieces will need to be filled in from SPICE kernels. For now,
525
547
  # they are placeholders. GLOWS example code has better placeholders if needed.
526
548
  self.spin_period_ground_average = np.double(-999.9)
527
- self.spin_period_ground_variance = np.double(-999.9)
549
+ self.spin_period_ground_std_dev = np.double(-999.9)
528
550
  self.position_angle_offset_average = np.double(-999.9)
529
- self.position_angle_offset_variance = np.double(-999.9)
530
- self.spin_axis_orientation_variance = np.double(-999.9)
551
+ self.position_angle_offset_std_dev = np.double(-999.9)
552
+ self.spin_axis_orientation_std_dev = np.double(-999.9)
531
553
  self.spin_axis_orientation_average = np.double(-999.9)
532
554
  self.spacecraft_location_average = np.array([-999.9, -999.9, -999.9])
533
- self.spacecraft_location_variance = np.array([-999.9, -999.9, -999.9])
555
+ self.spacecraft_location_std_dev = np.array([-999.9, -999.9, -999.9])
534
556
  self.spacecraft_velocity_average = np.array([-999.9, -999.9, -999.9])
535
- self.spacecraft_velocity_variance = np.array([-999.9, -999.9, -999.9])
557
+ self.spacecraft_velocity_std_dev = np.array([-999.9, -999.9, -999.9])
536
558
  # Will require some additional inputs
537
559
  self.imap_spin_angle_bin_cntr = np.zeros((3600,))
538
560
 
@@ -546,34 +568,34 @@ class HistogramL1B:
546
568
  self.filter_temperature_average = self.ancillary_parameters.decode(
547
569
  "filter_temperature", self.filter_temperature_average
548
570
  )
549
- self.filter_temperature_variance = self.ancillary_parameters.decode_std_dev(
550
- "filter_temperature", self.filter_temperature_variance
571
+ self.filter_temperature_std_dev = self.ancillary_parameters.decode_std_dev(
572
+ "filter_temperature", filter_temperature_variance
551
573
  )
552
574
 
553
575
  self.hv_voltage_average = self.ancillary_parameters.decode(
554
576
  "hv_voltage", self.hv_voltage_average
555
577
  )
556
- self.hv_voltage_variance = self.ancillary_parameters.decode_std_dev(
557
- "hv_voltage", self.hv_voltage_variance
578
+ self.hv_voltage_std_dev = self.ancillary_parameters.decode_std_dev(
579
+ "hv_voltage", hv_voltage_variance
558
580
  )
559
581
  self.spin_period_average = self.ancillary_parameters.decode(
560
582
  "spin_period", self.spin_period_average
561
583
  )
562
- self.spin_period_variance = self.ancillary_parameters.decode_std_dev(
563
- "spin_period", self.spin_period_variance
584
+ self.spin_period_std_dev = self.ancillary_parameters.decode_std_dev(
585
+ "spin_period", spin_period_variance
564
586
  )
565
587
  self.pulse_length_average = self.ancillary_parameters.decode(
566
588
  "pulse_length", self.pulse_length_average
567
589
  )
568
- self.pulse_length_variance = self.ancillary_parameters.decode_std_dev(
569
- "pulse_length", self.pulse_length_variance
590
+ self.pulse_length_std_dev = self.ancillary_parameters.decode_std_dev(
591
+ "pulse_length", pulse_length_variance
570
592
  )
571
593
 
572
594
  self.histogram_flag_array = np.zeros((4, 3600), dtype=np.uint8)
573
595
  # self.unique_block_identifier = np.datetime_as_string(
574
596
  # np.datetime64(int(self.imap_start_time), "ns"), "s"
575
597
  # )
576
- self.flags = np.zeros((17, 3600), dtype=np.uint8)
598
+ self.flags = np.ones((17,), dtype=np.uint8)
577
599
 
578
600
  def output_data(self) -> tuple:
579
601
  """