imap-processing 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imap-processing might be problematic. Click here for more details.
- imap_processing/__init__.py +1 -1
- imap_processing/_version.py +2 -2
- imap_processing/ccsds/excel_to_xtce.py +34 -2
- imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +1 -1
- imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +145 -30
- imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +36 -36
- imap_processing/cdf/config/imap_hi_variable_attrs.yaml +36 -8
- imap_processing/cdf/config/imap_hit_l1b_variable_attrs.yaml +9 -0
- imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +7 -7
- imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +32 -33
- imap_processing/cdf/config/imap_mag_l1_variable_attrs.yaml +24 -28
- imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml +1 -0
- imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +133 -78
- imap_processing/cdf/config/imap_variable_schema.yaml +13 -0
- imap_processing/cdf/imap_cdf_manager.py +31 -27
- imap_processing/cli.py +12 -10
- imap_processing/codice/codice_l1a.py +151 -61
- imap_processing/codice/constants.py +1 -1
- imap_processing/codice/decompress.py +4 -9
- imap_processing/codice/utils.py +1 -0
- imap_processing/glows/l1b/glows_l1b.py +3 -3
- imap_processing/glows/l1b/glows_l1b_data.py +59 -37
- imap_processing/glows/l2/glows_l2_data.py +123 -0
- imap_processing/hi/l1a/histogram.py +1 -1
- imap_processing/hi/l1a/science_direct_event.py +1 -1
- imap_processing/hi/l1b/hi_l1b.py +85 -11
- imap_processing/hi/l1c/hi_l1c.py +23 -1
- imap_processing/hi/utils.py +1 -1
- imap_processing/hit/hit_utils.py +221 -0
- imap_processing/hit/l0/constants.py +118 -0
- imap_processing/hit/l0/decom_hit.py +186 -153
- imap_processing/hit/l1a/hit_l1a.py +20 -175
- imap_processing/hit/l1b/hit_l1b.py +33 -153
- imap_processing/idex/idex_l1a.py +10 -9
- imap_processing/lo/l0/decompression_tables/decompression_tables.py +1 -1
- imap_processing/lo/l0/lo_science.py +1 -1
- imap_processing/lo/packet_definitions/lo_xtce.xml +1 -3296
- imap_processing/mag/l0/decom_mag.py +4 -3
- imap_processing/mag/l1a/mag_l1a.py +11 -11
- imap_processing/mag/l1b/mag_l1b.py +89 -7
- imap_processing/spice/geometry.py +126 -4
- imap_processing/swapi/l1/swapi_l1.py +1 -1
- imap_processing/swapi/l2/swapi_l2.py +1 -1
- imap_processing/swe/l1b/swe_l1b_science.py +8 -8
- imap_processing/tests/ccsds/test_data/expected_output.xml +1 -0
- imap_processing/tests/ccsds/test_excel_to_xtce.py +4 -4
- imap_processing/tests/cdf/test_imap_cdf_manager.py +0 -10
- imap_processing/tests/codice/conftest.py +1 -17
- imap_processing/tests/codice/data/imap_codice_l0_raw_20241110_v001.pkts +0 -0
- imap_processing/tests/codice/test_codice_l0.py +8 -2
- imap_processing/tests/codice/test_codice_l1a.py +127 -107
- imap_processing/tests/codice/test_codice_l1b.py +1 -0
- imap_processing/tests/codice/test_decompress.py +7 -7
- imap_processing/tests/conftest.py +54 -15
- imap_processing/tests/glows/conftest.py +6 -0
- imap_processing/tests/glows/test_glows_l1b.py +9 -9
- imap_processing/tests/glows/test_glows_l1b_data.py +9 -9
- imap_processing/tests/glows/test_glows_l2_data.py +0 -0
- imap_processing/tests/hi/test_data/l1a/imap_hi_l1a_45sensor-de_20250415_v000.cdf +0 -0
- imap_processing/tests/hi/test_hi_l1b.py +71 -1
- imap_processing/tests/hi/test_hi_l1c.py +10 -2
- imap_processing/tests/hi/test_utils.py +4 -3
- imap_processing/tests/hit/{test_hit_decom.py → test_decom_hit.py} +84 -35
- imap_processing/tests/hit/test_hit_l1a.py +2 -197
- imap_processing/tests/hit/test_hit_l1b.py +156 -25
- imap_processing/tests/hit/test_hit_utils.py +218 -0
- imap_processing/tests/idex/conftest.py +1 -1
- imap_processing/tests/idex/imap_idex_l0_raw_20231214_v001.pkts +0 -0
- imap_processing/tests/idex/impact_14_tof_high_data.txt +4444 -4444
- imap_processing/tests/idex/test_idex_l0.py +3 -3
- imap_processing/tests/idex/test_idex_l1a.py +1 -1
- imap_processing/tests/lo/test_lo_science.py +2 -2
- imap_processing/tests/mag/imap_mag_l1a_norm-magi_20251017_v001.cdf +0 -0
- imap_processing/tests/mag/test_mag_l1b.py +59 -3
- imap_processing/tests/spice/test_data/imap_ena_sim_metakernel.template +3 -1
- imap_processing/tests/spice/test_geometry.py +84 -4
- imap_processing/tests/swe/conftest.py +33 -0
- imap_processing/tests/swe/l1_validation/swe_l0_unpacked-data_20240510_v001_VALIDATION_L1B_v3.dat +4332 -0
- imap_processing/tests/swe/test_swe_l1b.py +29 -8
- imap_processing/tests/test_utils.py +1 -1
- imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E12.cdf +0 -0
- imap_processing/tests/ultra/test_data/l1/dps_exposure_helio_45_E24.cdf +0 -0
- imap_processing/tests/ultra/unit/test_de.py +108 -0
- imap_processing/tests/ultra/unit/test_ultra_l1b.py +27 -3
- imap_processing/tests/ultra/unit/test_ultra_l1b_annotated.py +31 -10
- imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +21 -11
- imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +9 -44
- imap_processing/ultra/constants.py +8 -3
- imap_processing/ultra/l1b/de.py +174 -30
- imap_processing/ultra/l1b/ultra_l1b_annotated.py +24 -10
- imap_processing/ultra/l1b/ultra_l1b_extended.py +21 -14
- imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +70 -119
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/METADATA +15 -14
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/RECORD +98 -113
- imap_processing/cdf/cdf_attribute_manager.py +0 -322
- imap_processing/cdf/config/shared/default_global_cdf_attrs_schema.yaml +0 -246
- imap_processing/cdf/config/shared/default_variable_cdf_attrs_schema.yaml +0 -466
- imap_processing/hit/l0/data_classes/housekeeping.py +0 -240
- imap_processing/hit/l0/data_classes/science_packet.py +0 -259
- imap_processing/hit/l0/utils/hit_base.py +0 -57
- imap_processing/tests/cdf/shared/default_global_cdf_attrs_schema.yaml +0 -246
- imap_processing/tests/cdf/shared/default_variable_cdf_attrs_schema.yaml +0 -466
- imap_processing/tests/cdf/test_cdf_attribute_manager.py +0 -353
- imap_processing/tests/codice/data/imap_codice_l0_hi-counters-aggregated_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-counters-singles_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-omni_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-pha_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hi-sectored_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_hskp_20100101_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-counters-aggregated_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-counters-singles_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-angular_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-priority_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-nsw-species_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-pha_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-angular_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-priority_20240429_v001.pkts +0 -0
- imap_processing/tests/codice/data/imap_codice_l0_lo-sw-species_20240429_v001.pkts +0 -0
- imap_processing/tests/idex/imap_idex_l0_raw_20230725_v001.pkts +0 -0
- imap_processing/tests/mag/imap_mag_l1a_burst-magi_20231025_v001.cdf +0 -0
- /imap_processing/tests/hit/test_data/{imap_hit_l0_hk_20100105_v001.pkts → imap_hit_l0_raw_20100105_v001.pkts} +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/LICENSE +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/WHEEL +0 -0
- {imap_processing-0.7.0.dist-info → imap_processing-0.8.0.dist-info}/entry_points.txt +0 -0
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Perform CoDICE l1a processing.
|
|
3
3
|
|
|
4
|
-
This module processes
|
|
4
|
+
This module processes CoDICE L0 files and creates L1a data products.
|
|
5
5
|
|
|
6
6
|
Notes
|
|
7
7
|
-----
|
|
8
|
-
from imap_processing.codice.codice_l0 import decom_packets
|
|
9
8
|
from imap_processing.codice.codice_l1a import process_codice_l1a
|
|
10
|
-
|
|
11
|
-
dataset = process_codice_l1a(packets)
|
|
9
|
+
processed_datasets = process_codice_l1a(path_to_l0_file)
|
|
12
10
|
"""
|
|
13
11
|
|
|
14
12
|
from __future__ import annotations
|
|
15
13
|
|
|
14
|
+
import ast
|
|
16
15
|
import logging
|
|
17
16
|
from pathlib import Path
|
|
18
17
|
from typing import Any
|
|
@@ -28,12 +27,10 @@ from imap_processing.codice import constants
|
|
|
28
27
|
from imap_processing.codice.codice_l0 import decom_packets
|
|
29
28
|
from imap_processing.codice.decompress import decompress
|
|
30
29
|
from imap_processing.codice.utils import CODICEAPID
|
|
31
|
-
from imap_processing.utils import convert_to_binary_string
|
|
32
30
|
|
|
33
31
|
logger = logging.getLogger(__name__)
|
|
34
32
|
logger.setLevel(logging.INFO)
|
|
35
33
|
|
|
36
|
-
# TODO: Add support for decomming multiple APIDs from a single file
|
|
37
34
|
# TODO: Determine what should go in event data CDF and how it should be
|
|
38
35
|
# structured.
|
|
39
36
|
|
|
@@ -87,23 +84,39 @@ class CoDICEL1aPipeline:
|
|
|
87
84
|
self.plan_step = plan_step
|
|
88
85
|
self.view_id = view_id
|
|
89
86
|
|
|
90
|
-
def decompress_data(self, science_values: str) -> None:
|
|
87
|
+
def decompress_data(self, science_values: list[str]) -> None:
|
|
91
88
|
"""
|
|
92
89
|
Perform decompression on the data.
|
|
93
90
|
|
|
94
|
-
The science data within the packet is a compressed
|
|
91
|
+
The science data within the packet is a compressed byte string of
|
|
95
92
|
values. Apply the appropriate decompression algorithm to get an array
|
|
96
93
|
of decompressed values.
|
|
97
94
|
|
|
98
95
|
Parameters
|
|
99
96
|
----------
|
|
100
|
-
science_values : str
|
|
101
|
-
A
|
|
97
|
+
science_values : list[str]
|
|
98
|
+
A list of byte strings representing the science values of the data
|
|
99
|
+
for each packet.
|
|
102
100
|
"""
|
|
103
|
-
|
|
101
|
+
# The compression algorithm depends on the instrument and view ID
|
|
102
|
+
if self.config["instrument"] == "lo":
|
|
103
|
+
compression_algorithm = constants.LO_COMPRESSION_ID_LOOKUP[self.view_id]
|
|
104
|
+
elif self.config["instrument"] == "hi":
|
|
105
|
+
compression_algorithm = constants.HI_COMPRESSION_ID_LOOKUP[self.view_id]
|
|
106
|
+
|
|
107
|
+
self.raw_data = []
|
|
108
|
+
for packet_data, byte_count in zip(
|
|
109
|
+
science_values, self.dataset.byte_count.data
|
|
110
|
+
):
|
|
111
|
+
# Convert from numpy array to byte object
|
|
112
|
+
values = ast.literal_eval(str(packet_data))
|
|
104
113
|
|
|
105
|
-
|
|
106
|
-
|
|
114
|
+
# Only use the values up to the byte count. Bytes after this are
|
|
115
|
+
# used as padding and are not needed
|
|
116
|
+
values = values[:byte_count]
|
|
117
|
+
|
|
118
|
+
decompressed_values = decompress(values, compression_algorithm)
|
|
119
|
+
self.raw_data.append(decompressed_values)
|
|
107
120
|
|
|
108
121
|
def define_coordinates(self) -> None:
|
|
109
122
|
"""
|
|
@@ -115,7 +128,7 @@ class CoDICEL1aPipeline:
|
|
|
115
128
|
|
|
116
129
|
for name in self.config["coords"]:
|
|
117
130
|
if name == "epoch":
|
|
118
|
-
values = self.
|
|
131
|
+
values = self.dataset.epoch.data
|
|
119
132
|
elif name == "inst_az":
|
|
120
133
|
values = np.arange(self.config["num_positions"])
|
|
121
134
|
elif name == "spin_sector":
|
|
@@ -145,7 +158,7 @@ class CoDICEL1aPipeline:
|
|
|
145
158
|
|
|
146
159
|
Returns
|
|
147
160
|
-------
|
|
148
|
-
|
|
161
|
+
processed_dataset : xarray.Dataset
|
|
149
162
|
The 'final' ``xarray`` dataset.
|
|
150
163
|
"""
|
|
151
164
|
# Create the main dataset to hold all the variables
|
|
@@ -154,12 +167,18 @@ class CoDICEL1aPipeline:
|
|
|
154
167
|
attrs=self.cdf_attrs.get_global_attributes(self.config["dataset_name"]),
|
|
155
168
|
)
|
|
156
169
|
|
|
157
|
-
#
|
|
158
|
-
|
|
159
|
-
|
|
170
|
+
# Stack the data so that it is easier to reshape and iterate over
|
|
171
|
+
all_data = np.stack(self.data)
|
|
172
|
+
|
|
173
|
+
# The dimension of all data is (epoch, num_counters, num_positions,
|
|
174
|
+
# num_spin_sectors, num_energy_steps) (or may be slightly different
|
|
175
|
+
# depending on the data product). In any case, iterate over the
|
|
176
|
+
# num_counters dimension to isolate the data for each counter so
|
|
177
|
+
# that it can be placed in a CDF data variable.
|
|
178
|
+
for counter, variable_name in zip(
|
|
179
|
+
range(all_data.shape[1]), self.config["variable_names"]
|
|
160
180
|
):
|
|
161
|
-
|
|
162
|
-
reshaped_variable_data = np.expand_dims(variable_data, axis=0)
|
|
181
|
+
counter_data = all_data[:, counter, :, :, :]
|
|
163
182
|
|
|
164
183
|
# Get the CDF attributes
|
|
165
184
|
descriptor = self.config["dataset_name"].split("imap_codice_l1a_")[-1]
|
|
@@ -168,7 +187,7 @@ class CoDICEL1aPipeline:
|
|
|
168
187
|
|
|
169
188
|
# Create the CDF data variable
|
|
170
189
|
dataset[variable_name] = xr.DataArray(
|
|
171
|
-
|
|
190
|
+
counter_data,
|
|
172
191
|
name=variable_name,
|
|
173
192
|
dims=self.config["dims"],
|
|
174
193
|
attrs=attrs,
|
|
@@ -322,32 +341,51 @@ class CoDICEL1aPipeline:
|
|
|
322
341
|
3D arrays representing dimensions such as spin sectors, positions, and
|
|
323
342
|
energies (depending on the data product).
|
|
324
343
|
"""
|
|
344
|
+
self.data = []
|
|
345
|
+
|
|
325
346
|
# For CoDICE-lo, data are a 3D arrays with a shape representing
|
|
326
347
|
# [<num_positions>,<num_spin_sectors>,<num_energy_steps>]
|
|
327
348
|
if self.config["instrument"] == "lo":
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
349
|
+
for packet_data in self.raw_data:
|
|
350
|
+
if packet_data:
|
|
351
|
+
reshaped_packet_data = np.array(
|
|
352
|
+
packet_data, dtype=np.uint32
|
|
353
|
+
).reshape(
|
|
354
|
+
(
|
|
355
|
+
self.config["num_counters"],
|
|
356
|
+
self.config["num_positions"],
|
|
357
|
+
self.config["num_spin_sectors"],
|
|
358
|
+
self.config["num_energy_steps"],
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
self.data.append(reshaped_packet_data)
|
|
362
|
+
else:
|
|
363
|
+
self.data.append(None)
|
|
336
364
|
|
|
337
365
|
# For CoDICE-hi, data are a 3D array with a shape representing
|
|
338
366
|
# [<num_energy_steps>,<num_positions>,<num_spin_sectors>]
|
|
339
367
|
elif self.config["instrument"] == "hi":
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
368
|
+
for packet_data in self.raw_data:
|
|
369
|
+
if packet_data:
|
|
370
|
+
reshaped_packet_data = np.array(
|
|
371
|
+
packet_data, dtype=np.uint32
|
|
372
|
+
).reshape(
|
|
373
|
+
(
|
|
374
|
+
self.config["num_counters"],
|
|
375
|
+
self.config["num_energy_steps"],
|
|
376
|
+
self.config["num_positions"],
|
|
377
|
+
self.config["num_spin_sectors"],
|
|
378
|
+
)
|
|
379
|
+
)
|
|
380
|
+
self.data.append(reshaped_packet_data)
|
|
381
|
+
else:
|
|
382
|
+
self.data.append(None)
|
|
383
|
+
|
|
384
|
+
# No longer need to keep the raw data around
|
|
385
|
+
del self.raw_data
|
|
348
386
|
|
|
349
387
|
def set_data_product_config(
|
|
350
|
-
self, apid: int,
|
|
388
|
+
self, apid: int, dataset: xr.Dataset, data_version: str
|
|
351
389
|
) -> None:
|
|
352
390
|
"""
|
|
353
391
|
Set the various settings for defining the data products.
|
|
@@ -356,14 +394,14 @@ class CoDICEL1aPipeline:
|
|
|
356
394
|
----------
|
|
357
395
|
apid : int
|
|
358
396
|
The APID of interest.
|
|
359
|
-
|
|
360
|
-
|
|
397
|
+
dataset : xarray.Dataset
|
|
398
|
+
The dataset for the APID of interest.
|
|
361
399
|
data_version : str
|
|
362
400
|
Version of the data product being created.
|
|
363
401
|
"""
|
|
364
402
|
# Set the packet dataset so that it can be easily called from various
|
|
365
403
|
# methods
|
|
366
|
-
self.
|
|
404
|
+
self.dataset = dataset
|
|
367
405
|
|
|
368
406
|
# Set various configurations of the data product
|
|
369
407
|
self.config: dict[str, Any] = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid) # type: ignore
|
|
@@ -473,7 +511,7 @@ def create_hskp_dataset(
|
|
|
473
511
|
return dataset
|
|
474
512
|
|
|
475
513
|
|
|
476
|
-
def get_params(
|
|
514
|
+
def get_params(dataset: xr.Dataset) -> tuple[int, int, int, int]:
|
|
477
515
|
"""
|
|
478
516
|
Return the four 'main' parameters used for l1a processing.
|
|
479
517
|
|
|
@@ -483,8 +521,10 @@ def get_params(packet: xr.Dataset) -> tuple[int, int, int, int]:
|
|
|
483
521
|
|
|
484
522
|
Parameters
|
|
485
523
|
----------
|
|
486
|
-
|
|
487
|
-
|
|
524
|
+
dataset : xarray.Dataset
|
|
525
|
+
The dataset for the APID of interest. We expect each packet in the
|
|
526
|
+
dataset to have the same values for the four main parameters, so the
|
|
527
|
+
first index of the dataset can be used to determine them.
|
|
488
528
|
|
|
489
529
|
Returns
|
|
490
530
|
-------
|
|
@@ -502,15 +542,37 @@ def get_params(packet: xr.Dataset) -> tuple[int, int, int, int]:
|
|
|
502
542
|
view_id : int
|
|
503
543
|
Provides information about how data was collapsed and/or compressed.
|
|
504
544
|
"""
|
|
505
|
-
table_id = int(
|
|
506
|
-
plan_id = int(
|
|
507
|
-
plan_step = int(
|
|
508
|
-
view_id = int(
|
|
545
|
+
table_id = int(dataset.table_id.data[0])
|
|
546
|
+
plan_id = int(dataset.plan_id.data[0])
|
|
547
|
+
plan_step = int(dataset.plan_step.data[0])
|
|
548
|
+
view_id = int(dataset.view_id.data[0])
|
|
509
549
|
|
|
510
550
|
return table_id, plan_id, plan_step, view_id
|
|
511
551
|
|
|
512
552
|
|
|
513
|
-
def
|
|
553
|
+
def log_dataset_info(datasets: dict[int, xr.Dataset]) -> None:
|
|
554
|
+
"""
|
|
555
|
+
Log info about the input data to help with tracking and/or debugging.
|
|
556
|
+
|
|
557
|
+
Parameters
|
|
558
|
+
----------
|
|
559
|
+
datasets : dict[int, xarray.Dataset]
|
|
560
|
+
Mapping from apid to ``xarray`` dataset, one dataset per apid.
|
|
561
|
+
"""
|
|
562
|
+
launch_time = np.datetime64("2010-01-01T00:01:06.184", "ns")
|
|
563
|
+
logger.info("\nThis input file contains the following APIDs:\n")
|
|
564
|
+
for apid in datasets:
|
|
565
|
+
num_packets = len(datasets[apid].epoch.data)
|
|
566
|
+
time_deltas = [np.timedelta64(item, "ns") for item in datasets[apid].epoch.data]
|
|
567
|
+
times = [launch_time + delta for delta in time_deltas]
|
|
568
|
+
start = np.datetime_as_string(times[0])
|
|
569
|
+
end = np.datetime_as_string(times[-1])
|
|
570
|
+
logger.info(
|
|
571
|
+
f"{CODICEAPID(apid).name}: {num_packets} packets spanning {start} to {end}"
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def process_codice_l1a(file_path: Path, data_version: str) -> list[xr.Dataset]:
|
|
514
576
|
"""
|
|
515
577
|
Will process CoDICE l0 data to create l1a data products.
|
|
516
578
|
|
|
@@ -523,38 +585,66 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset:
|
|
|
523
585
|
|
|
524
586
|
Returns
|
|
525
587
|
-------
|
|
526
|
-
|
|
527
|
-
|
|
588
|
+
processed_datasets : list[xarray.Dataset]
|
|
589
|
+
A list of the ``xarray`` datasets containing the science data and
|
|
590
|
+
supporting metadata.
|
|
528
591
|
"""
|
|
529
592
|
# Decom the packets, group data by APID, and sort by time
|
|
530
593
|
datasets = decom_packets(file_path)
|
|
531
594
|
|
|
595
|
+
# Log some information about the contents of the data
|
|
596
|
+
log_dataset_info(datasets)
|
|
597
|
+
|
|
598
|
+
# Placeholder to hold the final, processed datasets
|
|
599
|
+
processed_datasets = []
|
|
600
|
+
|
|
601
|
+
# Process each APID separately
|
|
532
602
|
for apid in datasets:
|
|
533
|
-
|
|
603
|
+
dataset = datasets[apid]
|
|
534
604
|
logger.info(f"\nProcessing {CODICEAPID(apid).name} packet")
|
|
535
605
|
|
|
606
|
+
# Housekeeping data
|
|
536
607
|
if apid == CODICEAPID.COD_NHK:
|
|
537
|
-
|
|
608
|
+
processed_dataset = create_hskp_dataset(dataset, data_version)
|
|
609
|
+
logger.info(f"\nFinal data product:\n{processed_dataset}\n")
|
|
538
610
|
|
|
611
|
+
# Event data
|
|
539
612
|
elif apid in [CODICEAPID.COD_LO_PHA, CODICEAPID.COD_HI_PHA]:
|
|
540
|
-
|
|
613
|
+
processed_dataset = create_event_dataset(apid, dataset, data_version)
|
|
614
|
+
logger.info(f"\nFinal data product:\n{processed_dataset}\n")
|
|
541
615
|
|
|
616
|
+
# Everything else
|
|
542
617
|
elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING:
|
|
543
618
|
# Extract the data
|
|
544
|
-
science_values =
|
|
545
|
-
science_values = convert_to_binary_string(science_values)
|
|
619
|
+
science_values = [packet.data for packet in dataset.data]
|
|
546
620
|
|
|
547
621
|
# Get the four "main" parameters for processing
|
|
548
|
-
table_id, plan_id, plan_step, view_id = get_params(
|
|
622
|
+
table_id, plan_id, plan_step, view_id = get_params(dataset)
|
|
549
623
|
|
|
550
624
|
# Run the pipeline to create a dataset for the product
|
|
551
625
|
pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id)
|
|
552
|
-
pipeline.set_data_product_config(apid,
|
|
626
|
+
pipeline.set_data_product_config(apid, dataset, data_version)
|
|
553
627
|
pipeline.decompress_data(science_values)
|
|
554
628
|
pipeline.reshape_data()
|
|
555
629
|
pipeline.define_coordinates()
|
|
556
|
-
|
|
630
|
+
processed_dataset = pipeline.define_data_variables()
|
|
557
631
|
|
|
558
|
-
|
|
632
|
+
logger.info(f"\nFinal data product:\n{processed_dataset}\n")
|
|
559
633
|
|
|
560
|
-
|
|
634
|
+
# TODO: Still need to implement I-ALiRT and hi-priorities data products
|
|
635
|
+
elif apid in [
|
|
636
|
+
CODICEAPID.COD_HI_INST_COUNTS_PRIORITIES,
|
|
637
|
+
CODICEAPID.COD_HI_IAL,
|
|
638
|
+
CODICEAPID.COD_LO_IAL,
|
|
639
|
+
]:
|
|
640
|
+
logger.info("\tStill need to properly implement")
|
|
641
|
+
processed_dataset = None
|
|
642
|
+
|
|
643
|
+
# For APIDs that don't require processing
|
|
644
|
+
else:
|
|
645
|
+
logger.info(f"\t{apid} does not require processing")
|
|
646
|
+
continue
|
|
647
|
+
|
|
648
|
+
processed_datasets.append(processed_dataset)
|
|
649
|
+
|
|
650
|
+
return processed_datasets
|
|
@@ -124,7 +124,7 @@ DATA_PRODUCT_CONFIGURATIONS = {
|
|
|
124
124
|
"instrument": "hi",
|
|
125
125
|
"num_counters": 3,
|
|
126
126
|
"num_energy_steps": 1, # TODO: Double check with Joey
|
|
127
|
-
"num_positions":
|
|
127
|
+
"num_positions": 12, # TODO: Double check with Joey
|
|
128
128
|
"num_spin_sectors": 1,
|
|
129
129
|
"support_variables": [], # No support variables for this one
|
|
130
130
|
"variable_names": HI_INST_COUNTS_SINGLES_VARIABLE_NAMES,
|
|
@@ -94,9 +94,9 @@ def _apply_lzma_lossless(compressed_bytes: bytes) -> bytes:
|
|
|
94
94
|
return lzma_decompressed_values
|
|
95
95
|
|
|
96
96
|
|
|
97
|
-
def decompress(
|
|
97
|
+
def decompress(compressed_bytes: bytes, algorithm: IntEnum) -> list[int]:
|
|
98
98
|
"""
|
|
99
|
-
Perform decompression on a
|
|
99
|
+
Perform decompression on a byte stream into a list of integers.
|
|
100
100
|
|
|
101
101
|
Apply the appropriate decompression algorithm(s) based on the value
|
|
102
102
|
of the ``algorithm`` attribute. One or more individual algorithms may be
|
|
@@ -104,8 +104,8 @@ def decompress(compressed_binary: str, algorithm: IntEnum) -> list[int]:
|
|
|
104
104
|
|
|
105
105
|
Parameters
|
|
106
106
|
----------
|
|
107
|
-
|
|
108
|
-
The compressed
|
|
107
|
+
compressed_bytes : bytes
|
|
108
|
+
The compressed byte stream.
|
|
109
109
|
algorithm : int
|
|
110
110
|
The algorithm to apply. Supported algorithms are provided in the
|
|
111
111
|
``codice_utils.CoDICECompression`` class.
|
|
@@ -115,11 +115,6 @@ def decompress(compressed_binary: str, algorithm: IntEnum) -> list[int]:
|
|
|
115
115
|
decompressed_values : list[int]
|
|
116
116
|
The 24- or 32-bit decompressed values.
|
|
117
117
|
"""
|
|
118
|
-
# Convert the binary string to a byte stream
|
|
119
|
-
compressed_bytes = int(compressed_binary, 2).to_bytes(
|
|
120
|
-
(len(compressed_binary) + 7) // 8, byteorder="big"
|
|
121
|
-
)
|
|
122
|
-
|
|
123
118
|
# Apply the appropriate decompression algorithm
|
|
124
119
|
if algorithm == CoDICECompression.NO_COMPRESSION:
|
|
125
120
|
decompressed_values = list(compressed_bytes)
|
imap_processing/codice/utils.py
CHANGED
|
@@ -255,10 +255,10 @@ def process_histogram(l1a: xr.Dataset) -> xr.Dataset:
|
|
|
255
255
|
"imap_spin_angle_bin_cntr": ["bins"],
|
|
256
256
|
"histogram_flag_array": ["bad_angle_flags", "bins"],
|
|
257
257
|
"spacecraft_location_average": ["ecliptic"],
|
|
258
|
-
"
|
|
258
|
+
"spacecraft_location_std_dev": ["ecliptic"],
|
|
259
259
|
"spacecraft_velocity_average": ["ecliptic"],
|
|
260
|
-
"
|
|
261
|
-
"flags": ["flag_dim"
|
|
260
|
+
"spacecraft_velocity_std_dev": ["ecliptic"],
|
|
261
|
+
"flags": ["flag_dim"],
|
|
262
262
|
}
|
|
263
263
|
|
|
264
264
|
# For each attribute, retrieve the dims from output_dimension_mapping or use an
|
|
@@ -417,19 +417,19 @@ class HistogramL1B:
|
|
|
417
417
|
IMAP spin angle ψ for bin centers
|
|
418
418
|
filter_temperature_average
|
|
419
419
|
block-averaged value, decoded to Celsius degrees using Eq. (47)
|
|
420
|
-
|
|
420
|
+
filter_temperature_std_dev
|
|
421
421
|
standard deviation (1 sigma), decoded to Celsius degrees using Eq. (51)
|
|
422
422
|
hv_voltage_average
|
|
423
423
|
block-averaged value, decoded to volts using Eq. (47)
|
|
424
|
-
|
|
424
|
+
hv_voltage_std_dev
|
|
425
425
|
standard deviation (1 sigma), decoded to volts using Eq. (51)
|
|
426
426
|
spin_period_average
|
|
427
427
|
block-averaged onboard value, decoded to seconds using Eq. (47)
|
|
428
|
-
|
|
428
|
+
spin_period_std_dev
|
|
429
429
|
standard deviation (1 sigma), decoded to seconds using Eq. (51)
|
|
430
430
|
pulse_length_average
|
|
431
431
|
block-averaged value, decoded to μs using Eq. (47)
|
|
432
|
-
|
|
432
|
+
pulse_length_std_dev
|
|
433
433
|
standard deviation (1 sigma), decoded to μs using Eq. (51)
|
|
434
434
|
glows_start_time
|
|
435
435
|
GLOWS clock, subseconds as decimal part of float
|
|
@@ -444,24 +444,24 @@ class HistogramL1B:
|
|
|
444
444
|
is_inside_excluded_region, is_excluded_by_instr_team, is_suspected_transient]
|
|
445
445
|
spin_period_ground_average
|
|
446
446
|
block-averaged value computed on ground
|
|
447
|
-
|
|
447
|
+
spin_period_ground_std_dev
|
|
448
448
|
standard deviation (1 sigma)
|
|
449
449
|
position_angle_offset_average
|
|
450
450
|
block-averaged value in degrees
|
|
451
|
-
|
|
451
|
+
position_angle_offset_std_dev
|
|
452
452
|
standard deviation (1 sigma)
|
|
453
|
-
|
|
453
|
+
spin_axis_orientation_std_dev
|
|
454
454
|
standard deviation( 1 sigma): ∆λ, ∆φ for ⟨λ⟩, ⟨φ⟩
|
|
455
455
|
spin_axis_orientation_average
|
|
456
456
|
block-averaged spin-axis ecliptic longitude ⟨λ⟩ and latitude ⟨φ⟩ in degrees
|
|
457
457
|
spacecraft_location_average
|
|
458
458
|
block-averaged Cartesian ecliptic coordinates ⟨X⟩, ⟨Y ⟩, ⟨Z⟩ [km] of IMAP
|
|
459
|
-
|
|
459
|
+
spacecraft_location_std_dev
|
|
460
460
|
standard deviations (1 sigma) ∆X, ∆Y , ∆Z for ⟨X⟩, ⟨Y ⟩, ⟨Z⟩
|
|
461
461
|
spacecraft_velocity_average
|
|
462
462
|
block-averaged values ⟨VX⟩, ⟨VY⟩, ⟨VZ⟩ [km/s] of IMAP velocity components
|
|
463
463
|
(Cartesian ecliptic frame)
|
|
464
|
-
|
|
464
|
+
spacecraft_velocity_std_dev
|
|
465
465
|
standard deviations (1 sigma) ∆VX , ∆VY , ∆VZ for ⟨VX ⟩, ⟨VY ⟩, ⟨VZ ⟩
|
|
466
466
|
flags
|
|
467
467
|
flags for extra information, per histogram. This should be a human-readable
|
|
@@ -470,9 +470,7 @@ class HistogramL1B:
|
|
|
470
470
|
|
|
471
471
|
histogram: np.ndarray
|
|
472
472
|
flight_software_version: str
|
|
473
|
-
# pkts_file_name: str TODO: add this in L0
|
|
474
473
|
seq_count_in_pkts_file: int
|
|
475
|
-
# l1a_file_name: str TODO: add this
|
|
476
474
|
# ancillary_data_files: np.ndarray TODO Add this
|
|
477
475
|
first_spin_id: int
|
|
478
476
|
last_spin_id: int
|
|
@@ -482,13 +480,17 @@ class HistogramL1B:
|
|
|
482
480
|
number_of_bins_per_histogram: int
|
|
483
481
|
number_of_events: int
|
|
484
482
|
filter_temperature_average: np.double
|
|
485
|
-
filter_temperature_variance: np.double
|
|
483
|
+
filter_temperature_variance: InitVar[np.double]
|
|
484
|
+
filter_temperature_std_dev: np.double = field(init=False)
|
|
486
485
|
hv_voltage_average: np.double
|
|
487
|
-
hv_voltage_variance: np.double
|
|
486
|
+
hv_voltage_variance: InitVar[np.double]
|
|
487
|
+
hv_voltage_std_dev: np.double = field(init=False)
|
|
488
488
|
spin_period_average: np.double
|
|
489
|
-
spin_period_variance: np.double
|
|
489
|
+
spin_period_variance: InitVar[np.double]
|
|
490
|
+
spin_period_std_dev: np.double = field(init=False)
|
|
490
491
|
pulse_length_average: np.double
|
|
491
|
-
pulse_length_variance: np.double
|
|
492
|
+
pulse_length_variance: InitVar[np.double]
|
|
493
|
+
pulse_length_std_dev: np.double = field(init=False)
|
|
492
494
|
imap_start_time: np.double # No conversion needed from l1a->l1b
|
|
493
495
|
imap_time_offset: np.double # No conversion needed from l1a->l1b
|
|
494
496
|
glows_start_time: np.double # No conversion needed from l1a->l1b
|
|
@@ -499,40 +501,60 @@ class HistogramL1B:
|
|
|
499
501
|
imap_spin_angle_bin_cntr: np.ndarray = field(init=False) # Same size as bins
|
|
500
502
|
histogram_flag_array: np.ndarray = field(init=False)
|
|
501
503
|
spin_period_ground_average: np.double = field(init=False) # retrieved from SPICE?
|
|
502
|
-
|
|
504
|
+
spin_period_ground_std_dev: np.double = field(init=False) # retrieved from SPICE?
|
|
503
505
|
position_angle_offset_average: np.double = field(init=False) # retrieved from SPICE
|
|
504
|
-
|
|
505
|
-
|
|
506
|
+
position_angle_offset_std_dev: np.double = field(init=False) # from SPICE
|
|
507
|
+
spin_axis_orientation_std_dev: np.double = field(init=False) # from SPICE
|
|
506
508
|
spin_axis_orientation_average: np.double = field(init=False) # retrieved from SPICE
|
|
507
509
|
spacecraft_location_average: np.ndarray = field(init=False) # retrieved from SPIC
|
|
508
|
-
|
|
510
|
+
spacecraft_location_std_dev: np.ndarray = field(init=False) # retrieved from SPIC
|
|
509
511
|
spacecraft_velocity_average: np.ndarray = field(init=False) # retrieved from SPIC
|
|
510
|
-
|
|
512
|
+
spacecraft_velocity_std_dev: np.ndarray = field(init=False) # retrieved from SPIC
|
|
511
513
|
flags: np.ndarray = field(init=False)
|
|
512
514
|
# TODO:
|
|
513
515
|
# - Determine a good way to output flags as "human readable"
|
|
514
516
|
# - Add spice pieces
|
|
515
|
-
# - add in the filenames for the input files - should they be global attributes?
|
|
516
517
|
# - also unique identifiers
|
|
517
518
|
# - Bad angle algorithm using SPICE locations
|
|
518
519
|
# - Move ancillary file to AWS
|
|
519
520
|
|
|
520
|
-
def __post_init__(
|
|
521
|
-
|
|
521
|
+
def __post_init__(
|
|
522
|
+
self,
|
|
523
|
+
filter_temperature_variance: np.double,
|
|
524
|
+
hv_voltage_variance: np.double,
|
|
525
|
+
spin_period_variance: np.double,
|
|
526
|
+
pulse_length_variance: np.double,
|
|
527
|
+
) -> None:
|
|
528
|
+
"""
|
|
529
|
+
Will process data.
|
|
530
|
+
|
|
531
|
+
The input variance values are used to calculate the output standard deviation.
|
|
532
|
+
|
|
533
|
+
Parameters
|
|
534
|
+
----------
|
|
535
|
+
filter_temperature_variance : numpy.double
|
|
536
|
+
Encoded filter temperature variance.
|
|
537
|
+
hv_voltage_variance : numpy.double
|
|
538
|
+
Encoded HV voltage variance.
|
|
539
|
+
spin_period_variance : numpy.double
|
|
540
|
+
Encoded spin period variance.
|
|
541
|
+
pulse_length_variance : numpy.double
|
|
542
|
+
Encoded pulse length variance.
|
|
543
|
+
"""
|
|
522
544
|
# self.histogram_flag_array = np.zeros((2,))
|
|
523
545
|
|
|
524
546
|
# TODO: These pieces will need to be filled in from SPICE kernels. For now,
|
|
525
547
|
# they are placeholders. GLOWS example code has better placeholders if needed.
|
|
526
548
|
self.spin_period_ground_average = np.double(-999.9)
|
|
527
|
-
self.
|
|
549
|
+
self.spin_period_ground_std_dev = np.double(-999.9)
|
|
528
550
|
self.position_angle_offset_average = np.double(-999.9)
|
|
529
|
-
self.
|
|
530
|
-
self.
|
|
551
|
+
self.position_angle_offset_std_dev = np.double(-999.9)
|
|
552
|
+
self.spin_axis_orientation_std_dev = np.double(-999.9)
|
|
531
553
|
self.spin_axis_orientation_average = np.double(-999.9)
|
|
532
554
|
self.spacecraft_location_average = np.array([-999.9, -999.9, -999.9])
|
|
533
|
-
self.
|
|
555
|
+
self.spacecraft_location_std_dev = np.array([-999.9, -999.9, -999.9])
|
|
534
556
|
self.spacecraft_velocity_average = np.array([-999.9, -999.9, -999.9])
|
|
535
|
-
self.
|
|
557
|
+
self.spacecraft_velocity_std_dev = np.array([-999.9, -999.9, -999.9])
|
|
536
558
|
# Will require some additional inputs
|
|
537
559
|
self.imap_spin_angle_bin_cntr = np.zeros((3600,))
|
|
538
560
|
|
|
@@ -546,34 +568,34 @@ class HistogramL1B:
|
|
|
546
568
|
self.filter_temperature_average = self.ancillary_parameters.decode(
|
|
547
569
|
"filter_temperature", self.filter_temperature_average
|
|
548
570
|
)
|
|
549
|
-
self.
|
|
550
|
-
"filter_temperature",
|
|
571
|
+
self.filter_temperature_std_dev = self.ancillary_parameters.decode_std_dev(
|
|
572
|
+
"filter_temperature", filter_temperature_variance
|
|
551
573
|
)
|
|
552
574
|
|
|
553
575
|
self.hv_voltage_average = self.ancillary_parameters.decode(
|
|
554
576
|
"hv_voltage", self.hv_voltage_average
|
|
555
577
|
)
|
|
556
|
-
self.
|
|
557
|
-
"hv_voltage",
|
|
578
|
+
self.hv_voltage_std_dev = self.ancillary_parameters.decode_std_dev(
|
|
579
|
+
"hv_voltage", hv_voltage_variance
|
|
558
580
|
)
|
|
559
581
|
self.spin_period_average = self.ancillary_parameters.decode(
|
|
560
582
|
"spin_period", self.spin_period_average
|
|
561
583
|
)
|
|
562
|
-
self.
|
|
563
|
-
"spin_period",
|
|
584
|
+
self.spin_period_std_dev = self.ancillary_parameters.decode_std_dev(
|
|
585
|
+
"spin_period", spin_period_variance
|
|
564
586
|
)
|
|
565
587
|
self.pulse_length_average = self.ancillary_parameters.decode(
|
|
566
588
|
"pulse_length", self.pulse_length_average
|
|
567
589
|
)
|
|
568
|
-
self.
|
|
569
|
-
"pulse_length",
|
|
590
|
+
self.pulse_length_std_dev = self.ancillary_parameters.decode_std_dev(
|
|
591
|
+
"pulse_length", pulse_length_variance
|
|
570
592
|
)
|
|
571
593
|
|
|
572
594
|
self.histogram_flag_array = np.zeros((4, 3600), dtype=np.uint8)
|
|
573
595
|
# self.unique_block_identifier = np.datetime_as_string(
|
|
574
596
|
# np.datetime64(int(self.imap_start_time), "ns"), "s"
|
|
575
597
|
# )
|
|
576
|
-
self.flags = np.
|
|
598
|
+
self.flags = np.ones((17,), dtype=np.uint8)
|
|
577
599
|
|
|
578
600
|
def output_data(self) -> tuple:
|
|
579
601
|
"""
|