imap-processing 0.15.0__py3-none-any.whl → 0.16.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of imap-processing might be problematic. Click here for more details.
- imap_processing/_version.py +2 -2
- imap_processing/cdf/config/imap_hit_l1a_variable_attrs.yaml +1404 -93
- imap_processing/cdf/config/imap_ialirt_l1_variable_attrs.yaml +113 -130
- imap_processing/cli.py +1 -4
- imap_processing/codice/codice_l1a.py +87 -62
- imap_processing/codice/codice_l2.py +0 -8
- imap_processing/codice/constants.py +16 -5
- imap_processing/hi/hi_l1a.py +447 -0
- imap_processing/hi/{l1b/hi_l1b.py → hi_l1b.py} +1 -1
- imap_processing/hi/{l1c/hi_l1c.py → hi_l1c.py} +21 -21
- imap_processing/hi/{l2/hi_l2.py → hi_l2.py} +13 -13
- imap_processing/hi/utils.py +10 -9
- imap_processing/hit/l0/constants.py +3 -1
- imap_processing/hit/l0/decom_hit.py +45 -11
- imap_processing/hit/l1a/hit_l1a.py +31 -24
- imap_processing/hit/l1b/hit_l1b.py +30 -11
- imap_processing/hit/l2/hit_l2.py +8 -11
- imap_processing/ialirt/constants.py +38 -0
- imap_processing/ialirt/l0/parse_mag.py +1 -1
- imap_processing/ialirt/l0/process_codice.py +91 -0
- imap_processing/ialirt/l0/process_hit.py +12 -21
- imap_processing/ialirt/l0/process_swapi.py +172 -23
- imap_processing/ialirt/l0/process_swe.py +3 -10
- imap_processing/ialirt/utils/constants.py +16 -2
- imap_processing/ialirt/utils/create_xarray.py +59 -11
- imap_processing/ultra/utils/ultra_l1_utils.py +4 -2
- {imap_processing-0.15.0.dist-info → imap_processing-0.16.1.dist-info}/METADATA +1 -1
- {imap_processing-0.15.0.dist-info → imap_processing-0.16.1.dist-info}/RECORD +31 -37
- imap_processing/hi/l1a/__init__.py +0 -0
- imap_processing/hi/l1a/hi_l1a.py +0 -98
- imap_processing/hi/l1a/histogram.py +0 -152
- imap_processing/hi/l1a/science_direct_event.py +0 -214
- imap_processing/hi/l1b/__init__.py +0 -0
- imap_processing/hi/l1c/__init__.py +0 -0
- imap_processing/hi/l2/__init__.py +0 -0
- imap_processing/ialirt/l0/process_codicehi.py +0 -156
- imap_processing/ialirt/l0/process_codicelo.py +0 -41
- {imap_processing-0.15.0.dist-info → imap_processing-0.16.1.dist-info}/LICENSE +0 -0
- {imap_processing-0.15.0.dist-info → imap_processing-0.16.1.dist-info}/WHEEL +0 -0
- {imap_processing-0.15.0.dist-info → imap_processing-0.16.1.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,447 @@
|
|
|
1
|
+
"""IMAP-HI L1A processing module."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Union
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import xarray as xr
|
|
10
|
+
from numpy import _typing as npt
|
|
11
|
+
from numpy._typing import NDArray
|
|
12
|
+
|
|
13
|
+
from imap_processing import imap_module_directory
|
|
14
|
+
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
|
|
15
|
+
from imap_processing.hi.utils import HIAPID
|
|
16
|
+
from imap_processing.spice.time import met_to_ttj2000ns
|
|
17
|
+
from imap_processing.utils import packet_file_to_datasets
|
|
18
|
+
|
|
19
|
+
# TODO: read DE_CLOCK_TICK_US from
|
|
20
|
+
# instrument status summary later. This value
|
|
21
|
+
# is rarely change but want to be able to change
|
|
22
|
+
# it if needed. It stores information about how
|
|
23
|
+
# fast the time was ticking. It is in microseconds.
|
|
24
|
+
DE_CLOCK_TICK_US = 1999
|
|
25
|
+
DE_CLOCK_TICK_S = DE_CLOCK_TICK_US / 1e6
|
|
26
|
+
HALF_CLOCK_TICK_S = DE_CLOCK_TICK_S / 2
|
|
27
|
+
|
|
28
|
+
MILLISECOND_TO_S = 1e-3
|
|
29
|
+
|
|
30
|
+
# define the names of the 24 counter arrays
|
|
31
|
+
# contained in the histogram packet
|
|
32
|
+
QUALIFIED_COUNTERS = (
|
|
33
|
+
"ab_qualified",
|
|
34
|
+
"c1c2_qualified",
|
|
35
|
+
"ac1_qualified",
|
|
36
|
+
"bc1_qualified",
|
|
37
|
+
"abc1_qualified",
|
|
38
|
+
"ac1c2_qualified",
|
|
39
|
+
"bc1c2_qualified",
|
|
40
|
+
"abc1c2_qualified",
|
|
41
|
+
)
|
|
42
|
+
LONG_COUNTERS = (
|
|
43
|
+
"a_first_only",
|
|
44
|
+
"b_first_only",
|
|
45
|
+
"c_first_only",
|
|
46
|
+
"ab_long",
|
|
47
|
+
"c1c2_long",
|
|
48
|
+
"ac1_long",
|
|
49
|
+
"bc1_long",
|
|
50
|
+
"abc1_long",
|
|
51
|
+
"ac1c2_long",
|
|
52
|
+
"bc1c2_long",
|
|
53
|
+
"abc1c2_long",
|
|
54
|
+
)
|
|
55
|
+
TOTAL_COUNTERS = ("a_total", "b_total", "c_total", "fee_de_recd", "fee_de_sent")
|
|
56
|
+
|
|
57
|
+
logger = logging.getLogger(__name__)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def hi_l1a(packet_file_path: Union[str, Path]) -> list[xr.Dataset]:
|
|
61
|
+
"""
|
|
62
|
+
Will process IMAP raw data to l1a.
|
|
63
|
+
|
|
64
|
+
Parameters
|
|
65
|
+
----------
|
|
66
|
+
packet_file_path : str
|
|
67
|
+
Data packet file path.
|
|
68
|
+
|
|
69
|
+
Returns
|
|
70
|
+
-------
|
|
71
|
+
processed_data : list[xarray.Dataset]
|
|
72
|
+
List of processed xarray dataset.
|
|
73
|
+
"""
|
|
74
|
+
datasets_by_apid = hi_packet_file_to_datasets(packet_file_path)
|
|
75
|
+
|
|
76
|
+
# Process science to l1a.
|
|
77
|
+
processed_data = []
|
|
78
|
+
for apid in datasets_by_apid:
|
|
79
|
+
try:
|
|
80
|
+
apid_enum = HIAPID(apid)
|
|
81
|
+
except ValueError as err:
|
|
82
|
+
raise RuntimeError(f"Encountered unexpected APID [{apid}]") from err
|
|
83
|
+
|
|
84
|
+
logger.info(f"Processing IMAP-Hi data for {apid_enum.name} packets")
|
|
85
|
+
|
|
86
|
+
if apid_enum in [HIAPID.H45_SCI_CNT, HIAPID.H90_SCI_CNT]:
|
|
87
|
+
data = finish_hist_dataset(datasets_by_apid[apid])
|
|
88
|
+
gattr_key = "imap_hi_l1a_hist_attrs"
|
|
89
|
+
elif apid_enum in [HIAPID.H45_SCI_DE, HIAPID.H90_SCI_DE]:
|
|
90
|
+
data = finish_de_dataset(datasets_by_apid[apid])
|
|
91
|
+
gattr_key = "imap_hi_l1a_de_attrs"
|
|
92
|
+
elif apid_enum in [HIAPID.H45_APP_NHK, HIAPID.H90_APP_NHK]:
|
|
93
|
+
data = datasets_by_apid[apid]
|
|
94
|
+
gattr_key = "imap_hi_l1a_hk_attrs"
|
|
95
|
+
elif apid_enum in [HIAPID.H45_DIAG_FEE, HIAPID.H90_DIAG_FEE]:
|
|
96
|
+
data = datasets_by_apid[apid]
|
|
97
|
+
gattr_key = "imap_hi_l1a_diagfee_attrs"
|
|
98
|
+
|
|
99
|
+
# Update dataset global attributes
|
|
100
|
+
attr_mgr = ImapCdfAttributes()
|
|
101
|
+
attr_mgr.add_instrument_global_attrs("hi")
|
|
102
|
+
data.attrs.update(attr_mgr.get_global_attributes(gattr_key))
|
|
103
|
+
|
|
104
|
+
# set the sensor string in Logical_source
|
|
105
|
+
sensor_str = apid_enum.sensor
|
|
106
|
+
data.attrs["Logical_source"] = data.attrs["Logical_source"].format(
|
|
107
|
+
sensor=sensor_str
|
|
108
|
+
)
|
|
109
|
+
processed_data.append(data)
|
|
110
|
+
return processed_data
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def hi_packet_file_to_datasets(
|
|
114
|
+
packet_file_path: Union[str, Path], use_derived_value: bool = False
|
|
115
|
+
) -> dict[int, xr.Dataset]:
|
|
116
|
+
"""
|
|
117
|
+
Extract hi datasets from packet file.
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
packet_file_path : str
|
|
122
|
+
L0 packet file path.
|
|
123
|
+
use_derived_value : bool
|
|
124
|
+
Whether to use the derived value from the XTCE definition. Default is False.
|
|
125
|
+
|
|
126
|
+
Returns
|
|
127
|
+
-------
|
|
128
|
+
datasets : dict[int, xarray.Dataset]
|
|
129
|
+
Dictionary of xarray datasets keyed by APID.
|
|
130
|
+
"""
|
|
131
|
+
packet_def_file = (
|
|
132
|
+
imap_module_directory / "hi/packet_definitions/TLM_HI_COMBINED_SCI.xml"
|
|
133
|
+
)
|
|
134
|
+
datasets_by_apid = packet_file_to_datasets(
|
|
135
|
+
packet_file=packet_file_path,
|
|
136
|
+
xtce_packet_definition=packet_def_file,
|
|
137
|
+
use_derived_value=use_derived_value,
|
|
138
|
+
)
|
|
139
|
+
return datasets_by_apid
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def finish_de_dataset(packets_data: xr.Dataset) -> xr.Dataset:
|
|
143
|
+
"""
|
|
144
|
+
Unpack IMAP-Hi direct event data.
|
|
145
|
+
|
|
146
|
+
Processing step:
|
|
147
|
+
|
|
148
|
+
| 1. Break binary stream data into unit of 48-bits
|
|
149
|
+
| 2. Parse direct event data
|
|
150
|
+
| 5. Save the data into xarray dataset.
|
|
151
|
+
|
|
152
|
+
Parameters
|
|
153
|
+
----------
|
|
154
|
+
packets_data : xarray.Dataset
|
|
155
|
+
Packets extracted into a dataset.
|
|
156
|
+
|
|
157
|
+
Returns
|
|
158
|
+
-------
|
|
159
|
+
dataset : xarray.Dataset
|
|
160
|
+
Xarray dataset.
|
|
161
|
+
"""
|
|
162
|
+
de_data_dict: dict[str, list] = defaultdict(list)
|
|
163
|
+
|
|
164
|
+
# Add packet data to the dictionary, renaming some fields
|
|
165
|
+
# This is done first so that these variables are first in the CDF
|
|
166
|
+
for from_key, to_key in {
|
|
167
|
+
"shcoarse": "ccsds_met",
|
|
168
|
+
"src_seq_ctr": "src_seq_ctr",
|
|
169
|
+
"pkt_len": "pkt_len",
|
|
170
|
+
"last_spin_num": "last_spin_num",
|
|
171
|
+
"spin_invalids": "spin_invalids",
|
|
172
|
+
"esa_step_num": "esa_step",
|
|
173
|
+
"meta_seconds": "meta_seconds",
|
|
174
|
+
"meta_subseconds": "meta_subseconds",
|
|
175
|
+
}.items():
|
|
176
|
+
de_data_dict[to_key] = packets_data[from_key].data
|
|
177
|
+
|
|
178
|
+
# For each packet, parse the DE data and add it to the Pointing
|
|
179
|
+
# list of DE data usint `extend()`
|
|
180
|
+
for i, data in enumerate(packets_data["de_tof"].data):
|
|
181
|
+
parsed_de_data = parse_direct_events(data)
|
|
182
|
+
for key, new_data in parsed_de_data.items():
|
|
183
|
+
de_data_dict[key].extend(new_data)
|
|
184
|
+
# Record the ccsds packet index for each DE
|
|
185
|
+
de_data_dict["ccsds_index"].extend([i] * len(parsed_de_data["de_tag"]))
|
|
186
|
+
|
|
187
|
+
# create dataset
|
|
188
|
+
return create_de_dataset(de_data_dict)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def create_de_dataset(de_data_dict: dict[str, npt.ArrayLike]) -> xr.Dataset:
|
|
192
|
+
"""
|
|
193
|
+
Create Hi L1A direct event xarray dataset.
|
|
194
|
+
|
|
195
|
+
Parameters
|
|
196
|
+
----------
|
|
197
|
+
de_data_dict : Dict[list]
|
|
198
|
+
Dictionary of packet telemetry and direct event data lists.
|
|
199
|
+
|
|
200
|
+
Returns
|
|
201
|
+
-------
|
|
202
|
+
dataset : xarray.Dataset
|
|
203
|
+
Xarray dataset.
|
|
204
|
+
"""
|
|
205
|
+
# Load the CDF attributes
|
|
206
|
+
attr_mgr = ImapCdfAttributes()
|
|
207
|
+
attr_mgr.add_instrument_global_attrs("hi")
|
|
208
|
+
attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
|
|
209
|
+
|
|
210
|
+
# check_schema=False keeps DEPEND_0 = '' from being auto added
|
|
211
|
+
epoch_attrs = attr_mgr.get_variable_attributes("epoch", check_schema=False)
|
|
212
|
+
epoch_attrs["CATDESC"] = (
|
|
213
|
+
"CCSDS creation time, number of nanoseconds since J2000 with leap "
|
|
214
|
+
"seconds included"
|
|
215
|
+
)
|
|
216
|
+
epoch = xr.DataArray(
|
|
217
|
+
met_to_ttj2000ns(de_data_dict["ccsds_met"]),
|
|
218
|
+
name="epoch",
|
|
219
|
+
dims=["epoch"],
|
|
220
|
+
attrs=epoch_attrs,
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
event_met_attrs = attr_mgr.get_variable_attributes(
|
|
224
|
+
"hi_de_event_met", check_schema=False
|
|
225
|
+
)
|
|
226
|
+
# For L1A DE, event_met is its own dimension, so we remove the DEPEND_0 attribute
|
|
227
|
+
_ = event_met_attrs.pop("DEPEND_0")
|
|
228
|
+
|
|
229
|
+
# Compute the meta-event MET in seconds
|
|
230
|
+
meta_event_met = (
|
|
231
|
+
np.array(de_data_dict["meta_seconds"]).astype(np.float64)
|
|
232
|
+
+ np.array(de_data_dict["meta_subseconds"]) * MILLISECOND_TO_S
|
|
233
|
+
)
|
|
234
|
+
# Compute the MET of each event in seconds
|
|
235
|
+
# event MET = meta_event_met + de_clock
|
|
236
|
+
# See Hi Algorithm Document section 2.2.5
|
|
237
|
+
event_met_array = np.array(
|
|
238
|
+
meta_event_met[de_data_dict["ccsds_index"]]
|
|
239
|
+
+ np.array(de_data_dict["de_tag"]) * DE_CLOCK_TICK_S,
|
|
240
|
+
dtype=event_met_attrs.pop("dtype"),
|
|
241
|
+
)
|
|
242
|
+
event_met = xr.DataArray(
|
|
243
|
+
event_met_array,
|
|
244
|
+
name="event_met",
|
|
245
|
+
dims=["event_met"],
|
|
246
|
+
attrs=event_met_attrs,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
dataset = xr.Dataset(
|
|
250
|
+
coords={"epoch": epoch, "event_met": event_met},
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
for var_name, data in de_data_dict.items():
|
|
254
|
+
attrs = attr_mgr.get_variable_attributes(
|
|
255
|
+
f"hi_de_{var_name}", check_schema=False
|
|
256
|
+
).copy()
|
|
257
|
+
dtype = attrs.pop("dtype")
|
|
258
|
+
dataset[var_name] = xr.DataArray(
|
|
259
|
+
np.array(data, dtype=np.dtype(dtype)),
|
|
260
|
+
dims=attrs["DEPEND_0"],
|
|
261
|
+
attrs=attrs,
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
return dataset
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def parse_direct_events(de_data: bytes) -> dict[str, npt.ArrayLike]:
|
|
268
|
+
"""
|
|
269
|
+
Parse event data from a binary blob.
|
|
270
|
+
|
|
271
|
+
IMAP-Hi direct event data information is stored in
|
|
272
|
+
48-bits as follows:
|
|
273
|
+
|
|
274
|
+
| Read 48-bits into 16, 2, 10, 10, 10, bits. Each of these breaks
|
|
275
|
+
| down as:
|
|
276
|
+
|
|
|
277
|
+
| de_tag - 16 bits
|
|
278
|
+
| start_bitmask_data - 2 bits (tA=1, tB=2, tC1=3)
|
|
279
|
+
| tof_1 - 10 bit counter
|
|
280
|
+
| tof_2 - 10 bit counter
|
|
281
|
+
| tof_3 - 10 bit counter
|
|
282
|
+
|
|
283
|
+
There are at most total of 664 of 48-bits in each data packet.
|
|
284
|
+
This data packet is of variable length. If there is one event, then
|
|
285
|
+
DE_TOF will contain 48-bits. If there are 664 events, then
|
|
286
|
+
DE_TOF will contain 664 x 48-bits. If there is no event, then
|
|
287
|
+
DE_TOF will contain 0-bits.
|
|
288
|
+
|
|
289
|
+
There should be two data packets per ESA. Each packet contains meta-event
|
|
290
|
+
data that is identical between the two packets for a common ESA.
|
|
291
|
+
If there is no event record for certain ESA step, then both packets will
|
|
292
|
+
contain 0-bits in DE_TOF.
|
|
293
|
+
|
|
294
|
+
Parameters
|
|
295
|
+
----------
|
|
296
|
+
de_data : bytes
|
|
297
|
+
Binary blob from de_tag field of SCI_DE packet. Must be an integer
|
|
298
|
+
multiple of 48-bits of data.
|
|
299
|
+
|
|
300
|
+
Returns
|
|
301
|
+
-------
|
|
302
|
+
Dict[str, list]
|
|
303
|
+
Parsed event data.
|
|
304
|
+
"""
|
|
305
|
+
# The de_data is a binary blob with Nx6 bytes of data where N = number of
|
|
306
|
+
# direct events encoded into the binary blob. Interpreting the data as
|
|
307
|
+
# big-endian uint16 data and reshaping into a (3, -1) ndarray results
|
|
308
|
+
# in an array with shape (3, N). Indexing the first axis of that array
|
|
309
|
+
# (e.g. data_uint16[i]) gives the ith 2-bytes of data for each of the N
|
|
310
|
+
# direct events.
|
|
311
|
+
# Considering the 6-bytes of data for each DE as 3 2-byte words,
|
|
312
|
+
# each word contains the following:
|
|
313
|
+
# word_0: full 16-bits is the de_tag
|
|
314
|
+
# word_1: 2-bits of Trigger ID, 10-bits tof_1, upper 4-bits of tof_2
|
|
315
|
+
# word_2: lower 6-bits of tof_2, 10-bits of tof_3
|
|
316
|
+
data_uint16 = np.reshape(
|
|
317
|
+
np.frombuffer(de_data, dtype=">u2"), (3, -1), order="F"
|
|
318
|
+
).astype(np.uint16)
|
|
319
|
+
|
|
320
|
+
de_dict = dict()
|
|
321
|
+
de_dict["de_tag"] = data_uint16[0]
|
|
322
|
+
de_dict["trigger_id"] = (data_uint16[1] >> 14).astype(np.uint8)
|
|
323
|
+
de_dict["tof_1"] = (data_uint16[1] & int(b"00111111_11110000", 2)) >> 4
|
|
324
|
+
de_dict["tof_2"] = ((data_uint16[1] & int(b"00000000_00001111", 2)) << 6) + (
|
|
325
|
+
data_uint16[2] >> 10
|
|
326
|
+
)
|
|
327
|
+
de_dict["tof_3"] = data_uint16[2] & int(b"00000011_11111111", 2)
|
|
328
|
+
|
|
329
|
+
return de_dict
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def finish_hist_dataset(input_ds: xr.Dataset) -> xr.Dataset:
|
|
333
|
+
"""
|
|
334
|
+
Create dataset for a number of Hi Histogram packets.
|
|
335
|
+
|
|
336
|
+
Parameters
|
|
337
|
+
----------
|
|
338
|
+
input_ds : xarray.Dataset
|
|
339
|
+
Dataset of packets generated using the
|
|
340
|
+
`imap_processing.utils.packet_file_to_datasets` function.
|
|
341
|
+
|
|
342
|
+
Returns
|
|
343
|
+
-------
|
|
344
|
+
dataset : xarray.Dataset
|
|
345
|
+
Dataset with all metadata field data in xr.DataArray.
|
|
346
|
+
"""
|
|
347
|
+
attr_mgr = ImapCdfAttributes()
|
|
348
|
+
attr_mgr.add_instrument_global_attrs(instrument="hi")
|
|
349
|
+
attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None)
|
|
350
|
+
|
|
351
|
+
# Rename shcoarse variable (do this first since it copies the input_ds)
|
|
352
|
+
dataset = input_ds.rename_vars({"shcoarse": "ccsds_met"})
|
|
353
|
+
|
|
354
|
+
dataset.epoch.attrs.update(
|
|
355
|
+
attr_mgr.get_variable_attributes("epoch"),
|
|
356
|
+
)
|
|
357
|
+
# Add the hist_angle coordinate
|
|
358
|
+
# Histogram data is binned in 90, 4-degree bins
|
|
359
|
+
attrs = attr_mgr.get_variable_attributes("hi_hist_angle")
|
|
360
|
+
dataset.coords.update(
|
|
361
|
+
{
|
|
362
|
+
"angle": xr.DataArray(
|
|
363
|
+
np.arange(2, 360, 4),
|
|
364
|
+
name="angle",
|
|
365
|
+
dims=["angle"],
|
|
366
|
+
attrs=attrs,
|
|
367
|
+
)
|
|
368
|
+
}
|
|
369
|
+
)
|
|
370
|
+
# Update existing variable attributes
|
|
371
|
+
for var_name in [
|
|
372
|
+
"version",
|
|
373
|
+
"type",
|
|
374
|
+
"sec_hdr_flg",
|
|
375
|
+
"pkt_apid",
|
|
376
|
+
"seq_flgs",
|
|
377
|
+
"src_seq_ctr",
|
|
378
|
+
"pkt_len",
|
|
379
|
+
"ccsds_met",
|
|
380
|
+
"esa_step",
|
|
381
|
+
"num_of_spins",
|
|
382
|
+
"cksum",
|
|
383
|
+
]:
|
|
384
|
+
attrs = attr_mgr.get_variable_attributes(f"hi_hist_{var_name}")
|
|
385
|
+
dataset.data_vars[var_name].attrs.update(attrs)
|
|
386
|
+
|
|
387
|
+
new_vars = dict()
|
|
388
|
+
# Populate 90-element histogram counters
|
|
389
|
+
default_counter_attrs = attr_mgr.get_variable_attributes("hi_hist_counters")
|
|
390
|
+
for counter_name in (*QUALIFIED_COUNTERS, *LONG_COUNTERS, *TOTAL_COUNTERS):
|
|
391
|
+
# Inject counter name into generic counter attributes
|
|
392
|
+
counter_attrs = default_counter_attrs.copy()
|
|
393
|
+
for key, val in counter_attrs.items():
|
|
394
|
+
if isinstance(val, str) and "{counter_name}" in val:
|
|
395
|
+
counter_attrs[key] = val.format(counter_name=counter_name)
|
|
396
|
+
# Instantiate the counter DataArray
|
|
397
|
+
new_vars[counter_name] = xr.DataArray(
|
|
398
|
+
data=unpack_hist_counter(input_ds[counter_name].data.sum()),
|
|
399
|
+
dims=["epoch", "angle"],
|
|
400
|
+
attrs=counter_attrs,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
# Generate label variable for angle coordinate
|
|
404
|
+
new_vars["angle_label"] = xr.DataArray(
|
|
405
|
+
dataset.coords["angle"].values.astype(str),
|
|
406
|
+
name="angle_label",
|
|
407
|
+
dims=["angle"],
|
|
408
|
+
attrs=attr_mgr.get_variable_attributes(
|
|
409
|
+
"hi_hist_angle_label", check_schema=False
|
|
410
|
+
),
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
dataset.update(new_vars)
|
|
414
|
+
|
|
415
|
+
return dataset
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def unpack_hist_counter(counter_bytes: bytes) -> NDArray[np.uint16]:
|
|
419
|
+
"""
|
|
420
|
+
Unpack Hi SCI_CNT counter data for a single counter.
|
|
421
|
+
|
|
422
|
+
Parameters
|
|
423
|
+
----------
|
|
424
|
+
counter_bytes : bytes
|
|
425
|
+
Sum individual bytes for all epochs of a Hi SCI_CNT counter.
|
|
426
|
+
|
|
427
|
+
Returns
|
|
428
|
+
-------
|
|
429
|
+
output_array : numpy.ndarray[numpy.uint16]
|
|
430
|
+
The unpacked 12-bit unsigned integers for the input bytes. The
|
|
431
|
+
output array has a shape of (n, 90) where n is the number of SCI_CNT
|
|
432
|
+
packets in the input dataset.
|
|
433
|
+
"""
|
|
434
|
+
# Interpret bytes for all epochs of current counter as uint8 array
|
|
435
|
+
counter_uint8 = np.frombuffer(counter_bytes, dtype=np.uint8)
|
|
436
|
+
# Split into triplets of upper-byte, split-byte and lower-byte arrays
|
|
437
|
+
upper_uint8, split_unit8, lower_uint8 = np.reshape(
|
|
438
|
+
counter_uint8, (3, -1), order="F"
|
|
439
|
+
).astype(np.uint16)
|
|
440
|
+
# Compute even indexed uint12 values from upper-byte and first 4-bits of
|
|
441
|
+
# split-byte
|
|
442
|
+
even_uint12 = (upper_uint8 << 4) + (split_unit8 >> 4)
|
|
443
|
+
# Compute odd indexed uint12 values from lower 4-bits of split-byte and
|
|
444
|
+
# lower-byte
|
|
445
|
+
odd_uint12 = ((split_unit8 & (2**4 - 1)) << 8) + lower_uint8
|
|
446
|
+
output_array = np.column_stack((even_uint12, odd_uint12)).reshape(-1, 90)
|
|
447
|
+
return output_array
|
|
@@ -11,7 +11,7 @@ import xarray as xr
|
|
|
11
11
|
from imap_processing import imap_module_directory
|
|
12
12
|
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
|
|
13
13
|
from imap_processing.cdf.utils import parse_filename_like
|
|
14
|
-
from imap_processing.hi.
|
|
14
|
+
from imap_processing.hi.hi_l1a import HALF_CLOCK_TICK_S
|
|
15
15
|
from imap_processing.hi.utils import (
|
|
16
16
|
HIAPID,
|
|
17
17
|
CoincidenceBitmap,
|
|
@@ -14,7 +14,7 @@ from numpy._typing import NDArray
|
|
|
14
14
|
|
|
15
15
|
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
|
|
16
16
|
from imap_processing.cdf.utils import parse_filename_like
|
|
17
|
-
from imap_processing.hi.
|
|
17
|
+
from imap_processing.hi.hi_l1a import (
|
|
18
18
|
DE_CLOCK_TICK_S,
|
|
19
19
|
HALF_CLOCK_TICK_S,
|
|
20
20
|
)
|
|
@@ -57,7 +57,7 @@ def hi_l1c(
|
|
|
57
57
|
----------
|
|
58
58
|
de_dataset : xarray.Dataset
|
|
59
59
|
IMAP-Hi l1b de product.
|
|
60
|
-
calibration_prod_config_path : Path
|
|
60
|
+
calibration_prod_config_path : pathlib.Path
|
|
61
61
|
Calibration product configuration file.
|
|
62
62
|
|
|
63
63
|
Returns
|
|
@@ -84,7 +84,7 @@ def generate_pset_dataset(
|
|
|
84
84
|
----------
|
|
85
85
|
de_dataset : xarray.Dataset
|
|
86
86
|
IMAP-Hi l1b de product.
|
|
87
|
-
calibration_prod_config_path : Path
|
|
87
|
+
calibration_prod_config_path : pathlib.Path
|
|
88
88
|
Calibration product configuration file.
|
|
89
89
|
|
|
90
90
|
Returns
|
|
@@ -306,16 +306,16 @@ def pset_counts(
|
|
|
306
306
|
|
|
307
307
|
Parameters
|
|
308
308
|
----------
|
|
309
|
-
pset_coords : dict[str,
|
|
310
|
-
The PSET coordinates from the
|
|
311
|
-
config_df :
|
|
309
|
+
pset_coords : dict[str, xarray.DataArray]
|
|
310
|
+
The PSET coordinates from the xarray.Dataset.
|
|
311
|
+
config_df : pandas.DataFrame
|
|
312
312
|
The calibration product configuration dataframe.
|
|
313
|
-
l1b_de_dataset :
|
|
313
|
+
l1b_de_dataset : xarray.Dataset
|
|
314
314
|
The L1B dataset for the pointing being processed.
|
|
315
315
|
|
|
316
316
|
Returns
|
|
317
317
|
-------
|
|
318
|
-
dict[str,
|
|
318
|
+
dict[str, xarray.DataArray]
|
|
319
319
|
Dictionary containing new exposure_times DataArray to be added to the PSET
|
|
320
320
|
dataset.
|
|
321
321
|
"""
|
|
@@ -396,10 +396,10 @@ def get_tof_window_mask(
|
|
|
396
396
|
|
|
397
397
|
Parameters
|
|
398
398
|
----------
|
|
399
|
-
de_df :
|
|
399
|
+
de_df : pandas.DataFrame
|
|
400
400
|
The Direct Event dataframe for the DEs to filter based on the TOF
|
|
401
401
|
windows.
|
|
402
|
-
prod_config_row :
|
|
402
|
+
prod_config_row : NamedTuple
|
|
403
403
|
A single row of the prod config dataframe represented as a named tuple.
|
|
404
404
|
fill_vals : dict
|
|
405
405
|
A dictionary containing the fill values used in the input DE TOF
|
|
@@ -438,12 +438,12 @@ def pset_backgrounds(pset_coords: dict[str, xr.DataArray]) -> dict[str, xr.DataA
|
|
|
438
438
|
|
|
439
439
|
Parameters
|
|
440
440
|
----------
|
|
441
|
-
pset_coords : dict[str,
|
|
442
|
-
The PSET coordinates from the
|
|
441
|
+
pset_coords : dict[str, xarray.DataArray]
|
|
442
|
+
The PSET coordinates from the xarray.Dataset.
|
|
443
443
|
|
|
444
444
|
Returns
|
|
445
445
|
-------
|
|
446
|
-
dict[str,
|
|
446
|
+
dict[str, xarray.DataArray]
|
|
447
447
|
Dictionary containing background_rates and background_rates_unc DataArrays
|
|
448
448
|
to be added to the PSET dataset.
|
|
449
449
|
"""
|
|
@@ -475,14 +475,14 @@ def pset_exposure(
|
|
|
475
475
|
|
|
476
476
|
Parameters
|
|
477
477
|
----------
|
|
478
|
-
pset_coords : dict[str,
|
|
479
|
-
The PSET coordinates from the
|
|
480
|
-
l1b_de_dataset :
|
|
478
|
+
pset_coords : dict[str, xarray.DataArray]
|
|
479
|
+
The PSET coordinates from the xarray.Dataset.
|
|
480
|
+
l1b_de_dataset : xarray.Dataset
|
|
481
481
|
The L1B dataset for the pointing being processed.
|
|
482
482
|
|
|
483
483
|
Returns
|
|
484
484
|
-------
|
|
485
|
-
dict[str,
|
|
485
|
+
dict[str, xarray.DataArray]
|
|
486
486
|
Dictionary containing new exposure_times DataArray to be added to the PSET
|
|
487
487
|
dataset.
|
|
488
488
|
"""
|
|
@@ -552,12 +552,12 @@ def find_second_de_packet_data(l1b_dataset: xr.Dataset) -> xr.Dataset:
|
|
|
552
552
|
|
|
553
553
|
Parameters
|
|
554
554
|
----------
|
|
555
|
-
l1b_dataset :
|
|
555
|
+
l1b_dataset : xarray.Dataset
|
|
556
556
|
The L1B Direct Event Dataset for the current pointing.
|
|
557
557
|
|
|
558
558
|
Returns
|
|
559
559
|
-------
|
|
560
|
-
reduced_dataset :
|
|
560
|
+
reduced_dataset : xarray.Dataset
|
|
561
561
|
A dataset containing only the entries for the second packet at an ESA step.
|
|
562
562
|
"""
|
|
563
563
|
epoch_dataset = l1b_dataset.drop_dims("event_met")
|
|
@@ -606,7 +606,7 @@ def get_de_clock_ticks_for_esa_step(
|
|
|
606
606
|
----------
|
|
607
607
|
ccsds_met : float
|
|
608
608
|
The CCSDS MET of the second packet in a DE packet pair.
|
|
609
|
-
spin_df :
|
|
609
|
+
spin_df : pandas.DataFrame
|
|
610
610
|
Universal spin table dataframe.
|
|
611
611
|
|
|
612
612
|
Returns
|
|
@@ -760,7 +760,7 @@ class CalibrationProductConfig:
|
|
|
760
760
|
|
|
761
761
|
Parameters
|
|
762
762
|
----------
|
|
763
|
-
path : Path
|
|
763
|
+
path : pathlib.Path
|
|
764
764
|
Location of the Calibration Product configuration CSV file.
|
|
765
765
|
|
|
766
766
|
Returns
|
|
@@ -27,11 +27,11 @@ def hi_l2(
|
|
|
27
27
|
|
|
28
28
|
Parameters
|
|
29
29
|
----------
|
|
30
|
-
psets : list of str or Path
|
|
30
|
+
psets : list of str or pathlib.Path
|
|
31
31
|
List of input PSETs to make a map from.
|
|
32
|
-
geometric_factors_path : str or Path
|
|
32
|
+
geometric_factors_path : str or pathlib.Path
|
|
33
33
|
Where to get the geometric factors from.
|
|
34
|
-
esa_energies_path : str or Path
|
|
34
|
+
esa_energies_path : str or pathlib.Path
|
|
35
35
|
Where to get the energies from.
|
|
36
36
|
descriptor : str
|
|
37
37
|
Output filename descriptor. Contains full configuration for the options
|
|
@@ -39,7 +39,7 @@ def hi_l2(
|
|
|
39
39
|
|
|
40
40
|
Returns
|
|
41
41
|
-------
|
|
42
|
-
l2_dataset : list[
|
|
42
|
+
l2_dataset : list[xarray.Dataset]
|
|
43
43
|
Level 2 IMAP-Hi dataset ready to be written to a CDF file.
|
|
44
44
|
"""
|
|
45
45
|
# TODO: parse descriptor to determine map configuration
|
|
@@ -77,19 +77,19 @@ def generate_hi_map(
|
|
|
77
77
|
|
|
78
78
|
Parameters
|
|
79
79
|
----------
|
|
80
|
-
psets : list of str or Path
|
|
80
|
+
psets : list of str or pathlib.Path
|
|
81
81
|
List of input PSETs to make a map from.
|
|
82
|
-
geometric_factors_path : str or Path
|
|
82
|
+
geometric_factors_path : str or pathlib.Path
|
|
83
83
|
Where to get the geometric factors from.
|
|
84
|
-
esa_energies_path : str or Path
|
|
84
|
+
esa_energies_path : str or pathlib.Path
|
|
85
85
|
Where to get the energies from.
|
|
86
|
-
cg_corrected : bool,
|
|
86
|
+
cg_corrected : bool, Optional
|
|
87
87
|
Whether to apply Compton-Getting correction to the energies. Defaults to
|
|
88
88
|
False.
|
|
89
|
-
direction : str,
|
|
89
|
+
direction : str, Optional
|
|
90
90
|
Apply filtering to PSET data include ram or anti-ram or full spin data.
|
|
91
91
|
Defaults to "full".
|
|
92
|
-
map_spacing : int,
|
|
92
|
+
map_spacing : int, Optional
|
|
93
93
|
Pixel spacing, in degrees, of the output map in degrees. Defaults to 4.
|
|
94
94
|
|
|
95
95
|
Returns
|
|
@@ -209,11 +209,11 @@ def calculate_ena_intensity(
|
|
|
209
209
|
|
|
210
210
|
Parameters
|
|
211
211
|
----------
|
|
212
|
-
map_ds :
|
|
212
|
+
map_ds : xarray.Dataset
|
|
213
213
|
Map dataset that has ena_signal_rate fields calculated.
|
|
214
|
-
geometric_factors_path : str or Path
|
|
214
|
+
geometric_factors_path : str or pathlib.Path
|
|
215
215
|
Where to get the geometric factors from.
|
|
216
|
-
esa_energies_path : str or Path
|
|
216
|
+
esa_energies_path : str or pathlib.Path
|
|
217
217
|
Where to get the energies from.
|
|
218
218
|
|
|
219
219
|
Returns
|
imap_processing/hi/utils.py
CHANGED
|
@@ -64,9 +64,10 @@ class HiConstants:
|
|
|
64
64
|
TOF3_TICK_DUR = 0.5 # 0.5 ns
|
|
65
65
|
|
|
66
66
|
# These values are stored in the TOF telemetry when the TOF timer
|
|
67
|
-
# does not have valid data.
|
|
68
|
-
|
|
69
|
-
|
|
67
|
+
# does not have valid data. See IMAP-Hi Algorithm Document Section
|
|
68
|
+
# 2.2.5 Annotated Direct Events
|
|
69
|
+
TOF1_BAD_VALUES = (511,)
|
|
70
|
+
TOF2_BAD_VALUES = (511,)
|
|
70
71
|
TOF3_BAD_VALUES = (1023,)
|
|
71
72
|
|
|
72
73
|
|
|
@@ -116,15 +117,15 @@ def full_dataarray(
|
|
|
116
117
|
Variable name.
|
|
117
118
|
attrs : dict
|
|
118
119
|
CDF variable attributes. Usually retrieved from ImapCdfAttributes.
|
|
119
|
-
coords : dict,
|
|
120
|
+
coords : dict, Optional
|
|
120
121
|
Coordinate variables for the Dataset. This function will extract the
|
|
121
122
|
sizes of each dimension defined by the attributes dictionary to determine
|
|
122
123
|
the size of the DataArray to be created.
|
|
123
|
-
shape : int or tuple,
|
|
124
|
+
shape : int or tuple, Optional
|
|
124
125
|
Shape of ndarray data array to instantiate in the xarray.DataArray. If
|
|
125
126
|
shape is provided, the DataArray created will have this shape regardless
|
|
126
127
|
of whether coordinates are provided or not.
|
|
127
|
-
fill_value :
|
|
128
|
+
fill_value : Optional, float
|
|
128
129
|
Override the fill value that the DataArray will be filled with. If not
|
|
129
130
|
supplied, the "FILLVAL" value from `attrs` will be used.
|
|
130
131
|
|
|
@@ -171,15 +172,15 @@ def create_dataset_variables(
|
|
|
171
172
|
----------
|
|
172
173
|
variable_names : list[str]
|
|
173
174
|
List of variable names to create.
|
|
174
|
-
variable_shape : int or
|
|
175
|
+
variable_shape : int or Sequence of int, Optional
|
|
175
176
|
Shape of the new variables data ndarray. If not provided the shape will
|
|
176
177
|
attempt to be derived from the coords dictionary.
|
|
177
|
-
coords : dict,
|
|
178
|
+
coords : dict, Optional
|
|
178
179
|
Coordinate variables for the Dataset. If `variable_shape` is not provided
|
|
179
180
|
the dataset variables created will use this dictionary along with variable
|
|
180
181
|
attributes from the CdfAttributeManager to determine the shapes of the
|
|
181
182
|
dataset variables created.
|
|
182
|
-
fill_value :
|
|
183
|
+
fill_value : Optional, float
|
|
183
184
|
Value to fill the new variables data arrays with. If not supplied,
|
|
184
185
|
the fill value is pulled from the CDF variable attributes "FILLVAL"
|
|
185
186
|
attribute.
|