imap-processing 0.19.0__py3-none-any.whl → 0.19.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of imap-processing might be problematic. Click here for more details.

Files changed (64) hide show
  1. imap_processing/_version.py +2 -2
  2. imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +6 -0
  3. imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +31 -894
  4. imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml +279 -255
  5. imap_processing/cdf/config/imap_enamaps_l2-common_variable_attrs.yaml +11 -0
  6. imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml +3 -1
  7. imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml +5 -4
  8. imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +20 -8
  9. imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +33 -31
  10. imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +61 -1
  11. imap_processing/cli.py +62 -71
  12. imap_processing/codice/codice_l0.py +2 -1
  13. imap_processing/codice/codice_l1a.py +47 -49
  14. imap_processing/codice/codice_l1b.py +42 -32
  15. imap_processing/codice/codice_l2.py +105 -7
  16. imap_processing/codice/constants.py +50 -8
  17. imap_processing/codice/data/lo_stepping_values.csv +1 -1
  18. imap_processing/ena_maps/ena_maps.py +39 -18
  19. imap_processing/ena_maps/utils/corrections.py +291 -0
  20. imap_processing/ena_maps/utils/map_utils.py +20 -4
  21. imap_processing/glows/l1b/glows_l1b.py +38 -23
  22. imap_processing/glows/l1b/glows_l1b_data.py +10 -11
  23. imap_processing/hi/hi_l1c.py +4 -109
  24. imap_processing/hi/hi_l2.py +34 -23
  25. imap_processing/hi/utils.py +109 -0
  26. imap_processing/ialirt/l0/ialirt_spice.py +1 -0
  27. imap_processing/ialirt/utils/create_xarray.py +1 -1
  28. imap_processing/lo/ancillary_data/imap_lo_hydrogen-geometric-factor_v001.csv +75 -0
  29. imap_processing/lo/ancillary_data/imap_lo_oxygen-geometric-factor_v001.csv +75 -0
  30. imap_processing/lo/l1b/lo_l1b.py +90 -16
  31. imap_processing/lo/l1c/lo_l1c.py +164 -50
  32. imap_processing/lo/l2/lo_l2.py +941 -127
  33. imap_processing/mag/l1d/mag_l1d_data.py +36 -3
  34. imap_processing/mag/l2/mag_l2.py +2 -0
  35. imap_processing/mag/l2/mag_l2_data.py +4 -3
  36. imap_processing/quality_flags.py +14 -0
  37. imap_processing/spice/geometry.py +15 -8
  38. imap_processing/spice/pointing_frame.py +4 -2
  39. imap_processing/spice/repoint.py +49 -0
  40. imap_processing/ultra/constants.py +29 -0
  41. imap_processing/ultra/l1b/badtimes.py +35 -11
  42. imap_processing/ultra/l1b/de.py +15 -9
  43. imap_processing/ultra/l1b/extendedspin.py +24 -12
  44. imap_processing/ultra/l1b/goodtimes.py +112 -0
  45. imap_processing/ultra/l1b/lookup_utils.py +1 -1
  46. imap_processing/ultra/l1b/ultra_l1b.py +7 -7
  47. imap_processing/ultra/l1b/ultra_l1b_culling.py +8 -4
  48. imap_processing/ultra/l1b/ultra_l1b_extended.py +79 -43
  49. imap_processing/ultra/l1c/helio_pset.py +68 -39
  50. imap_processing/ultra/l1c/l1c_lookup_utils.py +45 -12
  51. imap_processing/ultra/l1c/spacecraft_pset.py +81 -37
  52. imap_processing/ultra/l1c/ultra_l1c.py +27 -22
  53. imap_processing/ultra/l1c/ultra_l1c_culling.py +7 -0
  54. imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +41 -41
  55. imap_processing/ultra/l2/ultra_l2.py +54 -10
  56. imap_processing/ultra/utils/ultra_l1_utils.py +10 -5
  57. {imap_processing-0.19.0.dist-info → imap_processing-0.19.2.dist-info}/METADATA +1 -1
  58. {imap_processing-0.19.0.dist-info → imap_processing-0.19.2.dist-info}/RECORD +62 -60
  59. imap_processing/ultra/l1b/cullingmask.py +0 -90
  60. imap_processing/ultra/l1c/histogram.py +0 -36
  61. /imap_processing/glows/ancillary/{imap_glows_pipeline_settings_20250923_v002.json → imap_glows_pipeline-settings_20250923_v002.json} +0 -0
  62. {imap_processing-0.19.0.dist-info → imap_processing-0.19.2.dist-info}/LICENSE +0 -0
  63. {imap_processing-0.19.0.dist-info → imap_processing-0.19.2.dist-info}/WHEEL +0 -0
  64. {imap_processing-0.19.0.dist-info → imap_processing-0.19.2.dist-info}/entry_points.txt +0 -0
@@ -163,6 +163,17 @@ ena_rate:
163
163
  DISPLAY_TYPE: image
164
164
  DICT_KEY: SPASE>Particle>ParticleType:Atom,ParticleQuantity:NumberFlux,Qualifier:Incident,CoordinateSystemName:HAE,CoordinateRepresentation:Spherical
165
165
 
166
+ background_rates:
167
+ <<: *default_float32
168
+ CATDESC: Estimated background count rate.
169
+ FIELDNAM: Background Rate
170
+ UNITS: counts/s
171
+ DEPEND_0: epoch
172
+ VAR_TYPE: data
173
+ LABLAXIS: Rate
174
+ DISPLAY_TYPE: image
175
+ DICT_KEY: SPASE>Particle>ParticleType:Atom,ParticleQuantity:NumberFlux,Qualifier:Incident,CoordinateSystemName:HAE,CoordinateRepresentation:Spherical
176
+
166
177
  ena_count:
167
178
  <<: *default_float32
168
179
  CATDESC: Mono-energetic ENA Count.
@@ -203,9 +203,11 @@ number_of_spins_per_block:
203
203
  VALIDMIN: 1
204
204
 
205
205
  unique_block_identifier:
206
- <<: *support_data_defaults
207
206
  CATDESC: YYYY-MM-DDThh:mm:ss based on IMAP UTC time
208
207
  FIELDNAM: YYYY-MM-DDThh:mm:ss based on IMAP UTC time
208
+ FORMAT: A19
209
+ VAR_TYPE: support_data
210
+ DEPEND_0: epoch
209
211
 
210
212
  number_of_bins_per_histogram:
211
213
  <<: *support_data_defaults
@@ -107,8 +107,9 @@ imap_lo_l1c_pset:
107
107
  Logical_source: imap_lo_l1c_pset
108
108
  Logical_source_description: IMAP Mission IMAP-Lo Instrument Level-1C Data
109
109
 
110
- imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-3mo:
110
+ # Global attributes for different sensors and sky tiling types and durations
111
+ imap_lo_l2_enamap:
111
112
  <<: *instrument_base
112
- Data_type: L2_l090-ena-h-sf-nsp-ram-hae-6deg-3mo>Level-2 Low-Energy ENA Maps
113
- Logical_source: imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-3mo
114
- Logical_source_description: IMAP Mission IMAP-Lo Instrument Level-2 Low-Energy ENA Maps
113
+ Data_type: L2_{descriptor}>Level-2 Intensity Map for Lo
114
+ Logical_source: imap_lo_l2_{descriptor}
115
+ Logical_source_description: IMAP-Lo Instrument Level-2 Intensity Map Data for Lo
@@ -224,17 +224,17 @@ imap_ultra_l1b_90sensor-extendedspin:
224
224
  Logical_source: imap_ultra_l1b_90sensor-extendedspin
225
225
  Logical_source_description: IMAP-Ultra Instrument Level-1B Extended Spin Data.
226
226
 
227
- imap_ultra_l1b_45sensor-cullingmask:
227
+ imap_ultra_l1b_45sensor-goodtimes:
228
228
  <<: *instrument_base
229
- Data_type: L1B_45Sensor-CullingMask>Level-1B Culling Mask for Ultra45
230
- Logical_source: imap_ultra_l1b_45sensor-cullingmask
231
- Logical_source_description: IMAP-Ultra Instrument Level-1B Culling Mask Data.
229
+ Data_type: L1B_45Sensor-Goodtimes>Level-1B Goodtimes for Ultra45
230
+ Logical_source: imap_ultra_l1b_45sensor-goodtimes
231
+ Logical_source_description: IMAP-Ultra Instrument Level-1B Goodtimes Data.
232
232
 
233
- imap_ultra_l1b_90sensor-cullingmask:
233
+ imap_ultra_l1b_90sensor-goodtimes:
234
234
  <<: *instrument_base
235
- Data_type: L1B_90Sensor-CullingMask>Level-1B Culling Mask for Ultra90
236
- Logical_source: imap_ultra_l1b_90sensor-cullingmask
237
- Logical_source_description: IMAP-Ultra Instrument Level-1B Culling Mask Data.
235
+ Data_type: L1B_90Sensor-Goodtimes>Level-1B Goodtimes for Ultra90
236
+ Logical_source: imap_ultra_l1b_90sensor-goodtimes
237
+ Logical_source_description: IMAP-Ultra Instrument Level-1B Goodtimes Data.
238
238
 
239
239
  imap_ultra_l1b_45sensor-badtimes:
240
240
  <<: *instrument_base
@@ -260,6 +260,18 @@ imap_ultra_l1c_90sensor-spacecraftpset:
260
260
  Logical_source: imap_ultra_l1c_90sensor-spacecraftpset
261
261
  Logical_source_description: IMAP-Ultra Instrument Level-1C Spacecraft Pointing Set Grid.
262
262
 
263
+ imap_ultra_l1c_45sensor-spacecraftpset-nonproton:
264
+ <<: *instrument_base
265
+ Data_type: L1C_45Sensor-PSET>Level-1C Pointing Set Grid for Ultra45
266
+ Logical_source: imap_ultra_l1c_45sensor-spacecraftpset-nonproton
267
+ Logical_source_description: IMAP-Ultra Instrument Level-1C Spacecraft Pointing Set Grid for Non-Proton Species.
268
+
269
+ imap_ultra_l1c_90sensor-spacecraftpset-nonproton:
270
+ <<: *instrument_base
271
+ Data_type: L1C_90Sensor-PSET>Level-1C Pointing Set Grid for Ultra90
272
+ Logical_source: imap_ultra_l1c_90sensor-spacecraftpset-nonproton
273
+ Logical_source_description: IMAP-Ultra Instrument Level-1C Spacecraft Pointing Set Grid for Non-Proton Species.
274
+
263
275
  imap_ultra_l1c_45sensor-heliopset:
264
276
  <<: *instrument_base
265
277
  Data_type: L1C_45Sensor-PSET>Level-1C Pointing Set Grid for Ultra45
@@ -72,6 +72,22 @@ default_float64_attrs: &default_float64
72
72
  VALIDMAX: 1.7976931348623157e+308
73
73
  dtype: float64
74
74
 
75
+ computed_ebin:
76
+ <<: *default_uint8
77
+ CATDESC: Computed event bin index
78
+ FIELDNAM: Computed ebin
79
+ LABLAXIS: computed_ebin
80
+ UNITS: " "
81
+ DEPEND_0: epoch
82
+
83
+ ebin:
84
+ <<: *default_uint8
85
+ CATDESC: Event bin index
86
+ FIELDNAM: Ebin
87
+ LABLAXIS: ebin
88
+ UNITS: " "
89
+ DEPEND_0: epoch
90
+
75
91
  x_front:
76
92
  <<: *default_float32
77
93
  CATDESC: x front position
@@ -223,7 +239,7 @@ energy_heliosphere:
223
239
  species:
224
240
  <<: *default_uint8
225
241
  DISPLAY_TYPE: no_plot
226
- CATDESC: Label species type.
242
+ CATDESC: Label species type. Non-proton (heavy ion) = 0, proton = 1, misc = 3
227
243
  FIELDNAM: Label species type.
228
244
  LABLAXIS: species
229
245
  UNITS: " "
@@ -348,8 +364,7 @@ spin_start_time:
348
364
  LABLAXIS: spin start time
349
365
  # TODO: come back to format
350
366
  UNITS: s
351
- DEPEND_0: epoch
352
- DEPEND_1: spin_number
367
+ DEPEND_0: spin_number
353
368
 
354
369
  spin_period:
355
370
  <<: *default
@@ -357,8 +372,7 @@ spin_period:
357
372
  FIELDNAM: spin_period
358
373
  LABLAXIS: spin_period
359
374
  UNITS: s
360
- DEPEND_0: epoch
361
- DEPEND_1: spin_number
375
+ DEPEND_0: spin_number
362
376
 
363
377
  spin_rate:
364
378
  <<: *default
@@ -366,8 +380,7 @@ spin_rate:
366
380
  FIELDNAM: spin_rate
367
381
  LABLAXIS: spin_rate
368
382
  UNITS: rpm
369
- DEPEND_0: epoch
370
- DEPEND_1: spin_number
383
+ DEPEND_0: spin_number
371
384
 
372
385
  rate_start_pulses:
373
386
  <<: *default
@@ -415,9 +428,8 @@ ena_rates:
415
428
  CATDESC: Rates calculated from de packet.
416
429
  FIELDNAM: ena_rates
417
430
  LABLAXIS: ena rates
418
- DEPEND_0: epoch
431
+ DEPEND_0: energy_bin_geometric_mean
419
432
  DEPEND_1: spin_number
420
- DEPEND_2: energy_bin_geometric_mean
421
433
  UNITS: " "
422
434
 
423
435
  start_pulses_per_spin:
@@ -425,8 +437,7 @@ start_pulses_per_spin:
425
437
  CATDESC: Total start pulses per spin.
426
438
  FIELDNAM: start_pulses_per_spin
427
439
  LABLAXIS: start pulses per spin
428
- DEPEND_0: epoch
429
- DEPEND_1: spin_number
440
+ DEPEND_0: spin_number
430
441
  UNITS: " "
431
442
 
432
443
  stop_pulses_per_spin:
@@ -434,8 +445,7 @@ stop_pulses_per_spin:
434
445
  CATDESC: Total start pulses per spin.
435
446
  FIELDNAM: stop_pulses_per_spin
436
447
  LABLAXIS: stop pulses per spin
437
- DEPEND_0: epoch
438
- DEPEND_1: spin_number
448
+ DEPEND_0: spin_number
439
449
  UNITS: " "
440
450
 
441
451
  coin_pulses_per_spin:
@@ -443,8 +453,7 @@ coin_pulses_per_spin:
443
453
  CATDESC: Total coincidence pulses per spin.
444
454
  FIELDNAM: coin_pulses_per_spin
445
455
  LABLAXIS: coin_pulses_per_spin
446
- DEPEND_0: epoch
447
- DEPEND_1: spin_number
456
+ DEPEND_0: spin_number
448
457
  UNITS: " "
449
458
 
450
459
  rejected_events_per_spin:
@@ -452,8 +461,7 @@ rejected_events_per_spin:
452
461
  CATDESC: Rejected events per spin.
453
462
  FIELDNAM: rejected_events_per_spin
454
463
  LABLAXIS: rejected events per spin
455
- DEPEND_0: epoch
456
- DEPEND_1: spin_number
464
+ DEPEND_0: spin_number
457
465
  UNITS: " "
458
466
 
459
467
  ena_rates_threshold:
@@ -461,9 +469,8 @@ ena_rates_threshold:
461
469
  CATDESC: Rates threshold used for flagging data.
462
470
  FIELDNAM: ena_rates_threshold
463
471
  LABLAXIS: ena_rates_threshold
464
- DEPEND_0: epoch
472
+ DEPEND_0: energy_bin_geometric_mean
465
473
  DEPEND_1: spin_number
466
- DEPEND_2: energy_bin_geometric_mean
467
474
  UNITS: " "
468
475
 
469
476
  quality_ena_rates:
@@ -471,9 +478,7 @@ quality_ena_rates:
471
478
  CATDESC: Spin filter derived from Ultra ena rates. Bitwise flagging used to filter the spin.
472
479
  FIELDNAM: quality_ena_rates
473
480
  LABLAXIS: quality_ena_rates
474
- DEPEND_0: epoch
475
- DEPEND_1: spin_number
476
- DEPEND_2: energy_bin_geometric_mean
481
+ DEPEND_0: spin_number
477
482
  UNITS: " "
478
483
 
479
484
  quality_attitude:
@@ -483,9 +488,8 @@ quality_attitude:
483
488
  LABLAXIS: quality attitude
484
489
  # TODO: come back to format
485
490
  UNITS: " "
486
- DEPEND_0: epoch
487
- DEPEND_1: spin_number
488
- DEPEND_2: energy_bin_geometric_mean
491
+ DEPEND_0: spin_number
492
+ DEPEND_1: energy_bin_geometric_mean
489
493
 
490
494
  quality_instruments:
491
495
  <<: *default_uint16
@@ -494,9 +498,8 @@ quality_instruments:
494
498
  LABLAXIS: quality instruments
495
499
  # TODO: come back to format
496
500
  UNITS: " "
497
- DEPEND_0: epoch
498
- DEPEND_1: spin_number
499
- DEPEND_2: energy_bin_geometric_mean
501
+ DEPEND_0: spin_number
502
+ DEPEND_1: energy_bin_geometric_mean
500
503
 
501
504
  quality_hk:
502
505
  <<: *default_uint16
@@ -505,9 +508,8 @@ quality_hk:
505
508
  LABLAXIS: quality hk
506
509
  # TODO: come back to format
507
510
  UNITS: " "
508
- DEPEND_0: epoch
509
- DEPEND_1: spin_number
510
- DEPEND_2: energy_bin_geometric_mean
511
+ DEPEND_0: spin_number
512
+ DEPEND_1: energy_bin_geometric_mean
511
513
 
512
514
  quality_outliers:
513
515
  <<: *default_uint16
@@ -17,6 +17,14 @@ default_float32_attrs: &default_float32
17
17
  VALIDMAX: 3.4028235e+38
18
18
  dtype: float32
19
19
 
20
+ default_uint16_attrs: &default_uint16
21
+ <<: *default
22
+ FILLVAL: 65535
23
+ FORMAT: I5
24
+ VALIDMIN: 0
25
+ VALIDMAX: 65535
26
+ dtype: uint16
27
+
20
28
  energy_dependent_attrs: &energy_dependent
21
29
  <<: *default_float32
22
30
  DEPEND_1: energy_bin_geometric_mean
@@ -68,7 +76,7 @@ geometric_function:
68
76
  CATDESC: The effective sensitive area as a function of theta and phi in instrument frame (energy independent).
69
77
  FIELDNAM: geometric_function
70
78
  LABLAXIS: Geometric Factor
71
- UNITS: " "
79
+ UNITS: cm^2
72
80
  VALIDMIN: 0.0
73
81
 
74
82
  efficiency:
@@ -96,6 +104,27 @@ helio_exposure_factor:
96
104
  # TODO: come back to format
97
105
  UNITS: seconds
98
106
 
107
+ scatter_theta:
108
+ <<: *energy_dependent
109
+ CATDESC: Scattering theta values for a pointing.
110
+ FIELDNAM: scatter_theta
111
+ LABLAXIS: scatter theta
112
+ UNITS: degrees
113
+
114
+ scatter_phi:
115
+ <<: *energy_dependent
116
+ CATDESC: Scattering phi values for a pointing.
117
+ FIELDNAM: scatter_phi
118
+ LABLAXIS: scatter phi
119
+ UNITS: degrees
120
+
121
+ scatter_threshold:
122
+ <<: *energy_dependent
123
+ CATDESC: Scattering threshold values for each energy bin for a pointing.
124
+ FIELDNAM: scatter_threshold
125
+ LABLAXIS: scatter threshold
126
+ UNITS: degrees
127
+
99
128
  counts:
100
129
  <<: *default
101
130
  CATDESC: Counts for a spin.
@@ -106,6 +135,17 @@ counts:
106
135
  # TODO: come back to format
107
136
  UNITS: counts
108
137
 
138
+
139
+ quality_flags:
140
+ <<: *default_uint16
141
+ CATDESC: Spin filter derived from Ultra ena rates. Bitwise flagging used to filter the spin.
142
+ FIELDNAM: quality_ena_rates
143
+ LABLAXIS: quality_ena_rates
144
+ DEPEND_0: epoch
145
+ DEPEND_1: spin_number
146
+ DEPEND_2: energy_bin_geometric_mean
147
+ UNITS: " "
148
+
109
149
  background_rates:
110
150
  <<: *default_float32
111
151
  CATDESC: Background rates. Background dominated by accidentals caused by a combination of UV light and misregistered low energy ENA events.
@@ -137,3 +177,23 @@ energy_bin_delta:
137
177
  FIELDNAM: energy_bin_delta
138
178
  LABLAXIS: energy bin delta
139
179
  UNITS: keV
180
+
181
+ energy_delta_minus:
182
+ <<: *default_float32
183
+ VAR_TYPE: support_data
184
+ CATDESC: Difference between the energy bin center and lower edge.
185
+ LABLAXIS: energy
186
+ UNITS: KeV
187
+ FIELDNAM: energy_bin_delta_minus
188
+ DISPLAY_TYPE: no_plot
189
+ DEPEND_1: energy_bin_geometric_mean
190
+
191
+ energy_delta_plus:
192
+ <<: *default_float32
193
+ VAR_TYPE: support_data
194
+ CATDESC: Difference between the energy bin center and upper edge.
195
+ LABLAXIS: energy
196
+ UNITS: KeV
197
+ FIELDNAM: energy_bin_delta_plus
198
+ DISPLAY_TYPE: no_plot
199
+ DEPEND_1: energy_bin_geometric_mean
imap_processing/cli.py CHANGED
@@ -24,8 +24,7 @@ import imap_data_access
24
24
  import numpy as np
25
25
  import spiceypy
26
26
  import xarray as xr
27
- from imap_data_access import ScienceFilePath
28
- from imap_data_access.io import download
27
+ from imap_data_access.io import IMAPDataAccessError, download
29
28
  from imap_data_access.processing_input import (
30
29
  ProcessingInputCollection,
31
30
  ProcessingInputType,
@@ -407,32 +406,23 @@ class ProcessInstrument(ABC):
407
406
  A list of file paths to upload to the SDC.
408
407
  """
409
408
  if self.upload_to_sdc:
410
- # Validate that the files don't already exist
411
- for filename in products:
412
- file_path = ScienceFilePath(filename)
413
- existing_file = imap_data_access.query(
414
- instrument=file_path.instrument,
415
- data_level=file_path.data_level,
416
- descriptor=file_path.descriptor,
417
- start_date=file_path.start_date,
418
- end_date=file_path.start_date,
419
- repointing=file_path.repointing,
420
- version=file_path.version,
421
- extension="cdf",
422
- table="science",
423
- )
424
- if existing_file:
425
- raise ProcessInstrument.ImapFileExistsError(
426
- f"File {filename} already exists in the IMAP SDC. "
427
- "No files were uploaded."
428
- f"Generated files: {products}."
429
- )
430
-
431
- if len(products) == 0:
409
+ if not products:
432
410
  logger.info("No files to upload.")
411
+ return
412
+
433
413
  for filename in products:
434
- logger.info(f"Uploading file: {filename}")
435
- imap_data_access.upload(filename)
414
+ try:
415
+ logger.info(f"Uploading file: {filename}")
416
+ imap_data_access.upload(filename)
417
+ except IMAPDataAccessError as e:
418
+ msg = str(e)
419
+ if "FileAlreadyExists" in msg and "409" in msg:
420
+ logger.warning("Skipping upload of existing file, %s", filename)
421
+ continue
422
+ else:
423
+ logger.error(f"Upload failed with error: {msg}")
424
+ except Exception as e:
425
+ logger.error(f"Upload failed unknown error: {e}")
436
426
 
437
427
  @final
438
428
  def process(self) -> None:
@@ -676,7 +666,7 @@ class Glows(ProcessInstrument):
676
666
  datasets: list[xr.Dataset] = []
677
667
 
678
668
  if self.data_level == "l1a":
679
- science_files = dependencies.get_file_paths(source="glows")
669
+ science_files = dependencies.get_file_paths(source="glows", data_type="l0")
680
670
  if len(science_files) != 1:
681
671
  raise ValueError(
682
672
  f"GLOWS L1A requires exactly one input science file, received: "
@@ -685,33 +675,30 @@ class Glows(ProcessInstrument):
685
675
  datasets = glows_l1a(science_files[0])
686
676
 
687
677
  if self.data_level == "l1b":
688
- science_files = dependencies.get_file_paths(source="glows")
678
+ science_files = dependencies.get_file_paths(source="glows", data_type="l1a")
689
679
  if len(science_files) != 1:
690
680
  raise ValueError(
691
681
  f"GLOWS L1B requires exactly one input science file, received: "
692
682
  f"{science_files}."
693
683
  )
694
684
  input_dataset = load_cdf(science_files[0])
695
- # TODO: Replace this by reading from AWS/ProcessingInputs
696
-
697
- glows_ancillary_dir = Path(__file__).parent / "glows" / "ancillary"
698
685
 
699
686
  # Create file lists for each ancillary type
700
- excluded_regions_files = [
701
- glows_ancillary_dir
702
- / "imap_glows_map-of-excluded-regions_20250923_v002.dat"
703
- ]
704
- uv_sources_files = [
705
- glows_ancillary_dir / "imap_glows_map-of-uv-sources_20250923_v002.dat"
706
- ]
707
- suspected_transients_files = [
708
- glows_ancillary_dir
709
- / "imap_glows_suspected-transients_20250923_v002.dat"
710
- ]
711
- exclusions_by_instr_team_files = [
712
- glows_ancillary_dir
713
- / "imap_glows_exclusions-by-instr-team_20250923_v002.dat"
714
- ]
687
+ excluded_regions_files = dependencies.get_processing_inputs(
688
+ descriptor="map-of-excluded-regions"
689
+ )[0]
690
+ uv_sources_files = dependencies.get_processing_inputs(
691
+ descriptor="map-of-uv-sources"
692
+ )[0]
693
+ suspected_transients_files = dependencies.get_processing_inputs(
694
+ descriptor="suspected-transients"
695
+ )[0]
696
+ exclusions_by_instr_team_files = dependencies.get_processing_inputs(
697
+ descriptor="exclusions-by-instr-team"
698
+ )[0]
699
+ pipeline_settings = dependencies.get_processing_inputs(
700
+ descriptor="pipeline-settings"
701
+ )[0]
715
702
 
716
703
  # Use end date buffer for ancillary data
717
704
  current_day = np.datetime64(
@@ -730,6 +717,9 @@ class Glows(ProcessInstrument):
730
717
  exclusions_by_instr_team_combiner = GlowsAncillaryCombiner(
731
718
  exclusions_by_instr_team_files, day_buffer
732
719
  )
720
+ pipeline_settings_combiner = GlowsAncillaryCombiner(
721
+ pipeline_settings, day_buffer
722
+ )
733
723
 
734
724
  datasets = [
735
725
  glows_l1b(
@@ -738,6 +728,7 @@ class Glows(ProcessInstrument):
738
728
  uv_sources_combiner.combined_dataset,
739
729
  suspected_transients_combiner.combined_dataset,
740
730
  exclusions_by_instr_team_combiner.combined_dataset,
731
+ pipeline_settings_combiner.combined_dataset,
741
732
  )
742
733
  ]
743
734
 
@@ -1020,16 +1011,19 @@ class Lo(ProcessInstrument):
1020
1011
  elif self.data_level == "l1b":
1021
1012
  data_dict = {}
1022
1013
  science_files = dependencies.get_file_paths(source="lo", data_type="l1a")
1014
+ ancillary_files = dependencies.get_file_paths(
1015
+ source="lo", data_type="ancillary"
1016
+ )
1023
1017
  logger.info(f"Science files for L1B: {science_files}")
1024
1018
  for file in science_files:
1025
1019
  dataset = load_cdf(file)
1026
1020
  data_dict[dataset.attrs["Logical_source"]] = dataset
1027
- datasets = lo_l1b.lo_l1b(data_dict)
1021
+ datasets = lo_l1b.lo_l1b(data_dict, ancillary_files)
1028
1022
 
1029
1023
  elif self.data_level == "l1c":
1030
1024
  data_dict = {}
1031
1025
  anc_dependencies: list = dependencies.get_file_paths(
1032
- source="lo", descriptor="goodtimes"
1026
+ source="lo", data_type="ancillary"
1033
1027
  )
1034
1028
  science_files = dependencies.get_file_paths(source="lo", descriptor="de")
1035
1029
  for file in science_files:
@@ -1039,13 +1033,11 @@ class Lo(ProcessInstrument):
1039
1033
 
1040
1034
  elif self.data_level == "l2":
1041
1035
  data_dict = {}
1042
- # TODO: Add ancillary descriptors when maps using them are
1043
- # implemented.
1044
- anc_dependencies = []
1045
1036
  science_files = dependencies.get_file_paths(source="lo", descriptor="pset")
1046
- psets = []
1047
- for file in science_files:
1048
- psets.append(load_cdf(file))
1037
+ anc_dependencies = dependencies.get_file_paths(data_type="ancillary")
1038
+
1039
+ # Load all pset files into datasets
1040
+ psets = [load_cdf(file) for file in science_files]
1049
1041
  data_dict[psets[0].attrs["Logical_source"]] = psets
1050
1042
  datasets = lo_l2.lo_l2(data_dict, anc_dependencies, self.descriptor)
1051
1043
  return datasets
@@ -1228,8 +1220,8 @@ class Spacecraft(ProcessInstrument):
1228
1220
  The list of processed products.
1229
1221
  """
1230
1222
  print(f"Processing Spacecraft {self.data_level}")
1231
-
1232
- if self.data_level == "l1a":
1223
+ processed_dataset = []
1224
+ if self.descriptor == "quaternions":
1233
1225
  # File path is expected output file path
1234
1226
  input_files = dependencies.get_file_paths(source="spacecraft")
1235
1227
  if len(input_files) > 1:
@@ -1238,26 +1230,21 @@ class Spacecraft(ProcessInstrument):
1238
1230
  f"{input_files}. Expected only one dependency."
1239
1231
  )
1240
1232
  datasets = list(quaternions.process_quaternions(input_files[0]))
1241
- return datasets
1242
- elif self.data_level == "spice":
1233
+ processed_dataset.extend(datasets)
1234
+ elif self.descriptor == "pointing-attitude":
1243
1235
  spice_inputs = dependencies.get_file_paths(
1244
1236
  data_type=SPICESource.SPICE.value
1245
1237
  )
1246
1238
  ah_paths = [path for path in spice_inputs if ".ah" in path.suffixes]
1247
- if len(ah_paths) != 1:
1248
- raise ValueError(
1249
- f"Unexpected spice dependencies found for Spacecraft "
1250
- f"pointing_kernel: {ah_paths}. Expected exactly one "
1251
- f"attitude history file."
1252
- )
1253
1239
  pointing_kernel_paths = pointing_frame.generate_pointing_attitude_kernel(
1254
- ah_paths[0]
1240
+ ah_paths[-1]
1255
1241
  )
1256
- return pointing_kernel_paths
1242
+ processed_dataset.extend(pointing_kernel_paths)
1257
1243
  else:
1258
1244
  raise NotImplementedError(
1259
1245
  f"Spacecraft processing not implemented for level {self.data_level}"
1260
1246
  )
1247
+ return processed_dataset
1261
1248
 
1262
1249
 
1263
1250
  class Swapi(ProcessInstrument):
@@ -1461,7 +1448,10 @@ class Ultra(ProcessInstrument):
1461
1448
  }
1462
1449
  science_files = dependencies.get_file_paths(source="ultra", data_type="l1b")
1463
1450
  l1b_dict = {
1464
- dataset.attrs["Logical_source"]: dataset
1451
+ # TODO remove
1452
+ dataset.attrs["Logical_source"].replace(
1453
+ "cullingmask", "goodtimes"
1454
+ ): dataset
1465
1455
  for dataset in [load_cdf(sci_file) for sci_file in science_files]
1466
1456
  }
1467
1457
  combined = {**l1a_dict, **l1b_dict}
@@ -1470,11 +1460,12 @@ class Ultra(ProcessInstrument):
1470
1460
  for path in anc_paths:
1471
1461
  ancillary_files[path.stem.split("_")[2]] = path
1472
1462
  spice_paths = dependencies.get_file_paths(data_type="spice")
1473
- if spice_paths:
1474
- has_spice = True
1463
+ # Only the helio pset needs IMAP frames
1464
+ if any("imap_frames" in path.as_posix() for path in spice_paths):
1465
+ imap_frames = True
1475
1466
  else:
1476
- has_spice = False
1477
- datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, has_spice)
1467
+ imap_frames = False
1468
+ datasets = ultra_l1c.ultra_l1c(combined, ancillary_files, imap_frames)
1478
1469
  elif self.data_level == "l2":
1479
1470
  all_pset_filepaths = dependencies.get_file_paths(
1480
1471
  source="ultra", descriptor="pset"
@@ -39,7 +39,8 @@ def decom_packets(packet_file: Path) -> dict[int, xr.Dataset]:
39
39
  # TODO: Currently need to use the 'old' packet definition for housekeeping
40
40
  # because the simulated housekeeping data being used has various
41
41
  # mis-matches from the telemetry definition. This may be updated
42
- # once new simulated housekeeping data are acquired.
42
+ # once new simulated housekeeping data are acquired. See GitHub issue
43
+ # #2135.
43
44
  if "hskp" in str(packet_file):
44
45
  xtce_filename = "P_COD_NHK.xml"
45
46
  else: