roc-film 1.13.4__py3-none-any.whl → 1.14.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. roc/__init__.py +2 -1
  2. roc/film/__init__.py +2 -2
  3. roc/film/commands.py +372 -323
  4. roc/film/config/__init__.py +0 -1
  5. roc/film/constants.py +101 -65
  6. roc/film/descriptor.json +126 -95
  7. roc/film/exceptions.py +28 -27
  8. roc/film/tasks/__init__.py +16 -16
  9. roc/film/tasks/cat_solo_hk.py +86 -74
  10. roc/film/tasks/cdf_postpro.py +438 -309
  11. roc/film/tasks/check_dds.py +39 -45
  12. roc/film/tasks/db_to_anc_bia_sweep_table.py +381 -0
  13. roc/film/tasks/dds_to_l0.py +232 -180
  14. roc/film/tasks/export_solo_coord.py +147 -0
  15. roc/film/tasks/file_handler.py +91 -75
  16. roc/film/tasks/l0_to_hk.py +117 -103
  17. roc/film/tasks/l0_to_l1_bia_current.py +38 -30
  18. roc/film/tasks/l0_to_l1_bia_sweep.py +417 -329
  19. roc/film/tasks/l0_to_l1_sbm.py +250 -208
  20. roc/film/tasks/l0_to_l1_surv.py +185 -130
  21. roc/film/tasks/make_daily_tm.py +40 -37
  22. roc/film/tasks/merge_tcreport.py +77 -71
  23. roc/film/tasks/merge_tmraw.py +102 -89
  24. roc/film/tasks/parse_dds_xml.py +21 -20
  25. roc/film/tasks/set_l0_utc.py +51 -49
  26. roc/film/tests/cdf_compare.py +565 -0
  27. roc/film/tests/hdf5_compare.py +84 -62
  28. roc/film/tests/test_dds_to_l0.py +93 -51
  29. roc/film/tests/test_dds_to_tc.py +8 -11
  30. roc/film/tests/test_dds_to_tm.py +8 -10
  31. roc/film/tests/test_film.py +161 -116
  32. roc/film/tests/test_l0_to_hk.py +64 -36
  33. roc/film/tests/test_l0_to_l1_bia.py +10 -14
  34. roc/film/tests/test_l0_to_l1_sbm.py +14 -19
  35. roc/film/tests/test_l0_to_l1_surv.py +68 -41
  36. roc/film/tests/test_metadata.py +21 -20
  37. roc/film/tests/tests.py +743 -396
  38. roc/film/tools/__init__.py +5 -5
  39. roc/film/tools/dataset_tasks.py +34 -2
  40. roc/film/tools/file_helpers.py +390 -269
  41. roc/film/tools/l0.py +402 -324
  42. roc/film/tools/metadata.py +147 -127
  43. roc/film/tools/skeleton.py +12 -17
  44. roc/film/tools/tools.py +109 -92
  45. roc/film/tools/xlsx2skt.py +161 -139
  46. {roc_film-1.13.4.dist-info → roc_film-1.14.0.dist-info}/LICENSE +127 -125
  47. roc_film-1.14.0.dist-info/METADATA +60 -0
  48. roc_film-1.14.0.dist-info/RECORD +50 -0
  49. {roc_film-1.13.4.dist-info → roc_film-1.14.0.dist-info}/WHEEL +1 -1
  50. roc/film/tasks/l0_to_anc_bia_sweep_table.py +0 -348
  51. roc_film-1.13.4.dist-info/METADATA +0 -120
  52. roc_film-1.13.4.dist-info/RECORD +0 -48
@@ -8,194 +8,204 @@ from datetime import datetime, timedelta
8
8
  from pathlib import Path
9
9
  import uuid
10
10
 
11
+ from sqlalchemy import and_
11
12
  import numpy as np
12
13
 
13
14
  from poppy.core.logger import logger
14
15
  from poppy.core.task import Task
15
16
  from poppy.core.target import FileTarget
17
+ from poppy.core.db.connector import Connector
16
18
  from roc.rap.tasks.bia.current import raw_to_na
17
19
 
18
20
  from roc.rpl.time import Time
19
21
 
20
- from roc.film.tasks.l0_to_anc_bia_sweep_table import L0ToAncBiaSweepTable
21
- from roc.film.constants import ANT_1_FLAG, ANT_2_FLAG, ANT_3_FLAG
22
+ from roc.dingo.models.data import EventLog
23
+ from roc.dingo.tools import query_db, get_columns
24
+
25
+ from roc.film.tasks.db_to_anc_bia_sweep_table import DbToAncBiaSweepTable
26
+ from roc.film.constants import (
27
+ ANT_1_FLAG,
28
+ ANT_2_FLAG,
29
+ ANT_3_FLAG,
30
+ PIPELINE_DATABASE,
31
+ TRYOUTS,
32
+ SQL_LIMIT,
33
+ TIME_WAIT_SEC,
34
+ BIA_SWEEP_TABLE_PACKETS,
35
+ )
22
36
  from roc.film.tools.l0 import L0
23
- from roc.film.tools.file_helpers import l0_to_trange_cdf, get_l0_trange, \
24
- get_l0_files, \
25
- get_output_dir, is_output_dir
26
- from roc.film.tools.tools import unique_dict_list, sort_dict_list, extract_file_fields
37
+ from roc.film.tools.file_helpers import (
38
+ l0_to_trange_cdf,
39
+ get_l0_trange,
40
+ get_l0_files,
41
+ get_output_dir,
42
+ is_output_dir,
43
+ )
44
+
45
+ __all__ = ["L0ToL1BiaSweep"]
27
46
 
28
- __all__ = ['L0ToL1BiaSweep']
29
47
 
30
48
  class L0ToL1BiaSweep(Task):
31
- plugin_name = 'roc.film'
32
- name = 'l0_to_l1_bia_sweep'
49
+ plugin_name = "roc.film"
50
+ name = "l0_to_l1_bia_sweep"
33
51
 
34
52
  def add_targets(self):
53
+ self.add_input(
54
+ target_class=FileTarget,
55
+ identifier="l0_file",
56
+ filepath=get_l0_files,
57
+ many=True,
58
+ )
35
59
 
36
- self.add_input(target_class=FileTarget,
37
- identifier='l0_file',
38
- filepath=get_l0_files,
39
- many=True)
60
+ self.add_output(target_class=FileTarget, identifier="l1_bia_sweep")
40
61
 
41
- self.add_input(target_class=FileTarget,
42
- identifier='anc_bia_sweep_table',
43
- filepath=self.get_sweep_tables)
62
+ def setup_inputs(self):
63
+ """Initialize inputs for the task"""
44
64
 
45
- self.add_output(target_class=FileTarget,
46
- identifier='l1_bia_sweep')
65
+ # get a database session, table model and table columns (except primary key)
66
+ self.session = Connector.manager[PIPELINE_DATABASE].session
67
+ self.model = EventLog
68
+ self.columns = get_columns(self.model, remove=["id"])
47
69
 
48
- def get_sweep_tables(self, pipeline):
49
- try:
50
- return pipeline.args.sweep_tables
51
- except:
52
- # If not defined as input argument, then assume that it is already
53
- # defined as target input
54
- pass
70
+ # Get tryouts from pipeline properties
71
+ self.tryouts = self.pipeline.get("tryouts", default=[TRYOUTS], create=True)[0]
55
72
 
56
- def setup_inputs(self):
57
- """Initialize inputs for the task"""
73
+ # Get wait from pipeline properties
74
+ self.wait = self.pipeline.get("wait", default=[TIME_WAIT_SEC], create=True)[0]
75
+
76
+ # Retrieve --limit keyword value
77
+ self.limit = self.pipeline.get(
78
+ "limit",
79
+ default=[SQL_LIMIT],
80
+ )[0]
58
81
 
59
82
  # Get products directory (folder where final output files will be
60
83
  # moved)
61
- self.products_dir = self.pipeline.get('products_dir',
62
- default=[None], args=True)[0]
84
+ self.products_dir = self.pipeline.get(
85
+ "products_dir", default=[None], args=True
86
+ )[0]
63
87
 
64
88
  # Get output dir
65
89
  self.output_dir = get_output_dir(self.pipeline)
66
- if not is_output_dir(self.output_dir,
67
- products_dir=self.products_dir):
68
- logger.info(f'Making {self.output_dir}')
90
+ if not is_output_dir(self.output_dir, products_dir=self.products_dir):
91
+ logger.info(f"Making {self.output_dir}")
69
92
  os.makedirs(self.output_dir)
70
93
  else:
71
- logger.debug(f'Output files will be '
72
- f'saved into folder {self.output_dir}')
94
+ logger.debug(f"Output files will be saved into folder {self.output_dir}")
73
95
 
74
96
  # Get (optional) arguments for SPICE
75
- predictive = self.pipeline.get('predictive', default=False, args=True)
76
- kernel_date = self.pipeline.get('kernel_date', default=None, args=True)
77
- no_spice = self.pipeline.get('no_spice', default=False, args=True)
97
+ predictive = self.pipeline.get("predictive", default=False, args=True)
98
+ kernel_date = self.pipeline.get("kernel_date", default=None, args=True)
99
+ no_spice = self.pipeline.get("no_spice", default=False, args=True)
78
100
 
79
101
  # Get/create Time singleton
80
- self.time = Time(predictive=predictive,
81
- kernel_date=kernel_date,
82
- no_spice=no_spice)
102
+ self.time = Time(
103
+ predictive=predictive, kernel_date=kernel_date, no_spice=no_spice
104
+ )
83
105
 
84
106
  # Get list of input l0 file(s)
85
- self.l0_file_list = self.inputs['l0_file'].filepath
107
+ self.l0_file_list = self.inputs["l0_file"].filepath
86
108
  self.l0_file_num = len(self.l0_file_list)
87
109
 
88
- # Load data from bias sweep table files
89
- sweep_table_list = []
90
- sweep_table_file_list = []
91
- for sweep_table_file in self.inputs['anc_bia_sweep_table'].filepath:
92
- logger.info(f'Parsing {sweep_table_file}...')
93
- sweep_table = L0ToAncBiaSweepTable.parse_bia_sweep_table_file(
94
- sweep_table_file)
95
- sweep_table_list.extend(sweep_table)
96
- sweep_table_file_list.append(
97
- os.path.basename(sweep_table_file))
98
-
99
- # Make sure to have unique values in the list
100
- logger.debug('Making list of sweep table data unique...')
101
- sweep_table_list = unique_dict_list(sweep_table_list)
102
-
103
- # Re-order rows by ascending time values
104
- logger.debug('Re-ordering sweep table data by ascending times...')
105
- sweep_table_list = sort_dict_list(
106
- sweep_table_list, 'TC_EXE_UTC_TIME')
107
-
108
-
109
- self.sweep_table_list = sweep_table_list
110
- self.sweep_table_file_list = sweep_table_file_list
111
-
112
110
  # Get L0 files time_min/time_max
113
111
  l0_time_min, l0_time_max = get_l0_trange(self.l0_file_list)
114
112
 
115
113
  # Define output file start time
116
- self.start_time = self.pipeline.get('start_time', default=[None])[0]
114
+ self.start_time = self.pipeline.get("start_time", default=[None])[0]
117
115
  if not self.start_time:
118
116
  self.start_time = min(l0_time_min)
119
- logger.debug(f'start_time value is {self.start_time}')
117
+ logger.debug(f"start_time value is {self.start_time}")
120
118
 
121
119
  # Define output file end time
122
- self.end_time = self.pipeline.get('end_time', default=[None])[0]
120
+ self.end_time = self.pipeline.get("end_time", default=[None])[0]
123
121
  if not self.end_time:
124
122
  self.end_time = max(l0_time_max)
125
- logger.debug(f'end_time value is {self.end_time}')
123
+ logger.debug(f"end_time value is {self.end_time}")
126
124
 
127
125
  # Get or create failed_files list from pipeline properties
128
- self.failed_files = self.pipeline.get(
129
- 'failed_files', default=[], create=True)
126
+ self.failed_files = self.pipeline.get("failed_files", default=[], create=True)
130
127
 
131
128
  # Get or create processed_files list from pipeline properties
132
129
  self.processed_files = self.pipeline.get(
133
- 'processed_files', default=[], create=True)
130
+ "processed_files", default=[], create=True
131
+ )
134
132
 
135
133
  # Get force optional keyword
136
- self.force = self.pipeline.get('force', default=False, args=True)
134
+ self.force = self.pipeline.get("force", default=False, args=True)
137
135
 
138
136
  # Get overwrite argument
139
- self.overwrite = self.pipeline.get(
140
- 'overwrite', default=False, args=True)
137
+ self.overwrite = self.pipeline.get("overwrite", default=False, args=True)
141
138
 
142
139
  # Get cdag keyword
143
- self.is_cdag = self.pipeline.get('cdag', default=False, args=True)
140
+ self.is_cdag = self.pipeline.get("cdag", default=False, args=True)
144
141
 
145
142
  return True
146
143
 
147
144
  def run(self):
148
-
149
145
  # Import external classes, methods
150
146
  from spacepy.pycdf import CDF
151
147
 
152
148
  # Define task job ID (long and short)
153
149
  self.job_uuid = str(uuid.uuid4())
154
- self.job_id = f'L0ToL1BiaSweep-{self.job_uuid[:8]}'
155
- logger.info(f'Task {self.job_id} is starting')
150
+ self.job_id = self.job_uuid[:8]
151
+ logger.info(f"Task {self.job_id} is starting")
156
152
  try:
157
153
  self.setup_inputs()
158
- except:
159
- logger.exception(
160
- f'Initializing inputs has failed for {self.job_id}!')
154
+ except Exception as e:
155
+ logger.exception(f"Initializing inputs has failed for {self.job_id}:\n{e}")
161
156
  try:
162
- os.makedirs(os.path.join(self.output_dir, 'failed'))
163
- except:
164
- logger.error('output_dir argument is not defined!')
157
+ os.makedirs(os.path.join(self.output_dir, "failed"))
158
+ except Exception as e:
159
+ logger.error(f"output_dir argument is not defined:\n{e}")
165
160
  self.pipeline.exit()
166
161
  return
167
162
 
163
+ # return list of sweep tables from pipeline.event_log table
164
+ # (as a pandas.DataFrame object)
165
+ self.sweep_table_list = self.load_sweep_tables()
166
+ if self.sweep_table_list.shape[0] == 0:
167
+ return
168
+
168
169
  # Get start times/end times of the Bias sweeps
169
- logger.info(f'building sweep info list from {self.l0_file_num} input L0 files...')
170
- bia_sweep_list = self.build_sweep_info_list(self.l0_file_list,
171
- self.sweep_table_list)
170
+ logger.info(
171
+ f"building sweep info list from {self.l0_file_num} input L0 files..."
172
+ )
173
+ bia_sweep_list = self.build_sweep_info_list(
174
+ self.l0_file_list, self.sweep_table_list
175
+ )
172
176
  bia_sweep_num = len(bia_sweep_list)
173
177
  if bia_sweep_num == 0:
174
- logger.info('No Bias sweep found')
178
+ logger.info("No Bias sweep found")
175
179
  return
176
180
  else:
177
- logger.info(f'{bia_sweep_num} Bias sweeps to process')
181
+ logger.info(f"{bia_sweep_num} Bias sweeps to process")
178
182
 
179
183
  # Loop over each Bias sweep
180
184
  for i, current_sweep in enumerate(bia_sweep_list):
181
-
182
185
  # Get start_time/end_time of current sweep
183
- bia_sweep_start_time = current_sweep['start_time']
184
- bia_sweep_end_time = current_sweep['end_time']
185
- logger.info(f"Processing Bias sweep between {current_sweep['start_time']} and "
186
- f"{current_sweep['end_time']}... ({bia_sweep_num - i} sweeps remaining)")
186
+ bia_sweep_start_time = current_sweep["start_time"]
187
+ bia_sweep_end_time = current_sweep["end_time"]
188
+ logger.info(
189
+ f"Processing Bias sweep between {current_sweep['start_time']} and "
190
+ f"{current_sweep['end_time']}... ({bia_sweep_num - i} sweeps remaining)"
191
+ )
187
192
 
188
193
  # Generate output L1 CDF with LFR CWF F3 data, but not Bias sweep
189
194
  # current values.
190
195
  l1_cdf_path = None
191
196
  try:
192
- l1_cdf_path = l0_to_trange_cdf(self, 'l0_to_l1_bia_sweep',
193
- self.l0_file_list, self.output_dir,
194
- start_time=bia_sweep_start_time,
195
- end_time=bia_sweep_end_time,
196
- overwrite=self.overwrite,
197
- is_cdag=self.is_cdag)
198
- except:
197
+ l1_cdf_path = l0_to_trange_cdf(
198
+ self,
199
+ "l0_to_l1_bia_sweep",
200
+ self.l0_file_list,
201
+ self.output_dir,
202
+ time_instance=self.time,
203
+ start_time=bia_sweep_start_time,
204
+ end_time=bia_sweep_end_time,
205
+ overwrite=self.overwrite,
206
+ is_cdag=self.is_cdag,
207
+ )
208
+ except Exception:
199
209
  if l1_cdf_path and l1_cdf_path[0] not in self.failed_files:
200
210
  self.failed_files.append(l1_cdf_path[0])
201
211
  else:
@@ -203,76 +213,77 @@ class L0ToL1BiaSweep(Task):
203
213
  # What to do if the process has failed
204
214
  # before output file has been generated
205
215
  self.failed_files.append(
206
- (Path(self.output_dir) / Path('l0_to_l1_bia_sweep.failed')).touch())
216
+ (
217
+ Path(self.output_dir) / Path("l0_to_l1_bia_sweep.failed")
218
+ ).touch()
219
+ )
207
220
 
208
221
  logger.exception(
209
- 'Filling L1 Bias sweep CDF with TM BIAS CALIB data has failed!')
222
+ "Filling L1 Bias sweep CDF with TM BIAS CALIB data has failed!"
223
+ )
210
224
 
211
225
  # If output L1 CDF has been correctly generated, then insert Bias
212
226
  # sweep current values
213
227
  if l1_cdf_path and os.path.isfile(l1_cdf_path[0]):
214
-
215
228
  # Open CDF and loop over each Epoch time
216
- logger.info(f'Filling {l1_cdf_path[0]} file with Bias current values...')
217
- cdf = None
229
+ logger.info(
230
+ f"Filling {l1_cdf_path[0]} file with Bias current values..."
231
+ )
218
232
  try:
219
- cdf = CDF(l1_cdf_path[0])
220
- cdf.readonly(False)
221
- epoch_values = cdf['Epoch'][:]
222
-
223
- for i, current_epoch in enumerate(epoch_values):
224
-
225
- # For each epoch time get values of bias intensity on
226
- # each antenna
227
- current_sweep_idx = self._get_sweep_step_idx(current_epoch,
228
- current_sweep)
229
-
230
- if current_sweep_idx is None:
231
- logger.debug(f'No sweep value found for {current_epoch} '
232
- f"(start_time={current_sweep['start_time']} and"
233
- f" end_time={current_sweep['end_time']})")
234
- else:
235
- cdf['BIAS_SWEEP_CURRENT'][i, 0] = current_sweep[
236
- 'step_ibias'][current_sweep_idx][0]
237
- cdf['BIAS_SWEEP_CURRENT'][i, 1] = current_sweep[
238
- 'step_ibias'][current_sweep_idx][1]
239
- cdf['BIAS_SWEEP_CURRENT'][i, 2] = current_sweep[
240
- 'step_ibias'][current_sweep_idx][2]
241
-
242
- # Fill ant flag
243
- cdf['ANT_FLAG'][i] = current_sweep[
244
- 'ant_flag'][current_sweep_idx]
245
-
246
- # Fill bypass
247
- cdf['BIAS_MODE_BYPASS_PROBE1'][i] = current_sweep[
248
- 'bypass'][current_sweep_idx][0]
249
- cdf['BIAS_MODE_BYPASS_PROBE2'][i] = current_sweep[
250
- 'bypass'][current_sweep_idx][1]
251
- cdf['BIAS_MODE_BYPASS_PROBE3'][i] = current_sweep[
252
- 'bypass'][current_sweep_idx][2]
253
-
254
- except:
233
+ with CDF(l1_cdf_path[0]) as cdf:
234
+ cdf.readonly(False)
235
+ epoch_values = cdf["Epoch"][:]
236
+
237
+ for i, current_epoch in enumerate(epoch_values):
238
+ # For each epoch time get values of bias intensity on
239
+ # each antenna
240
+ current_sweep_idx = self._get_sweep_step_idx(
241
+ current_epoch, current_sweep
242
+ )
243
+
244
+ if current_sweep_idx is None:
245
+ logger.debug(
246
+ f"No sweep value found for {current_epoch} "
247
+ f"(start_time={current_sweep['start_time']} and"
248
+ f" end_time={current_sweep['end_time']})"
249
+ )
250
+ else:
251
+ cdf["BIAS_SWEEP_CURRENT"][i, 0] = current_sweep[
252
+ "step_ibias"
253
+ ][current_sweep_idx][0]
254
+ cdf["BIAS_SWEEP_CURRENT"][i, 1] = current_sweep[
255
+ "step_ibias"
256
+ ][current_sweep_idx][1]
257
+ cdf["BIAS_SWEEP_CURRENT"][i, 2] = current_sweep[
258
+ "step_ibias"
259
+ ][current_sweep_idx][2]
260
+
261
+ # Fill ant flag
262
+ cdf["ANT_FLAG"][i] = current_sweep["ant_flag"][
263
+ current_sweep_idx
264
+ ]
265
+
266
+ # Fill bypass
267
+ cdf["BIAS_MODE_BYPASS_PROBE1"][i] = current_sweep[
268
+ "bypass"
269
+ ][current_sweep_idx][0]
270
+ cdf["BIAS_MODE_BYPASS_PROBE2"][i] = current_sweep[
271
+ "bypass"
272
+ ][current_sweep_idx][1]
273
+ cdf["BIAS_MODE_BYPASS_PROBE3"][i] = current_sweep[
274
+ "bypass"
275
+ ][current_sweep_idx][2]
276
+
277
+ except Exception:
255
278
  logger.exception(
256
- 'Filling L1 Bias sweep CDF with Bias sweep intensity data has failed!')
279
+ "Filling L1 Bias sweep CDF with Bias sweep intensity data has failed!"
280
+ )
257
281
  if l1_cdf_path[0] not in self.failed_files:
258
282
  self.failed_files.append(l1_cdf_path[0])
259
-
260
283
  else:
261
- logger.info(f'{l1_cdf_path[0]} filled with Bias sweep intensities')
284
+ logger.info(f"{l1_cdf_path[0]} filled with Bias sweep intensities")
262
285
  if l1_cdf_path[0] not in self.processed_files:
263
286
  self.processed_files.append(l1_cdf_path[0])
264
- finally:
265
- if cdf:
266
- # Add sweep table files to parents
267
- parent_files = 'ANC>' + \
268
- ', '.join(self.sweep_table_file_list)
269
- parent_versions = ', '.join([extract_file_fields(current_version, get_version=True)[1:]
270
- for current_version in self.sweep_table_file_list])
271
- cdf.attrs['Parents'] = [
272
- cdf.attrs['Parents'][0], parent_files]
273
- cdf.attrs['Parent_version'] = [
274
- cdf.attrs['Parent_version'][0], parent_versions]
275
- cdf.close()
276
287
 
277
288
  def _get_sweep_step_idx(self, current_time, current_sweep):
278
289
  """
@@ -284,10 +295,11 @@ class L0ToL1BiaSweep(Task):
284
295
  """
285
296
 
286
297
  i = np.where(
287
- np.array(current_sweep['step_time'], dtype=datetime) <= current_time)
298
+ np.array(current_sweep["step_time"], dtype=datetime) <= current_time
299
+ )
288
300
  try:
289
301
  return i[0][-1]
290
- except:
302
+ except Exception:
291
303
  return None
292
304
 
293
305
  def build_sweep_info_list(self, l0_file_list, sweep_table_list):
@@ -304,23 +316,26 @@ class L0ToL1BiaSweep(Task):
304
316
 
305
317
  # List of Bias sweep packet to retrieve from l0
306
318
  bias_sweep_expected_packet_list = [
307
- 'TM_DPU_EVENT_PR_BIA_SWEEP',
308
- 'TC_DPU_START_BIAS_SWEEP',
309
- 'TM_DPU_BIA_HK'
319
+ "TM_DPU_EVENT_PR_BIA_SWEEP",
320
+ "TC_DPU_START_BIAS_SWEEP",
321
+ "TM_DPU_BIA_HK",
310
322
  ]
311
323
 
312
324
  # Extract wanted packets and re-order by increasing time
313
- bia_sweep_packet_list = L0.l0_to_packet_list(l0_file_list,
314
- include=bias_sweep_expected_packet_list,
315
- start_time=self.start_time,
316
- end_time=self.end_time,
317
- ascending=True,
318
- )
325
+ bia_sweep_packet_list = L0.l0_to_packet_list(
326
+ l0_file_list,
327
+ include=bias_sweep_expected_packet_list,
328
+ start_time=self.start_time,
329
+ end_time=self.end_time,
330
+ ascending=True,
331
+ )
319
332
 
320
333
  bia_sweep_packet_num = len(bia_sweep_packet_list)
321
334
  if bia_sweep_packet_num == 0:
322
- logger.info(f"No {','.join(bias_sweep_expected_packet_list)} found in the "
323
- f'input l0 file list')
335
+ logger.info(
336
+ f"No {','.join(bias_sweep_expected_packet_list)} found in the "
337
+ f"input l0 file list"
338
+ )
324
339
  return sweep_info_list
325
340
 
326
341
  # Initialize loop variables
@@ -331,32 +346,34 @@ class L0ToL1BiaSweep(Task):
331
346
 
332
347
  # Loop over each packet time in the sweep info list
333
348
  for packet_idx, current_packet in enumerate(bia_sweep_packet_list):
334
-
335
- current_time = current_packet['utc_time']
336
- current_name = current_packet['palisade_id']
337
- current_idb_source = current_packet['idb_source']
338
- current_idb_version = current_packet['idb_version']
349
+ current_time = current_packet["utc_time"]
350
+ current_name = current_packet["palisade_id"]
351
+ current_idb_source = current_packet["idb_source"]
352
+ current_idb_version = current_packet["idb_version"]
339
353
 
340
354
  # if current packet is a bias sweep start event
341
- if current_name == 'TM_DPU_EVENT_PR_BIA_SWEEP':
342
-
355
+ if current_name == "TM_DPU_EVENT_PR_BIA_SWEEP":
343
356
  # Get Bias sweep progress code
344
- sweep_pr_code = int(current_packet['PA_DPU_BIA_SWEEP_PR_CODE'])
357
+ sweep_pr_code = int(current_packet["PA_DPU_BIA_SWEEP_PR_CODE"])
345
358
 
346
359
  # Update current time to use PA_DPU_BIA_SWEEP_TIME (UTC) instead of
347
360
  # packet creation on-board time
348
- current_time = Time().obt_to_utc(np.array(
349
- current_packet['PA_DPU_BIA_SWEEP_TIME'][:2], dtype=int).reshape([1, 2]),
350
- to_datetime=True)[0]
351
-
352
- logger.debug(f'{packet_idx}: TM_DPU_EVENT_PR_BIA_SWEEP at {current_time} '
353
- f'with sweep_pr_code {sweep_pr_code} '
354
- f'(running status is {is_running})')
361
+ current_time = Time().obt_to_utc(
362
+ np.array(
363
+ current_packet["PA_DPU_BIA_SWEEP_TIME"][:2], dtype=int
364
+ ).reshape([1, 2]),
365
+ to_datetime=True,
366
+ )[0]
367
+
368
+ logger.debug(
369
+ f"{packet_idx}: TM_DPU_EVENT_PR_BIA_SWEEP at {current_time} "
370
+ f"with sweep_pr_code {sweep_pr_code} "
371
+ f"(running status is {is_running})"
372
+ )
355
373
 
356
374
  # If event indicates a sweep start on ANT1 (START_ANT1 == 1)
357
375
  # ...
358
376
  if sweep_pr_code == 1:
359
-
360
377
  # Set current ant flag
361
378
  current_ant_flag = ANT_1_FLAG
362
379
 
@@ -375,27 +392,42 @@ class L0ToL1BiaSweep(Task):
375
392
  prev_packet_idx = packet_idx
376
393
  has_full_setting = [False, False]
377
394
  while prev_packet_idx >= 0:
378
- prev_packet = bia_sweep_packet_list[
379
- prev_packet_idx]
380
- if (prev_packet['palisade_id'] == 'TC_DPU_START_BIAS_SWEEP'
381
- and prev_packet['tc_exe_state'] == 'PASSED'):
395
+ prev_packet = bia_sweep_packet_list[prev_packet_idx]
396
+ if (
397
+ prev_packet["palisade_id"] == "TC_DPU_START_BIAS_SWEEP"
398
+ and prev_packet["tc_exe_state"] == "PASSED"
399
+ ):
382
400
  # bias sweep step duration in sec
383
- current_sweep_setting['step_sec'] = prev_packet['CP_DPU_BIA_SWEEP_DURATION']
384
-
385
- current_sweep_setting['ibias_1_def'] = raw_to_na(prev_packet['CP_DPU_BIA_SWEEP_ANT_1_CUR'],
386
- idb_version=current_idb_version,
387
- idb_source=current_idb_source).astype(float)
388
- current_sweep_setting['ibias_2_def'] = raw_to_na(prev_packet['CP_DPU_BIA_SWEEP_ANT_2_CUR'],
389
- idb_version=current_idb_version,
390
- idb_source=current_idb_source).astype(float)
391
- current_sweep_setting['ibias_3_def'] = raw_to_na(prev_packet['CP_DPU_BIA_SWEEP_ANT_3_CUR'],
392
- idb_version=current_idb_version,
393
- idb_source=current_idb_source).astype(float)
401
+ current_sweep_setting["step_sec"] = prev_packet[
402
+ "CP_DPU_BIA_SWEEP_DURATION"
403
+ ]
404
+
405
+ current_sweep_setting["ibias_1_def"] = raw_to_na(
406
+ prev_packet["CP_DPU_BIA_SWEEP_ANT_1_CUR"],
407
+ idb_version=current_idb_version,
408
+ idb_source=current_idb_source,
409
+ ).astype(float)
410
+ current_sweep_setting["ibias_2_def"] = raw_to_na(
411
+ prev_packet["CP_DPU_BIA_SWEEP_ANT_2_CUR"],
412
+ idb_version=current_idb_version,
413
+ idb_source=current_idb_source,
414
+ ).astype(float)
415
+ current_sweep_setting["ibias_3_def"] = raw_to_na(
416
+ prev_packet["CP_DPU_BIA_SWEEP_ANT_3_CUR"],
417
+ idb_version=current_idb_version,
418
+ idb_source=current_idb_source,
419
+ ).astype(float)
394
420
  has_full_setting[0] = True
395
- elif prev_packet['palisade_id'] == 'TM_DPU_BIA_HK':
396
- current_sweep_setting['bypass_1'] = prev_packet['HK_BIA_MODE_BYPASS_PROBE1']
397
- current_sweep_setting['bypass_2'] = prev_packet['HK_BIA_MODE_BYPASS_PROBE2']
398
- current_sweep_setting['bypass_3'] = prev_packet['HK_BIA_MODE_BYPASS_PROBE3']
421
+ elif prev_packet["palisade_id"] == "TM_DPU_BIA_HK":
422
+ current_sweep_setting["bypass_1"] = prev_packet[
423
+ "HK_BIA_MODE_BYPASS_PROBE1"
424
+ ]
425
+ current_sweep_setting["bypass_2"] = prev_packet[
426
+ "HK_BIA_MODE_BYPASS_PROBE2"
427
+ ]
428
+ current_sweep_setting["bypass_3"] = prev_packet[
429
+ "HK_BIA_MODE_BYPASS_PROBE3"
430
+ ]
399
431
  has_full_setting[1] = True
400
432
 
401
433
  if all(has_full_setting):
@@ -405,20 +437,24 @@ class L0ToL1BiaSweep(Task):
405
437
 
406
438
  # if no TC_DPU_START_BIAS_SWEEP found
407
439
  if prev_packet_idx < 0 and not all(has_full_setting):
408
- logger.warning('No TC_DPU_START_BIAS_SWEEP and/or TM_DPU_BIA_HK data found '
409
- 'in input L0 files for current sweep, skipping')
440
+ logger.warning(
441
+ "No TC_DPU_START_BIAS_SWEEP and/or TM_DPU_BIA_HK data found "
442
+ "in input L0 files for current sweep, skipping"
443
+ )
410
444
  is_running = False
411
445
  continue
412
446
 
413
447
  # Initialize info dict for current sweep info
414
- current_sweep_info = {'sweep_pr_code': [],
415
- 'step_time': [],
416
- 'step_ibias': [],
417
- 'ant_flag': [],
418
- 'bypass': []}
448
+ current_sweep_info = {
449
+ "sweep_pr_code": [],
450
+ "step_time": [],
451
+ "step_ibias": [],
452
+ "ant_flag": [],
453
+ "bypass": [],
454
+ }
419
455
 
420
456
  # Set sweep start time
421
- current_sweep_info['start_time'] = current_time
457
+ current_sweep_info["start_time"] = current_time
422
458
 
423
459
  # Set is_running flag to True
424
460
  is_running = True
@@ -429,32 +465,32 @@ class L0ToL1BiaSweep(Task):
429
465
  # If event indicates a sweep step on ANT1 (STEP_ANT1 == 7) ...
430
466
  elif sweep_pr_code == 7 and is_running:
431
467
  # Set current sweep dictionary with actual values
432
- current_sweep_info['step_time'].append(current_time)
433
- current_sweep_info[
434
- 'sweep_pr_code'].append(sweep_pr_code)
435
- current_sweep_info['step_ibias'].append(
468
+ current_sweep_info["step_time"].append(current_time)
469
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
470
+ current_sweep_info["step_ibias"].append(
436
471
  [
437
- raw_to_na(current_packet['PA_DPU_BIA_SWEEP_VALUE'],
438
- idb_version=current_idb_version,
439
- idb_source=current_idb_source).astype(float),
440
- current_sweep_setting['ibias_2_def'],
441
- current_sweep_setting['ibias_3_def']
472
+ raw_to_na(
473
+ current_packet["PA_DPU_BIA_SWEEP_VALUE"],
474
+ idb_version=current_idb_version,
475
+ idb_source=current_idb_source,
476
+ ).astype(float),
477
+ current_sweep_setting["ibias_2_def"],
478
+ current_sweep_setting["ibias_3_def"],
442
479
  ]
443
480
  )
444
481
 
445
482
  # Save progress code
446
- current_sweep_info[
447
- 'sweep_pr_code'].append(sweep_pr_code)
483
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
448
484
 
449
485
  # Save antenna flag
450
- current_sweep_info['ant_flag'].append(current_ant_flag)
486
+ current_sweep_info["ant_flag"].append(current_ant_flag)
451
487
 
452
488
  # Save bypass values
453
- current_sweep_info['bypass'].append(
489
+ current_sweep_info["bypass"].append(
454
490
  [
455
- current_sweep_setting['bypass_1'],
456
- current_sweep_setting['bypass_2'],
457
- current_sweep_setting['bypass_3']
491
+ current_sweep_setting["bypass_1"],
492
+ current_sweep_setting["bypass_2"],
493
+ current_sweep_setting["bypass_3"],
458
494
  ]
459
495
  )
460
496
 
@@ -464,54 +500,69 @@ class L0ToL1BiaSweep(Task):
464
500
  # If event indicates a sweep end on ANT1 (END_ANT1 == 2)
465
501
  # ...
466
502
  elif sweep_pr_code == 2 and is_running:
467
-
468
503
  # If it is not a step-by-step sweep, then compute sweep times and intensities in nA
469
504
  # from the latest sweep table values
470
505
  if current_sweep_pr_code_list[-1] == 1:
471
-
472
506
  # Get latest sweep table status
473
- current_sweep_table = L0ToAncBiaSweepTable.get_latest_sweep_table(current_time,
474
- sweep_table_list)
475
- if not current_sweep_table:
476
- logger.info(f'No valid bias sweep table found for {str(current_time)}')
507
+ current_sweep_table = (
508
+ DbToAncBiaSweepTable.get_latest_sweep_table(
509
+ current_time, sweep_table_list
510
+ )
511
+ )
512
+ if current_sweep_table.shape[0] == 0:
513
+ logger.info(
514
+ f"No valid bias sweep table found for {str(current_time)}"
515
+ )
516
+ is_running = False
517
+ continue
518
+
519
+ if current_sweep_table["TC_NAME"] == "TC_DPU_CLEAR_BIAS_SWEEP":
520
+ logger.warning(
521
+ "On-board sweep table seems to be empty "
522
+ f"(TC_DPU_CLEAR_BIAS_SWEEP executed on {current_sweep_table['TC_EXE_UTC_TIME']})"
523
+ )
477
524
  is_running = False
478
525
  continue
479
526
 
480
527
  # Build array of intensity current values versus time for
481
528
  # current sweep
482
- current_table_data = current_sweep_table[
483
- 'BIA_SWEEP_TABLE_CUR']
529
+
530
+ current_table_data = current_sweep_table["BIA_SWEEP_TABLE_CUR"]
484
531
  current_table_size = len(current_table_data)
485
532
  for j in range(current_table_size):
486
-
487
533
  # If no valid value in the table, skip current step
488
- if current_table_data[j] == 'nan':
534
+ if not current_table_data[j]:
489
535
  continue
490
536
 
491
537
  # Save absolute time of current sweep step
492
- current_sweep_info['step_time'].append(
493
- current_sweep_start + timedelta(seconds=int(j * current_sweep_setting['step_sec'])))
538
+ current_sweep_info["step_time"].append(
539
+ current_sweep_start
540
+ + timedelta(
541
+ seconds=int(j * current_sweep_setting["step_sec"])
542
+ )
543
+ )
494
544
 
495
545
  # Save Bias intensity values on the three antennas
496
- current_sweep_info['step_ibias'].append([current_table_data[j],
497
- current_sweep_setting[
498
- 'ibias_2_def'],
499
- current_sweep_setting['ibias_3_def']])
546
+ current_sweep_info["step_ibias"].append(
547
+ [
548
+ current_table_data[j],
549
+ current_sweep_setting["ibias_2_def"],
550
+ current_sweep_setting["ibias_3_def"],
551
+ ]
552
+ )
500
553
 
501
554
  # Save progress code
502
- current_sweep_info[
503
- 'sweep_pr_code'].append(sweep_pr_code)
555
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
504
556
 
505
557
  # Save antenna flag
506
- current_sweep_info['ant_flag'].append(
507
- current_ant_flag)
558
+ current_sweep_info["ant_flag"].append(current_ant_flag)
508
559
 
509
560
  # Save bypass values
510
- current_sweep_info['bypass'].append(
561
+ current_sweep_info["bypass"].append(
511
562
  [
512
- current_sweep_setting['bypass_1'],
513
- current_sweep_setting['bypass_2'],
514
- current_sweep_setting['bypass_3']
563
+ current_sweep_setting["bypass_1"],
564
+ current_sweep_setting["bypass_2"],
565
+ current_sweep_setting["bypass_3"],
515
566
  ]
516
567
  )
517
568
 
@@ -521,7 +572,6 @@ class L0ToL1BiaSweep(Task):
521
572
  # If event indicates a sweep start on ANT2 (START_ANT2 == 3)
522
573
  # ...
523
574
  elif sweep_pr_code == 3 and is_running:
524
-
525
575
  # Set current ant flag
526
576
  current_ant_flag = ANT_2_FLAG
527
577
 
@@ -534,32 +584,32 @@ class L0ToL1BiaSweep(Task):
534
584
  # If event indicates a sweep step on ANT2 (STEP_ANT2 == 8) ...
535
585
  elif sweep_pr_code == 8 and is_running:
536
586
  # Set current sweep dictionary with actual values
537
- current_sweep_info['step_time'].append(current_time)
538
- current_sweep_info[
539
- 'sweep_pr_code'].append(sweep_pr_code)
540
- current_sweep_info['step_ibias'].append(
587
+ current_sweep_info["step_time"].append(current_time)
588
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
589
+ current_sweep_info["step_ibias"].append(
541
590
  [
542
- current_sweep_setting['ibias_1_def'],
543
- raw_to_na(current_packet['PA_DPU_BIA_SWEEP_VALUE'],
544
- idb_version=current_idb_version,
545
- idb_source=current_idb_source).astype(float),
546
- current_sweep_setting['ibias_3_def']
591
+ current_sweep_setting["ibias_1_def"],
592
+ raw_to_na(
593
+ current_packet["PA_DPU_BIA_SWEEP_VALUE"],
594
+ idb_version=current_idb_version,
595
+ idb_source=current_idb_source,
596
+ ).astype(float),
597
+ current_sweep_setting["ibias_3_def"],
547
598
  ]
548
599
  )
549
600
 
550
601
  # Save progress code
551
- current_sweep_info[
552
- 'sweep_pr_code'].append(sweep_pr_code)
602
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
553
603
 
554
604
  # Save antenna flag
555
- current_sweep_info['ant_flag'].append(current_ant_flag)
605
+ current_sweep_info["ant_flag"].append(current_ant_flag)
556
606
 
557
607
  # Save bypass values
558
- current_sweep_info['bypass'].append(
608
+ current_sweep_info["bypass"].append(
559
609
  [
560
- current_sweep_setting['bypass_1'],
561
- current_sweep_setting['bypass_2'],
562
- current_sweep_setting['bypass_3']
610
+ current_sweep_setting["bypass_1"],
611
+ current_sweep_setting["bypass_2"],
612
+ current_sweep_setting["bypass_3"],
563
613
  ]
564
614
  )
565
615
 
@@ -568,40 +618,44 @@ class L0ToL1BiaSweep(Task):
568
618
 
569
619
  # If event indicates a sweep end on ANT2 (END_ANT2 == 4) ...
570
620
  elif sweep_pr_code == 4 and is_running:
571
-
572
621
  # If it is not a step-by-step sweep, then compute sweep times and intensities in nA
573
622
  # from the latest sweep table values
574
623
  if current_sweep_pr_code_list[-1] == 3:
575
624
  for j in range(current_table_size):
576
-
577
625
  # If no valid value in the table, skip current step
578
- if current_table_data[j] == 'nan':
626
+ if not current_table_data[j]:
579
627
  continue
580
628
 
581
629
  # Define absolute time of current sweep step
582
- current_sweep_info['step_time'].append(
583
- current_sweep_start + timedelta(seconds=int(j * current_sweep_setting['step_sec'])))
630
+ current_sweep_info["step_time"].append(
631
+ current_sweep_start
632
+ + timedelta(
633
+ seconds=int(j * current_sweep_setting["step_sec"])
634
+ )
635
+ )
584
636
 
585
637
  # Define Bias intensity values on the three
586
638
  # antennas
587
- current_sweep_info['step_ibias'].append([current_sweep_setting['ibias_1_def'],
588
- current_table_data[
589
- j],
590
- current_sweep_setting['ibias_3_def']])
639
+ current_sweep_info["step_ibias"].append(
640
+ [
641
+ current_sweep_setting["ibias_1_def"],
642
+ current_table_data[j],
643
+ current_sweep_setting["ibias_3_def"],
644
+ ]
645
+ )
591
646
 
592
647
  # Save progress code
593
- current_sweep_info[
594
- 'sweep_pr_code'].append(sweep_pr_code)
648
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
595
649
 
596
650
  # Define antenna flag (ANT2 = 2)
597
- current_sweep_info['ant_flag'].append(ANT_2_FLAG)
651
+ current_sweep_info["ant_flag"].append(ANT_2_FLAG)
598
652
 
599
653
  # Define bypass values
600
- current_sweep_info['bypass'].append(
654
+ current_sweep_info["bypass"].append(
601
655
  [
602
- current_sweep_setting['bypass_1'],
603
- current_sweep_setting['bypass_2'],
604
- current_sweep_setting['bypass_3']
656
+ current_sweep_setting["bypass_1"],
657
+ current_sweep_setting["bypass_2"],
658
+ current_sweep_setting["bypass_3"],
605
659
  ]
606
660
  )
607
661
 
@@ -611,7 +665,6 @@ class L0ToL1BiaSweep(Task):
611
665
  # If event indicates a sweep start on ANT3 (START_ANT3 == 5)
612
666
  # ...
613
667
  elif sweep_pr_code == 5 and is_running:
614
-
615
668
  # Set current ant flag
616
669
  current_ant_flag = ANT_3_FLAG
617
670
 
@@ -624,32 +677,32 @@ class L0ToL1BiaSweep(Task):
624
677
  # If event indicates a sweep step on ANT3 (STEP_ANT3 == 9) ...
625
678
  elif sweep_pr_code == 9 and is_running:
626
679
  # Set current sweep dictionary with actual values
627
- current_sweep_info['step_time'].append(current_time)
628
- current_sweep_info[
629
- 'sweep_pr_code'].append(sweep_pr_code)
630
- current_sweep_info['step_ibias'].append(
680
+ current_sweep_info["step_time"].append(current_time)
681
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
682
+ current_sweep_info["step_ibias"].append(
631
683
  [
632
- current_sweep_setting['ibias_1_def'],
633
- current_sweep_setting['ibias_2_def'],
634
- raw_to_na(current_packet['PA_DPU_BIA_SWEEP_VALUE'],
635
- idb_version=current_idb_version,
636
- idb_source=current_idb_source).astype(float),
684
+ current_sweep_setting["ibias_1_def"],
685
+ current_sweep_setting["ibias_2_def"],
686
+ raw_to_na(
687
+ current_packet["PA_DPU_BIA_SWEEP_VALUE"],
688
+ idb_version=current_idb_version,
689
+ idb_source=current_idb_source,
690
+ ).astype(float),
637
691
  ]
638
692
  )
639
693
 
640
694
  # Save progress code
641
- current_sweep_info[
642
- 'sweep_pr_code'].append(sweep_pr_code)
695
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
643
696
 
644
697
  # Save antenna flag
645
- current_sweep_info['ant_flag'].append(current_ant_flag)
698
+ current_sweep_info["ant_flag"].append(current_ant_flag)
646
699
 
647
700
  # Save bypass values
648
- current_sweep_info['bypass'].append(
701
+ current_sweep_info["bypass"].append(
649
702
  [
650
- current_sweep_setting['bypass_1'],
651
- current_sweep_setting['bypass_2'],
652
- current_sweep_setting['bypass_3']
703
+ current_sweep_setting["bypass_1"],
704
+ current_sweep_setting["bypass_2"],
705
+ current_sweep_setting["bypass_3"],
653
706
  ]
654
707
  )
655
708
 
@@ -658,48 +711,49 @@ class L0ToL1BiaSweep(Task):
658
711
 
659
712
  # If event indicates a sweep end on ANT3 (END_ANT3 == 6) ...
660
713
  elif sweep_pr_code == 6 and is_running:
661
-
662
714
  # If it is not a step-by-step sweep, then compute sweep times and intensities in nA
663
715
  # from the latest sweep table values
664
716
  if current_sweep_pr_code_list[-1] == 5:
665
717
  for j in range(current_table_size):
666
-
667
718
  # If no valid value in the table, skip current step
668
- if current_table_data[j] == 'nan':
719
+ if not current_table_data[j]:
669
720
  continue
670
721
 
671
722
  # Define absolute time of current sweep step
672
- current_sweep_info['step_time'].append(
673
- current_sweep_start + timedelta(seconds=int(j * current_sweep_setting['step_sec'])))
723
+ current_sweep_info["step_time"].append(
724
+ current_sweep_start
725
+ + timedelta(
726
+ seconds=int(j * current_sweep_setting["step_sec"])
727
+ )
728
+ )
674
729
 
675
730
  # Define Bias intensity values on the three
676
731
  # antennas
677
- current_sweep_info['step_ibias'].append([current_sweep_setting['ibias_1_def'],
678
- current_sweep_setting[
679
- 'ibias_2_def'],
680
- current_table_data[
681
- j],
682
- ])
732
+ current_sweep_info["step_ibias"].append(
733
+ [
734
+ current_sweep_setting["ibias_1_def"],
735
+ current_sweep_setting["ibias_2_def"],
736
+ current_table_data[j],
737
+ ]
738
+ )
683
739
 
684
740
  # Save progress code
685
- current_sweep_info[
686
- 'sweep_pr_code'].append(sweep_pr_code)
741
+ current_sweep_info["sweep_pr_code"].append(sweep_pr_code)
687
742
 
688
743
  # Define antenna flag
689
- current_sweep_info['ant_flag'].append(
690
- current_ant_flag)
744
+ current_sweep_info["ant_flag"].append(current_ant_flag)
691
745
 
692
746
  # Define bypass values
693
- current_sweep_info['bypass'].append(
747
+ current_sweep_info["bypass"].append(
694
748
  [
695
- current_sweep_setting['bypass_1'],
696
- current_sweep_setting['bypass_2'],
697
- current_sweep_setting['bypass_3']
749
+ current_sweep_setting["bypass_1"],
750
+ current_sweep_setting["bypass_2"],
751
+ current_sweep_setting["bypass_3"],
698
752
  ]
699
753
  )
700
754
 
701
755
  # Set sweep end time
702
- current_sweep_info['end_time'] = current_time
756
+ current_sweep_info["end_time"] = current_time
703
757
 
704
758
  # Disable is_running flag for current sweep
705
759
  is_running = False
@@ -711,11 +765,13 @@ class L0ToL1BiaSweep(Task):
711
765
  sweep_info_list.append(current_sweep_info)
712
766
 
713
767
  else:
714
- # Else skip on-going sweep
715
- logger.warning(f'Current sweep is not complete '
716
- f'(stopped at {current_time} '
717
- f'with sweep_pr_code {sweep_pr_code}. '
718
- f'Previous codes are {current_sweep_pr_code_list}), skip it ')
768
+ # Else skip ongoing sweep
769
+ logger.warning(
770
+ f"Current sweep is not complete "
771
+ f"(stopped at {current_time} "
772
+ f"with sweep_pr_code {sweep_pr_code}. "
773
+ f"Previous codes are {current_sweep_pr_code_list}), skip it "
774
+ )
719
775
 
720
776
  # Disable is_running flag for current sweep
721
777
  is_running = False
@@ -724,3 +780,35 @@ class L0ToL1BiaSweep(Task):
724
780
  continue
725
781
 
726
782
  return sweep_info_list
783
+
784
+ def load_sweep_tables(self):
785
+ """
786
+
787
+
788
+ :return: sweep_table_data pandas.DataFrame containing data from pipeline.event_log
789
+ """
790
+ sweep_table_data = None
791
+
792
+ # First retrieve sweep table data from pipeline.event_log table
793
+ self.filters = [self.model.label.in_(BIA_SWEEP_TABLE_PACKETS)]
794
+ logger.debug(
795
+ "Getting existing Bias sweep table data in pipeline.event_log table ..."
796
+ )
797
+ # Return existing data as a pandas.DataFrame object
798
+ sweep_table_data = query_db(
799
+ self.session,
800
+ self.model,
801
+ filters=and_(*self.filters),
802
+ tryouts=self.tryouts,
803
+ wait=self.wait,
804
+ limit=self.limit,
805
+ )
806
+ n_data = sweep_table_data.shape[0]
807
+ if n_data == 0:
808
+ logger.warning("No sweep table TC found in the database")
809
+ else:
810
+ logger.info(f"{n_data} sweep table TCs found in the database")
811
+ # Prepare table data to be used by L0ToL1BiaSweep task
812
+ sweep_table_data = DbToAncBiaSweepTable.prep_sweep_table(sweep_table_data)
813
+
814
+ return sweep_table_data