lstosa 0.10.18__py3-none-any.whl → 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/METADATA +4 -5
  2. lstosa-0.11.0.dist-info/RECORD +84 -0
  3. {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/WHEEL +1 -1
  4. {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/entry_points.txt +1 -0
  5. osa/_version.py +9 -4
  6. osa/configs/options.py +2 -0
  7. osa/configs/sequencer.cfg +21 -7
  8. osa/conftest.py +146 -6
  9. osa/high_level/significance.py +5 -3
  10. osa/high_level/tests/test_significance.py +3 -0
  11. osa/job.py +52 -26
  12. osa/nightsummary/extract.py +12 -3
  13. osa/nightsummary/tests/test_extract.py +5 -0
  14. osa/paths.py +111 -28
  15. osa/provenance/capture.py +1 -1
  16. osa/provenance/config/definition.yaml +7 -0
  17. osa/provenance/utils.py +22 -7
  18. osa/scripts/autocloser.py +0 -10
  19. osa/scripts/calibration_pipeline.py +9 -2
  20. osa/scripts/closer.py +136 -55
  21. osa/scripts/copy_datacheck.py +5 -3
  22. osa/scripts/datasequence.py +45 -71
  23. osa/scripts/gain_selection.py +14 -15
  24. osa/scripts/provprocess.py +16 -7
  25. osa/scripts/sequencer.py +49 -34
  26. osa/scripts/sequencer_catB_tailcuts.py +239 -0
  27. osa/scripts/sequencer_webmaker.py +4 -0
  28. osa/scripts/show_run_summary.py +2 -2
  29. osa/scripts/simulate_processing.py +4 -7
  30. osa/scripts/tests/test_osa_scripts.py +67 -22
  31. osa/scripts/update_source_catalog.py +45 -22
  32. osa/tests/test_jobs.py +28 -11
  33. osa/tests/test_paths.py +6 -6
  34. osa/tests/test_raw.py +4 -4
  35. osa/utils/cliopts.py +37 -32
  36. osa/utils/register.py +18 -13
  37. osa/utils/tests/test_utils.py +14 -0
  38. osa/utils/utils.py +186 -56
  39. osa/veto.py +1 -1
  40. osa/workflow/dl3.py +1 -2
  41. osa/workflow/stages.py +16 -11
  42. osa/workflow/tests/test_dl3.py +2 -1
  43. osa/workflow/tests/test_stages.py +7 -5
  44. lstosa-0.10.18.dist-info/RECORD +0 -83
  45. {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/LICENSE +0 -0
  46. {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/top_level.txt +0 -0
@@ -30,7 +30,6 @@ ALL_SCRIPTS = [
30
30
  options.date = datetime.datetime.fromisoformat("2020-01-17")
31
31
  options.tel_id = "LST1"
32
32
  options.prod_id = "v0.1.0"
33
- options.dl1_prod_id = "tailcut84"
34
33
  options.directory = "test_osa/test_files0/running_analysis/20200117/v0.1.0/"
35
34
 
36
35
 
@@ -65,7 +64,11 @@ def test_simulate_processing(
65
64
  run_summary_file,
66
65
  r0_data,
67
66
  merged_run_summary,
68
- drive_log
67
+ drive_log,
68
+ dl1b_config_files,
69
+ tailcuts_log_files,
70
+ rf_models,
71
+ tailcuts_finder_dir,
69
72
  ):
70
73
 
71
74
  for file in drs4_time_calibration_files:
@@ -80,13 +83,16 @@ def test_simulate_processing(
80
83
  assert run_summary_file.exists()
81
84
  assert merged_run_summary.exists()
82
85
  assert drive_log.exists()
86
+ assert rf_models[1].exists()
87
+ assert dl1b_config_files[0].exists()
88
+ assert tailcuts_log_files[0].exists()
83
89
 
84
90
  remove_provlog()
85
- rc = run_program("simulate_processing", "-p", "--force")
91
+ rc = run_program("simulate_processing", "-p", "--force", "-d", "2020-01-17", "LST1")
86
92
  assert rc.returncode == 0
87
93
 
88
94
  prov_dl1_path = Path("./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/log")
89
- prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/model2/log")
95
+ prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/tailcut84/nsb_tuning_0.14/log")
90
96
  prov_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.log"
91
97
  prov_file_dl2 = prov_dl2_path / "calibration_to_dl2_01807_prov.log"
92
98
  json_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.json"
@@ -101,23 +107,23 @@ def test_simulate_processing(
101
107
 
102
108
  with open(json_file_dl1) as file:
103
109
  dl1 = yaml.safe_load(file)
104
- assert len(dl1["entity"]) == 19
110
+ assert len(dl1["entity"]) == 44
105
111
  assert len(dl1["activity"]) == 5
106
112
  assert len(dl1["used"]) == 15
107
113
  assert len(dl1["wasGeneratedBy"]) == 10
108
114
 
109
115
  with open(json_file_dl2) as file:
110
116
  dl2 = yaml.safe_load(file)
111
- assert len(dl2["entity"]) == 25
112
- assert len(dl2["activity"]) == 6
113
- assert len(dl2["used"]) == 21
114
- assert len(dl2["wasGeneratedBy"]) == 12
117
+ assert len(dl2["entity"]) == 44
118
+ assert len(dl2["activity"]) == 5
119
+ assert len(dl2["used"]) == 15
120
+ assert len(dl2["wasGeneratedBy"]) == 10
115
121
 
116
- rc = run_program("simulate_processing", "-p")
122
+ rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
117
123
  assert rc.returncode == 0
118
124
 
119
125
  remove_provlog()
120
- rc = run_program("simulate_processing", "-p")
126
+ rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
121
127
  assert rc.returncode == 0
122
128
 
123
129
 
@@ -129,6 +135,10 @@ def test_simulated_sequencer(
129
135
  r0_data,
130
136
  merged_run_summary,
131
137
  gain_selection_flag_file,
138
+ dl1b_config_files,
139
+ tailcuts_log_files,
140
+ rf_models,
141
+ dl2_merged,
132
142
  ):
133
143
  assert run_summary_file.exists()
134
144
  assert run_catalog.exists()
@@ -143,7 +153,10 @@ def test_simulated_sequencer(
143
153
  for file in systematic_correction_files:
144
154
  assert file.exists()
145
155
 
146
- rc = run_program("sequencer", "-d", "2020-01-17", "-s", "-t", "LST1")
156
+ for file in dl2_merged:
157
+ assert file.exists()
158
+
159
+ rc = run_program("sequencer", "-d", "2020-01-17", "--no-gainsel", "-s", "-t", "LST1")
147
160
 
148
161
  assert rc.returncode == 0
149
162
  now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M")
@@ -152,8 +165,8 @@ def test_simulated_sequencer(
152
165
  =================================== Starting sequencer.py at {now} UTC for LST, Telescope: LST1, Date: 2020-01-17 ===================================
153
166
  Tel Seq Parent Type Run Subruns Source Action Tries JobID State CPU_time Exit DL1% MUONS% DL1AB% DATACHECK% DL2%
154
167
  LST1 1 None PEDCALIB 1809 5 None None None None None None None None None None None None
155
- LST1 2 1 DATA 1807 11 Crab None None None None None None 0 0 0 0 0
156
- LST1 3 1 DATA 1808 9 MadeUpSource None None None None None None 0 0 0 0 0
168
+ LST1 2 1 DATA 1807 11 Crab None None None None None None 0 0 0 0 100
169
+ LST1 3 1 DATA 1808 9 MadeUpSource None None None None None None 0 0 0 0 100
157
170
  """ # noqa: E501
158
171
  )
159
172
 
@@ -176,7 +189,8 @@ def test_autocloser(running_analysis_dir):
176
189
 
177
190
 
178
191
  def test_closer(
179
- r0_data,
192
+ r0g_data,
193
+ run_catalog,
180
194
  running_analysis_dir,
181
195
  test_observed_data,
182
196
  run_summary_file,
@@ -186,6 +200,9 @@ def test_closer(
186
200
  longterm_dir,
187
201
  longterm_link_latest_dir,
188
202
  daily_datacheck_dl1_files,
203
+ dl1b_config_files,
204
+ tailcuts_log_files,
205
+ rf_models,
189
206
  ):
190
207
  # First assure that the end of night flag is not set and remove it otherwise
191
208
  night_finished_flag = Path(
@@ -194,7 +211,7 @@ def test_closer(
194
211
  if night_finished_flag.exists():
195
212
  night_finished_flag.unlink()
196
213
 
197
- for r0_file in r0_data:
214
+ for r0_file in r0g_data:
198
215
  assert r0_file.exists()
199
216
  for file in drs4_time_calibration_files:
200
217
  assert file.exists()
@@ -209,6 +226,7 @@ def test_closer(
209
226
  assert longterm_link_latest_dir.exists()
210
227
  for check_file in daily_datacheck_dl1_files:
211
228
  assert check_file.exists()
229
+ assert rf_models[2].exists()
212
230
 
213
231
  run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1")
214
232
  closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed"
@@ -227,9 +245,6 @@ def test_closer(
227
245
  "./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/datacheck/"
228
246
  "datacheck_dl1_LST-1.Run01808.0011.h5"
229
247
  )
230
- assert os.path.exists(
231
- "./test_osa/test_files0/DL2/20200117/v0.1.0/model2/dl2_LST-1.Run01808.0011.h5"
232
- )
233
248
  # Assert that the link to dl1 and muons files have been created
234
249
  assert os.path.islink(
235
250
  "./test_osa/test_files0/running_analysis/20200117/v0.1.0/muons_LST-1.Run01808.0011.fits"
@@ -242,7 +257,16 @@ def test_closer(
242
257
  assert closed_seq_file.exists()
243
258
 
244
259
 
245
- def test_datasequence(running_analysis_dir):
260
+ def test_datasequence(
261
+ running_analysis_dir,
262
+ run_catalog,
263
+ run_catalog_dir,
264
+ rf_models_base_dir,
265
+ rf_models,
266
+ catB_closed_file,
267
+ dl1b_config_files,
268
+ tailcuts_log_files,
269
+ ):
246
270
  drs4_file = "drs4_pedestal.Run00001.0000.fits"
247
271
  calib_file = "calibration.Run00002.0000.hdf5"
248
272
  timecalib_file = "time_calibration.Run00002.0000.hdf5"
@@ -250,9 +274,16 @@ def test_datasequence(running_analysis_dir):
250
274
  drive_file = "DrivePosition_20200117.txt"
251
275
  runsummary_file = "RunSummary_20200117.ecsv"
252
276
  prod_id = "v0.1.0"
253
- run_number = "00003.0000"
277
+ run_number = "01807.0000"
254
278
  options.directory = running_analysis_dir
255
279
 
280
+ assert run_catalog_dir.exists()
281
+ assert run_catalog.exists()
282
+ assert rf_models_base_dir.exists()
283
+ assert rf_models[1].exists()
284
+ assert catB_closed_file.exists()
285
+ assert dl1b_config_files[0].exists()
286
+
256
287
  output = run_program(
257
288
  "datasequence",
258
289
  "--date=2020-01-17",
@@ -264,6 +295,8 @@ def test_datasequence(running_analysis_dir):
264
295
  f"--systematic-correction-file={systematic_correction_file}",
265
296
  f"--drive-file={drive_file}",
266
297
  f"--run-summary={runsummary_file}",
298
+ f"--dl1b-config={dl1b_config_files[0]}",
299
+ "--dl1-prod-id=tailcut84",
267
300
  run_number,
268
301
  "LST1",
269
302
  )
@@ -288,7 +321,15 @@ def test_calibration_pipeline(running_analysis_dir):
288
321
  assert output.returncode == 0
289
322
 
290
323
 
291
- def test_is_sequencer_successful(run_summary, running_analysis_dir):
324
+ def test_is_sequencer_successful(
325
+ run_summary,
326
+ running_analysis_dir,
327
+ dl1b_config_files,
328
+ tailcuts_log_files,
329
+ rf_models,
330
+ merged_run_summary,
331
+ ):
332
+ assert merged_run_summary.exists()
292
333
  options.directory = running_analysis_dir
293
334
  options.test = True
294
335
  seq_tuple = is_finished_check(run_summary)
@@ -300,12 +341,14 @@ def test_drs4_pedestal_cmd(base_test_dir):
300
341
  from osa.scripts.calibration_pipeline import drs4_pedestal_command
301
342
 
302
343
  cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804")
344
+ r0_dir = base_test_dir / "R0G"
303
345
  expected_command = [
304
346
  cfg.get("lstchain", "drs4_baseline"),
305
347
  "-r",
306
348
  "01804",
307
349
  "-b",
308
350
  base_test_dir,
351
+ f"--r0-dir={r0_dir}",
309
352
  "--no-progress",
310
353
  ]
311
354
  assert cmd == expected_command
@@ -315,6 +358,7 @@ def test_calibration_file_cmd(base_test_dir):
315
358
  from osa.scripts.calibration_pipeline import calibration_file_command
316
359
 
317
360
  cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809")
361
+ r0_dir = base_test_dir / "R0G"
318
362
  expected_command = [
319
363
  cfg.get("lstchain", "charge_calibration"),
320
364
  "-p",
@@ -323,6 +367,7 @@ def test_calibration_file_cmd(base_test_dir):
323
367
  "01809",
324
368
  "-b",
325
369
  base_test_dir,
370
+ f"--r0-dir={r0_dir}",
326
371
  ]
327
372
  assert cmd == expected_command
328
373
 
@@ -1,30 +1,49 @@
1
1
  import logging
2
2
  import subprocess as sp
3
- from datetime import datetime
4
3
  from pathlib import Path
5
4
  from textwrap import dedent
6
-
7
- import click
5
+ import argparse
8
6
  import pandas as pd
7
+
9
8
  from astropy import units as u
10
9
  from astropy.table import Table, join, unique, vstack
11
10
  from astropy.time import Time
12
11
  from lstchain.io.io import dl1_params_lstcam_key
13
12
  from lstchain.reco.utils import add_delta_t_key, get_effective_time
14
13
 
15
- from osa.paths import get_major_version
16
- from osa.utils.utils import get_lstchain_version
14
+ from osa.configs.config import cfg
15
+ from osa.utils.cliopts import valid_date
16
+ from osa.paths import DEFAULT_CFG, get_major_version, get_dl1_prod_id
17
+ from osa.utils.utils import get_lstchain_version, date_to_dir, date_to_iso
18
+
17
19
 
18
20
  pd.set_option("display.float_format", "{:.1f}".format)
19
21
 
20
22
  logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(message)s")
21
23
  log = logging.getLogger(__name__)
22
24
 
23
-
24
- BASE_DL1 = Path("/fefs/aswg/data/real/DL1")
25
- BASE_MONITORING = Path("/fefs/aswg/data/real/monitoring")
26
- CATALOG_DIR = Path("/fefs/aswg/data/real/OSA/Catalog")
27
-
25
+ parser = argparse.ArgumentParser()
26
+ parser.add_argument(
27
+ "-c",
28
+ "--config",
29
+ action="store",
30
+ type=Path,
31
+ default=DEFAULT_CFG,
32
+ help="Configuration file",
33
+ )
34
+ parser.add_argument(
35
+ "-d",
36
+ "--date",
37
+ default=None,
38
+ type=valid_date,
39
+ help="Night to apply the gain selection in YYYY-MM-DD format",
40
+ )
41
+ parser.add_argument(
42
+ "-v",
43
+ "--version",
44
+ type=str,
45
+ default=get_lstchain_version()
46
+ )
28
47
 
29
48
  def add_table_to_html(html_table):
30
49
  return dedent(
@@ -125,7 +144,10 @@ def add_start_and_elapsed(table: Table, datedir: str, version: str) -> None:
125
144
 
126
145
  for run in table["run_id"]:
127
146
  major_version = get_major_version(version)
128
- file = BASE_DL1 / datedir / major_version / f"tailcut84/dl1_LST-1.Run{run:05d}.h5"
147
+ dl1b_config_file = Path(cfg.get("LST1", "TAILCUTS_FINDER_DIR")) / f"dl1ab_Run{run:05d}.json"
148
+ dl1_prod_id = get_dl1_prod_id(dl1b_config_file)
149
+ dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
150
+ file = dl1_dir / datedir / major_version / dl1_prod_id / f"dl1_LST-1.Run{run:05d}.h5"
129
151
  df = pd.read_hdf(file, key=dl1_params_lstcam_key)
130
152
 
131
153
  # Timestamp of the first event
@@ -147,32 +169,33 @@ def copy_to_webserver(html_file, csv_file):
147
169
  sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
148
170
 
149
171
 
150
- @click.command()
151
- @click.argument("date", type=click.DateTime(formats=["%Y-%m-%d"]))
152
- @click.option("-v", "--version", type=str, default=get_lstchain_version())
153
- def main(date: datetime = None, version: str = get_lstchain_version()):
172
+ def main():
154
173
  """Update source catalog with new run entries from a given date in format YYYY-MM-DD.
155
174
 
156
175
  Notes
157
176
  -----
158
177
  It needs to be run as lstanalyzer user.
159
178
  """
160
- catalog_path = CATALOG_DIR / "LST_source_catalog.ecsv"
179
+ args = parser.parse_args()
180
+
181
+ catalog_path = Path(cfg.get("LST1", "SOURCE_CATALOG")) / "LST_source_catalog.ecsv"
161
182
  catalog_table = Table.read(catalog_path)
162
183
 
163
184
  # Open table for given date and append its content to the table with entire catalog
164
- datedir = date.strftime("%Y%m%d")
165
- today_catalog = Table.read(BASE_MONITORING / f"RunCatalog/RunCatalog_{datedir}.ecsv")
166
- today_runsummary = Table.read(BASE_MONITORING / f"RunSummary/RunSummary_{datedir}.ecsv")
185
+ datedir = date_to_dir(args.date)
186
+ run_catalog_dir = Path(cfg.get("LST1", "RUN_CATALOG"))
187
+ today_catalog = Table.read(run_catalog_dir / f"RunCatalog_{datedir}.ecsv")
188
+ run_summary_dir = Path(cfg.get("LST1", "RUN_SUMMARY_DIR"))
189
+ today_runsummary = Table.read(run_summary_dir / f"RunSummary_{datedir}.ecsv")
167
190
  # Keep only astronomical data runs
168
191
  today_runsummary = today_runsummary[today_runsummary["run_type"] == "DATA"]
169
192
  todays_info = join(today_runsummary, today_catalog)
170
- todays_info.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
193
+ todays_info.add_column(date_to_iso(args.date), name="date_dir")
171
194
  todays_info.keep_columns(["run_id", "source_name", "date_dir"])
172
195
 
173
196
  # Add start of run in iso format and elapsed time for each run
174
197
  log.info("Getting run start and elapsed time")
175
- add_start_and_elapsed(todays_info, datedir, version)
198
+ add_start_and_elapsed(todays_info, datedir, args.version)
176
199
 
177
200
  # Change column names
178
201
  todays_info.rename_column("run_id", "Run ID")
@@ -198,7 +221,7 @@ def main(date: datetime = None, version: str = get_lstchain_version()):
198
221
  html_content = add_query_table_to_html(html_table)
199
222
 
200
223
  # Save the HTML and ECSV files and copy them to the LST-1 webserver
201
- html_file = CATALOG_DIR / "LST_source_catalog.html"
224
+ html_file = Path(cfg.get("LST1", "SOURCE_CATALOG")) / "LST_source_catalog.html"
202
225
  html_file.write_text(html_content)
203
226
  table_unique.write(catalog_path, delimiter=",", overwrite=True)
204
227
 
osa/tests/test_jobs.py CHANGED
@@ -9,18 +9,21 @@ from osa.configs.config import cfg, DEFAULT_CFG
9
9
 
10
10
  extra_files = Path(os.getenv("OSA_TEST_DATA", "extra"))
11
11
  datasequence_history_file = extra_files / "history_files/sequence_LST1_04185.0010.history"
12
+ datasequence_history_file2 = extra_files / "history_files/sequence_LST1_04185.0001.history"
12
13
  calibration_history_file = extra_files / "history_files/sequence_LST1_04183.history"
13
14
  options.date = "2020-01-17"
14
15
  options.tel_id = "LST1"
15
16
  options.prod_id = "v0.1.0"
16
17
 
17
18
 
18
- def test_historylevel():
19
+ def test_historylevel(
20
+ run_catalog,
21
+ dl1b_config_files,
22
+ tailcuts_log_files,
23
+ rf_models,
24
+ ):
19
25
  from osa.job import historylevel
20
26
 
21
- options.dl1_prod_id = "tailcut84"
22
- options.dl2_prod_id = "model1"
23
-
24
27
  level, rc = historylevel(datasequence_history_file, "DATA")
25
28
  assert level == 0
26
29
  assert rc == 0
@@ -29,11 +32,8 @@ def test_historylevel():
29
32
  assert level == 0
30
33
  assert rc == 0
31
34
 
32
- options.dl1_prod_id = "tailcut84"
33
- options.dl2_prod_id = "model2"
34
-
35
- level, rc = historylevel(datasequence_history_file, "DATA")
36
- assert level == 1
35
+ level, rc = historylevel(datasequence_history_file2, "DATA")
36
+ assert level == 2
37
37
  assert rc == 0
38
38
 
39
39
 
@@ -41,6 +41,7 @@ def test_preparejobs(running_analysis_dir, sequence_list):
41
41
  from osa.job import prepare_jobs
42
42
 
43
43
  options.simulate = False
44
+ options.test = True
44
45
  options.directory = running_analysis_dir
45
46
  prepare_jobs(sequence_list)
46
47
  expected_calib_script = os.path.join(running_analysis_dir, "sequence_LST1_01809.py")
@@ -94,6 +95,7 @@ def test_job_header_template(sequence_list, running_analysis_dir):
94
95
  from osa.job import job_header_template
95
96
 
96
97
  # Extract the first sequence
98
+ options.test = False
97
99
  first_sequence = sequence_list[0]
98
100
  header = job_header_template(first_sequence)
99
101
  output_string1 = dedent(
@@ -138,13 +140,17 @@ def test_create_job_template_scheduler(
138
140
  calibration_file,
139
141
  run_summary_file,
140
142
  pedestal_ids_file,
143
+ dl1b_config_files,
144
+ rf_models,
141
145
  ):
142
146
  from osa.job import data_sequence_job_template
143
147
 
144
148
  assert pedestal_ids_file.exists()
149
+ assert rf_models[1].exists()
145
150
 
146
151
  options.test = False
147
152
  options.simulate = False
153
+
148
154
  content1 = data_sequence_job_template(sequence_list[1])
149
155
  expected_content1 = dedent(
150
156
  f"""\
@@ -184,6 +190,8 @@ def test_create_job_template_scheduler(
184
190
  '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
185
191
  '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
186
192
  '--run-summary={run_summary_file}',
193
+ '--dl1b-config={dl1b_config_files[0]}',
194
+ '--dl1-prod-id=tailcut84',
187
195
  f'01807.{{subruns:04d}}',
188
196
  'LST1'
189
197
  ])
@@ -230,6 +238,8 @@ def test_create_job_template_scheduler(
230
238
  '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
231
239
  '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
232
240
  '--run-summary={run_summary_file}',
241
+ '--dl1b-config={dl1b_config_files[1]}',
242
+ '--dl1-prod-id=tailcut84',
233
243
  f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
234
244
  f'01808.{{subruns:04d}}',
235
245
  'LST1'
@@ -252,6 +262,8 @@ def test_create_job_template_local(
252
262
  run_summary_file,
253
263
  pedestal_ids_file,
254
264
  r0_data,
265
+ dl1b_config_files,
266
+ rf_models,
255
267
  ):
256
268
  """Check the job file in local mode (assuming no scheduler)."""
257
269
  from osa.job import data_sequence_job_template
@@ -266,6 +278,7 @@ def test_create_job_template_local(
266
278
  assert file.exists()
267
279
 
268
280
  assert pedestal_ids_file.exists()
281
+ assert rf_models[0].exists()
269
282
 
270
283
  options.test = True
271
284
  options.simulate = False
@@ -296,6 +309,8 @@ def test_create_job_template_local(
296
309
  '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
297
310
  '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
298
311
  '--run-summary={run_summary_file}',
312
+ '--dl1b-config={dl1b_config_files[0]}',
313
+ '--dl1-prod-id=tailcut84',
299
314
  f'01807.{{subruns:04d}}',
300
315
  'LST1'
301
316
  ])
@@ -329,6 +344,8 @@ def test_create_job_template_local(
329
344
  '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
330
345
  '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
331
346
  '--run-summary={run_summary_file}',
347
+ '--dl1b-config={dl1b_config_files[1]}',
348
+ '--dl1-prod-id=tailcut84',
332
349
  f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
333
350
  f'01808.{{subruns:04d}}',
334
351
  'LST1'
@@ -337,11 +354,11 @@ def test_create_job_template_local(
337
354
  sys.exit(proc.returncode)"""
338
355
  )
339
356
 
340
- options.simulate = True
341
-
342
357
  assert content1 == expected_content1
343
358
  assert content2 == expected_content2
344
359
 
360
+ options.simulate = True
361
+
345
362
 
346
363
  def test_create_job_scheduler_calibration(sequence_list):
347
364
  """Check the pilot job file for the calibration pipeline."""
osa/tests/test_paths.py CHANGED
@@ -52,8 +52,8 @@ def test_destination_dir():
52
52
  from osa.paths import destination_dir
53
53
 
54
54
  datedir = date_to_dir(options.date)
55
- options.dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
56
- options.dl2_prod_id = cfg.get("LST1", "DL2_PROD_ID")
55
+ dl1_prod_id = "tailcut84"
56
+ dl2_prod_id = "tailcut84/nsb_tuning_0.14"
57
57
  options.prod_id = cfg.get("LST1", "PROD_ID")
58
58
  base_directory = cfg.get("LST1", "BASE")
59
59
  base_path = Path(base_directory)
@@ -67,14 +67,14 @@ def test_destination_dir():
67
67
  }
68
68
 
69
69
  for concept, dst_dir in data_types.items():
70
- directory = destination_dir(concept, create_dir=False)
70
+ directory = destination_dir(concept, create_dir=False, dl1_prod_id=dl1_prod_id, dl2_prod_id=dl2_prod_id)
71
71
  if concept == "DL1AB":
72
72
  expected_directory = (
73
- base_path / dst_dir / datedir / options.prod_id / options.dl1_prod_id
73
+ base_path / dst_dir / datedir / options.prod_id / dl1_prod_id
74
74
  )
75
75
  elif concept == "DATACHECK":
76
76
  expected_directory = (
77
- base_path / dst_dir / datedir / options.prod_id / options.dl1_prod_id / "datacheck"
77
+ base_path / dst_dir / datedir / options.prod_id / dl1_prod_id / "datacheck"
78
78
  )
79
79
  elif concept == "MUON":
80
80
  expected_directory = (
@@ -86,7 +86,7 @@ def test_destination_dir():
86
86
  )
87
87
  elif concept == "DL2":
88
88
  expected_directory = (
89
- base_path / dst_dir / datedir / options.prod_id / options.dl2_prod_id
89
+ base_path / dst_dir / datedir / options.prod_id / dl2_prod_id
90
90
  )
91
91
 
92
92
  assert directory == expected_directory
osa/tests/test_raw.py CHANGED
@@ -17,7 +17,7 @@ def test_get_raw_dir():
17
17
  assert get_raw_dir(options.date) == r0_dir
18
18
 
19
19
 
20
- def test_get_check_raw_dir(r0_dir):
20
+ def test_get_check_raw_dir(r0g_dir):
21
21
  options.date = datetime.fromisoformat("2020-01-18")
22
22
  from osa.raw import get_check_raw_dir
23
23
 
@@ -26,13 +26,13 @@ def test_get_check_raw_dir(r0_dir):
26
26
 
27
27
  options.date = datetime.fromisoformat("2020-01-17")
28
28
  raw_dir = get_check_raw_dir(options.date)
29
- assert raw_dir.resolve() == r0_dir
29
+ assert raw_dir.resolve() == r0g_dir
30
30
 
31
31
 
32
- def test_is_raw_data_available(r0_data):
32
+ def test_is_raw_data_available(r0g_data):
33
33
  from osa.raw import is_raw_data_available
34
34
 
35
- for file in r0_data:
35
+ for file in r0g_data:
36
36
  assert file.exists()
37
37
 
38
38
  options.date = datetime.fromisoformat("2020-01-17")