lstosa 0.10.18__py3-none-any.whl → 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/METADATA +4 -5
- lstosa-0.11.0.dist-info/RECORD +84 -0
- {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/WHEEL +1 -1
- {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/entry_points.txt +1 -0
- osa/_version.py +9 -4
- osa/configs/options.py +2 -0
- osa/configs/sequencer.cfg +21 -7
- osa/conftest.py +146 -6
- osa/high_level/significance.py +5 -3
- osa/high_level/tests/test_significance.py +3 -0
- osa/job.py +52 -26
- osa/nightsummary/extract.py +12 -3
- osa/nightsummary/tests/test_extract.py +5 -0
- osa/paths.py +111 -28
- osa/provenance/capture.py +1 -1
- osa/provenance/config/definition.yaml +7 -0
- osa/provenance/utils.py +22 -7
- osa/scripts/autocloser.py +0 -10
- osa/scripts/calibration_pipeline.py +9 -2
- osa/scripts/closer.py +136 -55
- osa/scripts/copy_datacheck.py +5 -3
- osa/scripts/datasequence.py +45 -71
- osa/scripts/gain_selection.py +14 -15
- osa/scripts/provprocess.py +16 -7
- osa/scripts/sequencer.py +49 -34
- osa/scripts/sequencer_catB_tailcuts.py +239 -0
- osa/scripts/sequencer_webmaker.py +4 -0
- osa/scripts/show_run_summary.py +2 -2
- osa/scripts/simulate_processing.py +4 -7
- osa/scripts/tests/test_osa_scripts.py +67 -22
- osa/scripts/update_source_catalog.py +45 -22
- osa/tests/test_jobs.py +28 -11
- osa/tests/test_paths.py +6 -6
- osa/tests/test_raw.py +4 -4
- osa/utils/cliopts.py +37 -32
- osa/utils/register.py +18 -13
- osa/utils/tests/test_utils.py +14 -0
- osa/utils/utils.py +186 -56
- osa/veto.py +1 -1
- osa/workflow/dl3.py +1 -2
- osa/workflow/stages.py +16 -11
- osa/workflow/tests/test_dl3.py +2 -1
- osa/workflow/tests/test_stages.py +7 -5
- lstosa-0.10.18.dist-info/RECORD +0 -83
- {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/LICENSE +0 -0
- {lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/top_level.txt +0 -0
|
@@ -30,7 +30,6 @@ ALL_SCRIPTS = [
|
|
|
30
30
|
options.date = datetime.datetime.fromisoformat("2020-01-17")
|
|
31
31
|
options.tel_id = "LST1"
|
|
32
32
|
options.prod_id = "v0.1.0"
|
|
33
|
-
options.dl1_prod_id = "tailcut84"
|
|
34
33
|
options.directory = "test_osa/test_files0/running_analysis/20200117/v0.1.0/"
|
|
35
34
|
|
|
36
35
|
|
|
@@ -65,7 +64,11 @@ def test_simulate_processing(
|
|
|
65
64
|
run_summary_file,
|
|
66
65
|
r0_data,
|
|
67
66
|
merged_run_summary,
|
|
68
|
-
drive_log
|
|
67
|
+
drive_log,
|
|
68
|
+
dl1b_config_files,
|
|
69
|
+
tailcuts_log_files,
|
|
70
|
+
rf_models,
|
|
71
|
+
tailcuts_finder_dir,
|
|
69
72
|
):
|
|
70
73
|
|
|
71
74
|
for file in drs4_time_calibration_files:
|
|
@@ -80,13 +83,16 @@ def test_simulate_processing(
|
|
|
80
83
|
assert run_summary_file.exists()
|
|
81
84
|
assert merged_run_summary.exists()
|
|
82
85
|
assert drive_log.exists()
|
|
86
|
+
assert rf_models[1].exists()
|
|
87
|
+
assert dl1b_config_files[0].exists()
|
|
88
|
+
assert tailcuts_log_files[0].exists()
|
|
83
89
|
|
|
84
90
|
remove_provlog()
|
|
85
|
-
rc = run_program("simulate_processing", "-p", "--force")
|
|
91
|
+
rc = run_program("simulate_processing", "-p", "--force", "-d", "2020-01-17", "LST1")
|
|
86
92
|
assert rc.returncode == 0
|
|
87
93
|
|
|
88
94
|
prov_dl1_path = Path("./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/log")
|
|
89
|
-
prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/
|
|
95
|
+
prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/tailcut84/nsb_tuning_0.14/log")
|
|
90
96
|
prov_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.log"
|
|
91
97
|
prov_file_dl2 = prov_dl2_path / "calibration_to_dl2_01807_prov.log"
|
|
92
98
|
json_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.json"
|
|
@@ -101,23 +107,23 @@ def test_simulate_processing(
|
|
|
101
107
|
|
|
102
108
|
with open(json_file_dl1) as file:
|
|
103
109
|
dl1 = yaml.safe_load(file)
|
|
104
|
-
assert len(dl1["entity"]) ==
|
|
110
|
+
assert len(dl1["entity"]) == 44
|
|
105
111
|
assert len(dl1["activity"]) == 5
|
|
106
112
|
assert len(dl1["used"]) == 15
|
|
107
113
|
assert len(dl1["wasGeneratedBy"]) == 10
|
|
108
114
|
|
|
109
115
|
with open(json_file_dl2) as file:
|
|
110
116
|
dl2 = yaml.safe_load(file)
|
|
111
|
-
assert len(dl2["entity"]) ==
|
|
112
|
-
assert len(dl2["activity"]) ==
|
|
113
|
-
assert len(dl2["used"]) ==
|
|
114
|
-
assert len(dl2["wasGeneratedBy"]) ==
|
|
117
|
+
assert len(dl2["entity"]) == 44
|
|
118
|
+
assert len(dl2["activity"]) == 5
|
|
119
|
+
assert len(dl2["used"]) == 15
|
|
120
|
+
assert len(dl2["wasGeneratedBy"]) == 10
|
|
115
121
|
|
|
116
|
-
rc = run_program("simulate_processing", "-p")
|
|
122
|
+
rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
|
|
117
123
|
assert rc.returncode == 0
|
|
118
124
|
|
|
119
125
|
remove_provlog()
|
|
120
|
-
rc = run_program("simulate_processing", "-p")
|
|
126
|
+
rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
|
|
121
127
|
assert rc.returncode == 0
|
|
122
128
|
|
|
123
129
|
|
|
@@ -129,6 +135,10 @@ def test_simulated_sequencer(
|
|
|
129
135
|
r0_data,
|
|
130
136
|
merged_run_summary,
|
|
131
137
|
gain_selection_flag_file,
|
|
138
|
+
dl1b_config_files,
|
|
139
|
+
tailcuts_log_files,
|
|
140
|
+
rf_models,
|
|
141
|
+
dl2_merged,
|
|
132
142
|
):
|
|
133
143
|
assert run_summary_file.exists()
|
|
134
144
|
assert run_catalog.exists()
|
|
@@ -143,7 +153,10 @@ def test_simulated_sequencer(
|
|
|
143
153
|
for file in systematic_correction_files:
|
|
144
154
|
assert file.exists()
|
|
145
155
|
|
|
146
|
-
|
|
156
|
+
for file in dl2_merged:
|
|
157
|
+
assert file.exists()
|
|
158
|
+
|
|
159
|
+
rc = run_program("sequencer", "-d", "2020-01-17", "--no-gainsel", "-s", "-t", "LST1")
|
|
147
160
|
|
|
148
161
|
assert rc.returncode == 0
|
|
149
162
|
now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M")
|
|
@@ -152,8 +165,8 @@ def test_simulated_sequencer(
|
|
|
152
165
|
=================================== Starting sequencer.py at {now} UTC for LST, Telescope: LST1, Date: 2020-01-17 ===================================
|
|
153
166
|
Tel Seq Parent Type Run Subruns Source Action Tries JobID State CPU_time Exit DL1% MUONS% DL1AB% DATACHECK% DL2%
|
|
154
167
|
LST1 1 None PEDCALIB 1809 5 None None None None None None None None None None None None
|
|
155
|
-
LST1 2 1 DATA 1807 11 Crab None None None None None None 0 0 0 0
|
|
156
|
-
LST1 3 1 DATA 1808 9 MadeUpSource None None None None None None 0 0 0 0
|
|
168
|
+
LST1 2 1 DATA 1807 11 Crab None None None None None None 0 0 0 0 100
|
|
169
|
+
LST1 3 1 DATA 1808 9 MadeUpSource None None None None None None 0 0 0 0 100
|
|
157
170
|
""" # noqa: E501
|
|
158
171
|
)
|
|
159
172
|
|
|
@@ -176,7 +189,8 @@ def test_autocloser(running_analysis_dir):
|
|
|
176
189
|
|
|
177
190
|
|
|
178
191
|
def test_closer(
|
|
179
|
-
|
|
192
|
+
r0g_data,
|
|
193
|
+
run_catalog,
|
|
180
194
|
running_analysis_dir,
|
|
181
195
|
test_observed_data,
|
|
182
196
|
run_summary_file,
|
|
@@ -186,6 +200,9 @@ def test_closer(
|
|
|
186
200
|
longterm_dir,
|
|
187
201
|
longterm_link_latest_dir,
|
|
188
202
|
daily_datacheck_dl1_files,
|
|
203
|
+
dl1b_config_files,
|
|
204
|
+
tailcuts_log_files,
|
|
205
|
+
rf_models,
|
|
189
206
|
):
|
|
190
207
|
# First assure that the end of night flag is not set and remove it otherwise
|
|
191
208
|
night_finished_flag = Path(
|
|
@@ -194,7 +211,7 @@ def test_closer(
|
|
|
194
211
|
if night_finished_flag.exists():
|
|
195
212
|
night_finished_flag.unlink()
|
|
196
213
|
|
|
197
|
-
for r0_file in
|
|
214
|
+
for r0_file in r0g_data:
|
|
198
215
|
assert r0_file.exists()
|
|
199
216
|
for file in drs4_time_calibration_files:
|
|
200
217
|
assert file.exists()
|
|
@@ -209,6 +226,7 @@ def test_closer(
|
|
|
209
226
|
assert longterm_link_latest_dir.exists()
|
|
210
227
|
for check_file in daily_datacheck_dl1_files:
|
|
211
228
|
assert check_file.exists()
|
|
229
|
+
assert rf_models[2].exists()
|
|
212
230
|
|
|
213
231
|
run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1")
|
|
214
232
|
closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed"
|
|
@@ -227,9 +245,6 @@ def test_closer(
|
|
|
227
245
|
"./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/datacheck/"
|
|
228
246
|
"datacheck_dl1_LST-1.Run01808.0011.h5"
|
|
229
247
|
)
|
|
230
|
-
assert os.path.exists(
|
|
231
|
-
"./test_osa/test_files0/DL2/20200117/v0.1.0/model2/dl2_LST-1.Run01808.0011.h5"
|
|
232
|
-
)
|
|
233
248
|
# Assert that the link to dl1 and muons files have been created
|
|
234
249
|
assert os.path.islink(
|
|
235
250
|
"./test_osa/test_files0/running_analysis/20200117/v0.1.0/muons_LST-1.Run01808.0011.fits"
|
|
@@ -242,7 +257,16 @@ def test_closer(
|
|
|
242
257
|
assert closed_seq_file.exists()
|
|
243
258
|
|
|
244
259
|
|
|
245
|
-
def test_datasequence(
|
|
260
|
+
def test_datasequence(
|
|
261
|
+
running_analysis_dir,
|
|
262
|
+
run_catalog,
|
|
263
|
+
run_catalog_dir,
|
|
264
|
+
rf_models_base_dir,
|
|
265
|
+
rf_models,
|
|
266
|
+
catB_closed_file,
|
|
267
|
+
dl1b_config_files,
|
|
268
|
+
tailcuts_log_files,
|
|
269
|
+
):
|
|
246
270
|
drs4_file = "drs4_pedestal.Run00001.0000.fits"
|
|
247
271
|
calib_file = "calibration.Run00002.0000.hdf5"
|
|
248
272
|
timecalib_file = "time_calibration.Run00002.0000.hdf5"
|
|
@@ -250,9 +274,16 @@ def test_datasequence(running_analysis_dir):
|
|
|
250
274
|
drive_file = "DrivePosition_20200117.txt"
|
|
251
275
|
runsummary_file = "RunSummary_20200117.ecsv"
|
|
252
276
|
prod_id = "v0.1.0"
|
|
253
|
-
run_number = "
|
|
277
|
+
run_number = "01807.0000"
|
|
254
278
|
options.directory = running_analysis_dir
|
|
255
279
|
|
|
280
|
+
assert run_catalog_dir.exists()
|
|
281
|
+
assert run_catalog.exists()
|
|
282
|
+
assert rf_models_base_dir.exists()
|
|
283
|
+
assert rf_models[1].exists()
|
|
284
|
+
assert catB_closed_file.exists()
|
|
285
|
+
assert dl1b_config_files[0].exists()
|
|
286
|
+
|
|
256
287
|
output = run_program(
|
|
257
288
|
"datasequence",
|
|
258
289
|
"--date=2020-01-17",
|
|
@@ -264,6 +295,8 @@ def test_datasequence(running_analysis_dir):
|
|
|
264
295
|
f"--systematic-correction-file={systematic_correction_file}",
|
|
265
296
|
f"--drive-file={drive_file}",
|
|
266
297
|
f"--run-summary={runsummary_file}",
|
|
298
|
+
f"--dl1b-config={dl1b_config_files[0]}",
|
|
299
|
+
"--dl1-prod-id=tailcut84",
|
|
267
300
|
run_number,
|
|
268
301
|
"LST1",
|
|
269
302
|
)
|
|
@@ -288,7 +321,15 @@ def test_calibration_pipeline(running_analysis_dir):
|
|
|
288
321
|
assert output.returncode == 0
|
|
289
322
|
|
|
290
323
|
|
|
291
|
-
def test_is_sequencer_successful(
|
|
324
|
+
def test_is_sequencer_successful(
|
|
325
|
+
run_summary,
|
|
326
|
+
running_analysis_dir,
|
|
327
|
+
dl1b_config_files,
|
|
328
|
+
tailcuts_log_files,
|
|
329
|
+
rf_models,
|
|
330
|
+
merged_run_summary,
|
|
331
|
+
):
|
|
332
|
+
assert merged_run_summary.exists()
|
|
292
333
|
options.directory = running_analysis_dir
|
|
293
334
|
options.test = True
|
|
294
335
|
seq_tuple = is_finished_check(run_summary)
|
|
@@ -300,12 +341,14 @@ def test_drs4_pedestal_cmd(base_test_dir):
|
|
|
300
341
|
from osa.scripts.calibration_pipeline import drs4_pedestal_command
|
|
301
342
|
|
|
302
343
|
cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804")
|
|
344
|
+
r0_dir = base_test_dir / "R0G"
|
|
303
345
|
expected_command = [
|
|
304
346
|
cfg.get("lstchain", "drs4_baseline"),
|
|
305
347
|
"-r",
|
|
306
348
|
"01804",
|
|
307
349
|
"-b",
|
|
308
350
|
base_test_dir,
|
|
351
|
+
f"--r0-dir={r0_dir}",
|
|
309
352
|
"--no-progress",
|
|
310
353
|
]
|
|
311
354
|
assert cmd == expected_command
|
|
@@ -315,6 +358,7 @@ def test_calibration_file_cmd(base_test_dir):
|
|
|
315
358
|
from osa.scripts.calibration_pipeline import calibration_file_command
|
|
316
359
|
|
|
317
360
|
cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809")
|
|
361
|
+
r0_dir = base_test_dir / "R0G"
|
|
318
362
|
expected_command = [
|
|
319
363
|
cfg.get("lstchain", "charge_calibration"),
|
|
320
364
|
"-p",
|
|
@@ -323,6 +367,7 @@ def test_calibration_file_cmd(base_test_dir):
|
|
|
323
367
|
"01809",
|
|
324
368
|
"-b",
|
|
325
369
|
base_test_dir,
|
|
370
|
+
f"--r0-dir={r0_dir}",
|
|
326
371
|
]
|
|
327
372
|
assert cmd == expected_command
|
|
328
373
|
|
|
@@ -1,30 +1,49 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import subprocess as sp
|
|
3
|
-
from datetime import datetime
|
|
4
3
|
from pathlib import Path
|
|
5
4
|
from textwrap import dedent
|
|
6
|
-
|
|
7
|
-
import click
|
|
5
|
+
import argparse
|
|
8
6
|
import pandas as pd
|
|
7
|
+
|
|
9
8
|
from astropy import units as u
|
|
10
9
|
from astropy.table import Table, join, unique, vstack
|
|
11
10
|
from astropy.time import Time
|
|
12
11
|
from lstchain.io.io import dl1_params_lstcam_key
|
|
13
12
|
from lstchain.reco.utils import add_delta_t_key, get_effective_time
|
|
14
13
|
|
|
15
|
-
from osa.
|
|
16
|
-
from osa.utils.
|
|
14
|
+
from osa.configs.config import cfg
|
|
15
|
+
from osa.utils.cliopts import valid_date
|
|
16
|
+
from osa.paths import DEFAULT_CFG, get_major_version, get_dl1_prod_id
|
|
17
|
+
from osa.utils.utils import get_lstchain_version, date_to_dir, date_to_iso
|
|
18
|
+
|
|
17
19
|
|
|
18
20
|
pd.set_option("display.float_format", "{:.1f}".format)
|
|
19
21
|
|
|
20
22
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(message)s")
|
|
21
23
|
log = logging.getLogger(__name__)
|
|
22
24
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
parser = argparse.ArgumentParser()
|
|
26
|
+
parser.add_argument(
|
|
27
|
+
"-c",
|
|
28
|
+
"--config",
|
|
29
|
+
action="store",
|
|
30
|
+
type=Path,
|
|
31
|
+
default=DEFAULT_CFG,
|
|
32
|
+
help="Configuration file",
|
|
33
|
+
)
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"-d",
|
|
36
|
+
"--date",
|
|
37
|
+
default=None,
|
|
38
|
+
type=valid_date,
|
|
39
|
+
help="Night to apply the gain selection in YYYY-MM-DD format",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"-v",
|
|
43
|
+
"--version",
|
|
44
|
+
type=str,
|
|
45
|
+
default=get_lstchain_version()
|
|
46
|
+
)
|
|
28
47
|
|
|
29
48
|
def add_table_to_html(html_table):
|
|
30
49
|
return dedent(
|
|
@@ -125,7 +144,10 @@ def add_start_and_elapsed(table: Table, datedir: str, version: str) -> None:
|
|
|
125
144
|
|
|
126
145
|
for run in table["run_id"]:
|
|
127
146
|
major_version = get_major_version(version)
|
|
128
|
-
|
|
147
|
+
dl1b_config_file = Path(cfg.get("LST1", "TAILCUTS_FINDER_DIR")) / f"dl1ab_Run{run:05d}.json"
|
|
148
|
+
dl1_prod_id = get_dl1_prod_id(dl1b_config_file)
|
|
149
|
+
dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
|
|
150
|
+
file = dl1_dir / datedir / major_version / dl1_prod_id / f"dl1_LST-1.Run{run:05d}.h5"
|
|
129
151
|
df = pd.read_hdf(file, key=dl1_params_lstcam_key)
|
|
130
152
|
|
|
131
153
|
# Timestamp of the first event
|
|
@@ -147,32 +169,33 @@ def copy_to_webserver(html_file, csv_file):
|
|
|
147
169
|
sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
|
|
148
170
|
|
|
149
171
|
|
|
150
|
-
|
|
151
|
-
@click.argument("date", type=click.DateTime(formats=["%Y-%m-%d"]))
|
|
152
|
-
@click.option("-v", "--version", type=str, default=get_lstchain_version())
|
|
153
|
-
def main(date: datetime = None, version: str = get_lstchain_version()):
|
|
172
|
+
def main():
|
|
154
173
|
"""Update source catalog with new run entries from a given date in format YYYY-MM-DD.
|
|
155
174
|
|
|
156
175
|
Notes
|
|
157
176
|
-----
|
|
158
177
|
It needs to be run as lstanalyzer user.
|
|
159
178
|
"""
|
|
160
|
-
|
|
179
|
+
args = parser.parse_args()
|
|
180
|
+
|
|
181
|
+
catalog_path = Path(cfg.get("LST1", "SOURCE_CATALOG")) / "LST_source_catalog.ecsv"
|
|
161
182
|
catalog_table = Table.read(catalog_path)
|
|
162
183
|
|
|
163
184
|
# Open table for given date and append its content to the table with entire catalog
|
|
164
|
-
datedir = date
|
|
165
|
-
|
|
166
|
-
|
|
185
|
+
datedir = date_to_dir(args.date)
|
|
186
|
+
run_catalog_dir = Path(cfg.get("LST1", "RUN_CATALOG"))
|
|
187
|
+
today_catalog = Table.read(run_catalog_dir / f"RunCatalog_{datedir}.ecsv")
|
|
188
|
+
run_summary_dir = Path(cfg.get("LST1", "RUN_SUMMARY_DIR"))
|
|
189
|
+
today_runsummary = Table.read(run_summary_dir / f"RunSummary_{datedir}.ecsv")
|
|
167
190
|
# Keep only astronomical data runs
|
|
168
191
|
today_runsummary = today_runsummary[today_runsummary["run_type"] == "DATA"]
|
|
169
192
|
todays_info = join(today_runsummary, today_catalog)
|
|
170
|
-
todays_info.add_column(date
|
|
193
|
+
todays_info.add_column(date_to_iso(args.date), name="date_dir")
|
|
171
194
|
todays_info.keep_columns(["run_id", "source_name", "date_dir"])
|
|
172
195
|
|
|
173
196
|
# Add start of run in iso format and elapsed time for each run
|
|
174
197
|
log.info("Getting run start and elapsed time")
|
|
175
|
-
add_start_and_elapsed(todays_info, datedir, version)
|
|
198
|
+
add_start_and_elapsed(todays_info, datedir, args.version)
|
|
176
199
|
|
|
177
200
|
# Change column names
|
|
178
201
|
todays_info.rename_column("run_id", "Run ID")
|
|
@@ -198,7 +221,7 @@ def main(date: datetime = None, version: str = get_lstchain_version()):
|
|
|
198
221
|
html_content = add_query_table_to_html(html_table)
|
|
199
222
|
|
|
200
223
|
# Save the HTML and ECSV files and copy them to the LST-1 webserver
|
|
201
|
-
html_file =
|
|
224
|
+
html_file = Path(cfg.get("LST1", "SOURCE_CATALOG")) / "LST_source_catalog.html"
|
|
202
225
|
html_file.write_text(html_content)
|
|
203
226
|
table_unique.write(catalog_path, delimiter=",", overwrite=True)
|
|
204
227
|
|
osa/tests/test_jobs.py
CHANGED
|
@@ -9,18 +9,21 @@ from osa.configs.config import cfg, DEFAULT_CFG
|
|
|
9
9
|
|
|
10
10
|
extra_files = Path(os.getenv("OSA_TEST_DATA", "extra"))
|
|
11
11
|
datasequence_history_file = extra_files / "history_files/sequence_LST1_04185.0010.history"
|
|
12
|
+
datasequence_history_file2 = extra_files / "history_files/sequence_LST1_04185.0001.history"
|
|
12
13
|
calibration_history_file = extra_files / "history_files/sequence_LST1_04183.history"
|
|
13
14
|
options.date = "2020-01-17"
|
|
14
15
|
options.tel_id = "LST1"
|
|
15
16
|
options.prod_id = "v0.1.0"
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
def test_historylevel(
|
|
19
|
+
def test_historylevel(
|
|
20
|
+
run_catalog,
|
|
21
|
+
dl1b_config_files,
|
|
22
|
+
tailcuts_log_files,
|
|
23
|
+
rf_models,
|
|
24
|
+
):
|
|
19
25
|
from osa.job import historylevel
|
|
20
26
|
|
|
21
|
-
options.dl1_prod_id = "tailcut84"
|
|
22
|
-
options.dl2_prod_id = "model1"
|
|
23
|
-
|
|
24
27
|
level, rc = historylevel(datasequence_history_file, "DATA")
|
|
25
28
|
assert level == 0
|
|
26
29
|
assert rc == 0
|
|
@@ -29,11 +32,8 @@ def test_historylevel():
|
|
|
29
32
|
assert level == 0
|
|
30
33
|
assert rc == 0
|
|
31
34
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
level, rc = historylevel(datasequence_history_file, "DATA")
|
|
36
|
-
assert level == 1
|
|
35
|
+
level, rc = historylevel(datasequence_history_file2, "DATA")
|
|
36
|
+
assert level == 2
|
|
37
37
|
assert rc == 0
|
|
38
38
|
|
|
39
39
|
|
|
@@ -41,6 +41,7 @@ def test_preparejobs(running_analysis_dir, sequence_list):
|
|
|
41
41
|
from osa.job import prepare_jobs
|
|
42
42
|
|
|
43
43
|
options.simulate = False
|
|
44
|
+
options.test = True
|
|
44
45
|
options.directory = running_analysis_dir
|
|
45
46
|
prepare_jobs(sequence_list)
|
|
46
47
|
expected_calib_script = os.path.join(running_analysis_dir, "sequence_LST1_01809.py")
|
|
@@ -94,6 +95,7 @@ def test_job_header_template(sequence_list, running_analysis_dir):
|
|
|
94
95
|
from osa.job import job_header_template
|
|
95
96
|
|
|
96
97
|
# Extract the first sequence
|
|
98
|
+
options.test = False
|
|
97
99
|
first_sequence = sequence_list[0]
|
|
98
100
|
header = job_header_template(first_sequence)
|
|
99
101
|
output_string1 = dedent(
|
|
@@ -138,13 +140,17 @@ def test_create_job_template_scheduler(
|
|
|
138
140
|
calibration_file,
|
|
139
141
|
run_summary_file,
|
|
140
142
|
pedestal_ids_file,
|
|
143
|
+
dl1b_config_files,
|
|
144
|
+
rf_models,
|
|
141
145
|
):
|
|
142
146
|
from osa.job import data_sequence_job_template
|
|
143
147
|
|
|
144
148
|
assert pedestal_ids_file.exists()
|
|
149
|
+
assert rf_models[1].exists()
|
|
145
150
|
|
|
146
151
|
options.test = False
|
|
147
152
|
options.simulate = False
|
|
153
|
+
|
|
148
154
|
content1 = data_sequence_job_template(sequence_list[1])
|
|
149
155
|
expected_content1 = dedent(
|
|
150
156
|
f"""\
|
|
@@ -184,6 +190,8 @@ def test_create_job_template_scheduler(
|
|
|
184
190
|
'--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
|
|
185
191
|
'--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
|
|
186
192
|
'--run-summary={run_summary_file}',
|
|
193
|
+
'--dl1b-config={dl1b_config_files[0]}',
|
|
194
|
+
'--dl1-prod-id=tailcut84',
|
|
187
195
|
f'01807.{{subruns:04d}}',
|
|
188
196
|
'LST1'
|
|
189
197
|
])
|
|
@@ -230,6 +238,8 @@ def test_create_job_template_scheduler(
|
|
|
230
238
|
'--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
|
|
231
239
|
'--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
|
|
232
240
|
'--run-summary={run_summary_file}',
|
|
241
|
+
'--dl1b-config={dl1b_config_files[1]}',
|
|
242
|
+
'--dl1-prod-id=tailcut84',
|
|
233
243
|
f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
|
|
234
244
|
f'01808.{{subruns:04d}}',
|
|
235
245
|
'LST1'
|
|
@@ -252,6 +262,8 @@ def test_create_job_template_local(
|
|
|
252
262
|
run_summary_file,
|
|
253
263
|
pedestal_ids_file,
|
|
254
264
|
r0_data,
|
|
265
|
+
dl1b_config_files,
|
|
266
|
+
rf_models,
|
|
255
267
|
):
|
|
256
268
|
"""Check the job file in local mode (assuming no scheduler)."""
|
|
257
269
|
from osa.job import data_sequence_job_template
|
|
@@ -266,6 +278,7 @@ def test_create_job_template_local(
|
|
|
266
278
|
assert file.exists()
|
|
267
279
|
|
|
268
280
|
assert pedestal_ids_file.exists()
|
|
281
|
+
assert rf_models[0].exists()
|
|
269
282
|
|
|
270
283
|
options.test = True
|
|
271
284
|
options.simulate = False
|
|
@@ -296,6 +309,8 @@ def test_create_job_template_local(
|
|
|
296
309
|
'--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
|
|
297
310
|
'--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
|
|
298
311
|
'--run-summary={run_summary_file}',
|
|
312
|
+
'--dl1b-config={dl1b_config_files[0]}',
|
|
313
|
+
'--dl1-prod-id=tailcut84',
|
|
299
314
|
f'01807.{{subruns:04d}}',
|
|
300
315
|
'LST1'
|
|
301
316
|
])
|
|
@@ -329,6 +344,8 @@ def test_create_job_template_local(
|
|
|
329
344
|
'--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
|
|
330
345
|
'--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
|
|
331
346
|
'--run-summary={run_summary_file}',
|
|
347
|
+
'--dl1b-config={dl1b_config_files[1]}',
|
|
348
|
+
'--dl1-prod-id=tailcut84',
|
|
332
349
|
f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
|
|
333
350
|
f'01808.{{subruns:04d}}',
|
|
334
351
|
'LST1'
|
|
@@ -337,11 +354,11 @@ def test_create_job_template_local(
|
|
|
337
354
|
sys.exit(proc.returncode)"""
|
|
338
355
|
)
|
|
339
356
|
|
|
340
|
-
options.simulate = True
|
|
341
|
-
|
|
342
357
|
assert content1 == expected_content1
|
|
343
358
|
assert content2 == expected_content2
|
|
344
359
|
|
|
360
|
+
options.simulate = True
|
|
361
|
+
|
|
345
362
|
|
|
346
363
|
def test_create_job_scheduler_calibration(sequence_list):
|
|
347
364
|
"""Check the pilot job file for the calibration pipeline."""
|
osa/tests/test_paths.py
CHANGED
|
@@ -52,8 +52,8 @@ def test_destination_dir():
|
|
|
52
52
|
from osa.paths import destination_dir
|
|
53
53
|
|
|
54
54
|
datedir = date_to_dir(options.date)
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
dl1_prod_id = "tailcut84"
|
|
56
|
+
dl2_prod_id = "tailcut84/nsb_tuning_0.14"
|
|
57
57
|
options.prod_id = cfg.get("LST1", "PROD_ID")
|
|
58
58
|
base_directory = cfg.get("LST1", "BASE")
|
|
59
59
|
base_path = Path(base_directory)
|
|
@@ -67,14 +67,14 @@ def test_destination_dir():
|
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
for concept, dst_dir in data_types.items():
|
|
70
|
-
directory = destination_dir(concept, create_dir=False)
|
|
70
|
+
directory = destination_dir(concept, create_dir=False, dl1_prod_id=dl1_prod_id, dl2_prod_id=dl2_prod_id)
|
|
71
71
|
if concept == "DL1AB":
|
|
72
72
|
expected_directory = (
|
|
73
|
-
base_path / dst_dir / datedir / options.prod_id /
|
|
73
|
+
base_path / dst_dir / datedir / options.prod_id / dl1_prod_id
|
|
74
74
|
)
|
|
75
75
|
elif concept == "DATACHECK":
|
|
76
76
|
expected_directory = (
|
|
77
|
-
base_path / dst_dir / datedir / options.prod_id /
|
|
77
|
+
base_path / dst_dir / datedir / options.prod_id / dl1_prod_id / "datacheck"
|
|
78
78
|
)
|
|
79
79
|
elif concept == "MUON":
|
|
80
80
|
expected_directory = (
|
|
@@ -86,7 +86,7 @@ def test_destination_dir():
|
|
|
86
86
|
)
|
|
87
87
|
elif concept == "DL2":
|
|
88
88
|
expected_directory = (
|
|
89
|
-
base_path / dst_dir / datedir / options.prod_id /
|
|
89
|
+
base_path / dst_dir / datedir / options.prod_id / dl2_prod_id
|
|
90
90
|
)
|
|
91
91
|
|
|
92
92
|
assert directory == expected_directory
|
osa/tests/test_raw.py
CHANGED
|
@@ -17,7 +17,7 @@ def test_get_raw_dir():
|
|
|
17
17
|
assert get_raw_dir(options.date) == r0_dir
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
def test_get_check_raw_dir(
|
|
20
|
+
def test_get_check_raw_dir(r0g_dir):
|
|
21
21
|
options.date = datetime.fromisoformat("2020-01-18")
|
|
22
22
|
from osa.raw import get_check_raw_dir
|
|
23
23
|
|
|
@@ -26,13 +26,13 @@ def test_get_check_raw_dir(r0_dir):
|
|
|
26
26
|
|
|
27
27
|
options.date = datetime.fromisoformat("2020-01-17")
|
|
28
28
|
raw_dir = get_check_raw_dir(options.date)
|
|
29
|
-
assert raw_dir.resolve() ==
|
|
29
|
+
assert raw_dir.resolve() == r0g_dir
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def test_is_raw_data_available(
|
|
32
|
+
def test_is_raw_data_available(r0g_data):
|
|
33
33
|
from osa.raw import is_raw_data_available
|
|
34
34
|
|
|
35
|
-
for file in
|
|
35
|
+
for file in r0g_data:
|
|
36
36
|
assert file.exists()
|
|
37
37
|
|
|
38
38
|
options.date = datetime.fromisoformat("2020-01-17")
|