lstosa 0.10.18__py3-none-any.whl → 0.11.0__py3-none-any.whl

osa/scripts/provprocess.py

@@ -17,6 +17,8 @@ from osa.provenance.io import provdoc2graph, provdoc2json, provlist2provdoc, rea
 from osa.provenance.utils import get_log_config
 from osa.utils.cliopts import provprocessparsing
 from osa.utils.logging import myLogger
+from osa.utils.utils import date_to_dir
+from osa.paths import get_dl1_prod_id_and_config, get_dl2_prod_id
 
 __all__ = ["copy_used_file", "parse_lines_log", "parse_lines_run", "produce_provenance"]
 
@@ -110,7 +112,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number):
                 keep = True
             # make session starts with calibration
             if session_id and filter_cut == "all" and not filtered:
-                prov_dict["session_id"] = f"{options.date}{run_number}"
+                nightdir = date_to_dir(options.date)
+                prov_dict["session_id"] = f"{nightdir}{run_number}"
                 prov_dict["name"] = run_number
                 prov_dict["observation_run"] = run_number
                 line = f"{ll[0]}{PROV_PREFIX}{ll[1]}{PROV_PREFIX}{prov_dict}\n"
@@ -336,7 +339,8 @@ def define_paths(grain, start_path, end_path, base_filename):
     paths = {}
 
     # check destination folder exists
-    step_path = Path(start_path) / options.date / options.prod_id / end_path
+    nightdir = date_to_dir(options.date)
+    step_path = Path(start_path) / nightdir / options.prod_id / end_path
     if not step_path.exists():
         log.error(f"Path {step_path} does not exist")
 
@@ -381,8 +385,9 @@ def produce_provenance(session_log_filename, base_filename):
     """
 
     if options.filter == "calibration" or not options.filter:
+        dl1_prod_id = get_dl1_prod_id_and_config(int(options.run))[0]
         paths_calibration = define_paths(
-            "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename
+            "calibration_to_dl1", PATH_DL1, dl1_prod_id, base_filename
         )
         plines_drs4 = parse_lines_run(
             "drs4_pedestal",
@@ -402,7 +407,8 @@ def produce_provenance(session_log_filename, base_filename):
         pass
 
     if options.filter == "r0_to_dl1" or not options.filter:
-        paths_r0_dl1 = define_paths("r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename)
+        dl1_prod_id = get_dl1_prod_id_and_config(int(options.run))[0]
+        paths_r0_dl1 = define_paths("r0_to_dl1", PATH_DL1, dl1_prod_id, base_filename)
         plines_r0 = parse_lines_run(
             "r0_to_dl1",
             read_prov(filename=session_log_filename),
@@ -425,8 +431,9 @@ def produce_provenance(session_log_filename, base_filename):
         produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1)
 
     if options.filter == "dl1_to_dl2" or not options.filter:
+        dl2_prod_id = get_dl2_prod_id(int(options.run))
         if not options.no_dl2:
-            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
+            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, dl2_prod_id, base_filename)
             plines_dl2 = parse_lines_run(
                 "dl1_to_dl2",
                 read_prov(filename=session_log_filename),
@@ -441,16 +448,18 @@ def produce_provenance(session_log_filename, base_filename):
 
     # create calibration_to_dl1 and calibration_to_dl2 prov files
     if not options.filter:
+        dl1_prod_id = get_dl1_prod_id_and_config(int(options.run))[0]
         calibration_to_dl1 = define_paths(
-            "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename
+            "calibration_to_dl1", PATH_DL1, dl1_prod_id, base_filename
         )
         calibration_to_dl1_lines = calibration_lines + dl1_lines[1:]
         lines_dl1 = copy.deepcopy(calibration_to_dl1_lines)
         produce_provenance_files(lines_dl1, calibration_to_dl1)
 
         if not options.no_dl2:
+            dl2_prod_id = get_dl2_prod_id(int(options.run))
             calibration_to_dl2 = define_paths(
-                "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename
+                "calibration_to_dl2", PATH_DL2, dl2_prod_id, base_filename
             )
             calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:]
             lines_dl2 = copy.deepcopy(calibration_to_dl2_lines)

osa/scripts/sequencer.py

@@ -5,6 +5,7 @@ Orchestrator script that creates and execute the calibration sequence and
 prepares a SLURM job array which launches the data sequences for every subrun.
 """
 
+import warnings
 import logging
 import os
 import sys
@@ -14,7 +15,15 @@ import datetime
 from osa import osadb
 from osa.configs import options
 from osa.configs.config import cfg
-from osa.job import (
+from osa.veto import get_closed_list, get_veto_list
+from osa.utils.logging import myLogger
+
+warnings.filterwarnings(
+    "ignore",
+    message="pkg_resources is deprecated as an API.*",
+    category=UserWarning
+)
+from osa.job import ( # noqa: E402
     set_queue_values,
     prepare_jobs,
     submit_jobs,
@@ -22,16 +31,19 @@ from osa.job import (
     get_squeue_output,
     run_sacct,
     run_squeue,
+    are_all_jobs_correctly_finished,
 )
-from osa.nightsummary.extract import build_sequences
-from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import analysis_path
-from osa.report import start
-from osa.utils.cliopts import sequencer_cli_parsing
-from osa.utils.logging import myLogger
-from osa.utils.utils import is_day_closed, gettag, date_to_iso
-from osa.veto import get_closed_list, get_veto_list
-from osa.scripts.gain_selection import GainSel_finished
+from osa.nightsummary.extract import ( # noqa: E402
+    build_sequences,
+    extract_runs,
+    extract_sequences
+)
+from osa.nightsummary.nightsummary import run_summary_table # noqa: E402
+from osa.paths import analysis_path, destination_dir # noqa: E402
+from osa.report import start # noqa: E402
+from osa.utils.cliopts import sequencer_cli_parsing # noqa: E402
+from osa.utils.utils import is_day_closed, gettag, date_to_iso # noqa: E402
+from osa.scripts.gain_selection import GainSel_finished # noqa: E402
 
 __all__ = [
     "single_process",
@@ -115,7 +127,7 @@ def single_process(telescope):
             log.info(f"Sequencer is still running for date {date_to_iso(options.date)}. Try again later.")
             sys.exit(0)
 
-        elif is_sequencer_completed(options.date) and not options.force_submit:
+        if is_sequencer_completed(options.date) and not options.force_submit:
             log.info(f"Sequencer already finished for date {date_to_iso(options.date)}. Exiting")
             sys.exit(0)
 
@@ -192,7 +204,7 @@ def update_sequence_status(seq_list):
                 Decimal(get_status_for_sequence(seq, "DATACHECK") * 100) / seq.subruns
             )
             seq.muonstatus = int(Decimal(get_status_for_sequence(seq, "MUON") * 100) / seq.subruns)
-            seq.dl2status = int(Decimal(get_status_for_sequence(seq, "DL2") * 100) / seq.subruns)
+            seq.dl2status = int(Decimal(get_status_for_sequence(seq, "DL2") * 100))
 
 
 def get_status_for_sequence(sequence, data_level) -> int:
@@ -210,17 +222,26 @@ def get_status_for_sequence(sequence, data_level) -> int:
     number_of_files : int
     """
     if data_level == "DL1AB":
-        directory = options.directory / options.dl1_prod_id
-        files = list(directory.glob(f"dl1_LST-1*{sequence.run}*.h5"))
-
+        try:
+            directory = options.directory / sequence.dl1_prod_id
+            files = list(directory.glob(f"dl1_LST-1*{sequence.run}*.h5"))
+        except AttributeError:
+            return 0
+        
     elif data_level == "DL2":
-        directory = options.directory / options.dl2_prod_id
-        files = list(directory.glob(f"dl2_LST-1*{sequence.run}*.h5"))
-
+        try:
+            directory = destination_dir(concept="DL2", create_dir=False, dl2_prod_id=sequence.dl2_prod_id)
+            files = list(directory.glob(f"dl2_LST-1*{sequence.run}*.h5"))
+        except AttributeError:
+            return 0
+        
     elif data_level == "DATACHECK":
-        directory = options.directory / options.dl1_prod_id
-        files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
-
+        try:
+            directory = options.directory / sequence.dl1_prod_id
+            files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+        except AttributeError:
+            return 0
+        
     else:
         prefix = cfg.get("PATTERN", f"{data_level}PREFIX")
         suffix = cfg.get("PATTERN", f"{data_level}SUFFIX")
@@ -340,20 +361,14 @@ def is_sequencer_completed(date: datetime.datetime) -> bool:
     """Check if the jobs launched by sequencer are already completed."""
     summary_table = run_summary_table(date)
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
-    sacct_output = run_sacct()
-    sacct_info = get_sacct_output(sacct_output)
-
-    for run in data_runs["run_id"]:
-        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
-        if len(jobs_run["JobID"].unique())>1:
-            last_job_id = sorted(jobs_run["JobID"].unique())[-1]
-            jobs_run = sacct_info[sacct_info["JobID"]==last_job_id]
-        incomplete_jobs = jobs_run[(jobs_run["State"] != "COMPLETED")]
-        if len(jobs_run) == 0 or len(incomplete_jobs) != 0:
-            return False
-
-    return True
+    run_list = extract_runs(data_runs)
+    sequence_list = extract_sequences(options.date, run_list)
 
+    if are_all_jobs_correctly_finished(sequence_list):
+        return True
+    else:
+        log.info("Jobs did not correctly/yet finish")
+        return False
 
 def timeout_in_sequencer(date: datetime.datetime) -> bool:
     """Check if any of the jobs launched by sequencer finished in timeout."""

osa/scripts/sequencer_catB_tailcuts.py

@@ -0,0 +1,239 @@
+import glob
+import re
+import argparse
+import logging
+from pathlib import Path
+from astropy.table import Table
+import subprocess as sp
+
+from osa.configs import options
+from osa.configs.config import cfg
+from osa.nightsummary.extract import get_last_pedcalib
+from osa.utils.cliopts import valid_date, set_default_date_if_needed
+from osa.utils.logging import myLogger
+from osa.job import run_sacct, get_sacct_output
+from osa.utils.utils import date_to_dir, get_calib_filters, get_lstchain_version
+from osa.paths import (
+    catB_closed_file_exists,
+    catB_calibration_file_exists,
+    analysis_path,
+    get_major_version
+)
+
+log = myLogger(logging.getLogger())
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "-c",                                                                                            
+    "--config",                                                                                      
+    action="store",
+    type=Path,
+    help="Configuration file",
+)
+parser.add_argument(                                                                                     
+    "-d",                                                                                            
+    "--date",
+    type=valid_date,
+    default=None,
+)
+parser.add_argument(
+    "-v",                                                                                      
+    "--verbose",
+    action="store_true",
+    default=False,
+    help="Activate debugging mode.",
+)
+parser.add_argument(
+    "-s",
+    "--simulate",
+    action="store_true",
+    default=False,
+    help="Simulate launching of the sequencer_catB_tailcuts script.",
+)
+parser.add_argument(
+    "tel_id",
+    choices=["ST", "LST1", "LST2", "all"],
+    help="telescope identifier LST1, LST2, ST or all.",
+)
+
+def are_all_history_files_created(run_id: int) -> bool:
+    """Check if all the history files (one per subrun) were created for a given run."""
+    run_summary_dir = Path(cfg.get(options.tel_id, "RUN_SUMMARY_DIR"))
+    run_summary_file = run_summary_dir / f"RunSummary_{date_to_dir(options.date)}.ecsv"
+    run_summary = Table.read(run_summary_file)
+    n_subruns = run_summary[run_summary["run_id"] == run_id]["n_subruns"]
+    analysis_dir = Path(options.directory)
+    history_files = glob.glob(f"{str(analysis_dir)}/sequence_LST1_{run_id:05d}.????.history")
+    if len(history_files) == n_subruns:
+        return True
+    else:
+        return False
+
+
+def r0_to_dl1_step_finished_for_run(run_id: int) -> bool:
+    """
+    Check if the step r0_to_dl1 finished successfully 
+    for a given run by looking the history files.
+    """
+    if not are_all_history_files_created(run_id):
+        log.debug(f"All history files for run {run_id:05d} were not created yet.")
+        return False
+    analysis_dir = Path(options.directory)
+    history_files = glob.glob(f"{str(analysis_dir)}/sequence_LST1_{run_id:05d}.????.history")
+    for file in history_files:
+        rc = Path(file).read_text().splitlines()[-1][-1]
+        if rc != "0":
+            print(f"r0_to_dl1 step did not finish successfully (check file {file})")
+            return False
+    return True
+
+
+def get_catB_last_job_id(run_id: int) -> int:
+    """Get job id of the last Cat-B calibration job that was launched for a given run."""
+    log_dir = Path(options.directory) / "log"
+    filenames = glob.glob(f"{log_dir}/catB_calibration_{run_id:05d}_*.err")
+    if filenames:
+        match = re.search(f"catB_calibration_{run_id:05d}_(\d+).err", sorted(filenames)[-1])
+        job_id = match.group(1)
+        return job_id
+
+
+def launch_catB_calibration(run_id: int):
+    """
+    Launch the Cat-B calibration script for a given run if the Cat-B calibration 
+    file has not been created yet. If the Cat-B calibration script was launched
+    before and it finished successfully, it creates a catB_{run}.closed file.
+    """
+    job_id = get_catB_last_job_id(run_id)
+    if job_id:
+        job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+        if job_status.item() in ["RUNNING", "PENDING"]:
+            log.debug(f"Job {job_id} (corresponding to run {run_id:05d}) is still running.")
+
+        elif job_status.item() == "COMPLETED":
+            catB_closed_file = Path(options.directory) / f"catB_{run_id:05d}.closed"
+            catB_closed_file.touch()
+            log.debug(
+                f"Cat-B job {job_id} (corresponding to run {run_id:05d}) finished "
+                f"successfully. Creating file {catB_closed_file}"
+            )
+
+        else: 
+            log.warning(f"Cat-B job {job_id} (corresponding to run {run_id:05d}) failed.")
+
+    else:
+        if catB_calibration_file_exists(run_id):
+            log.info(f"Cat-B calibration file already produced for run {run_id:05d}.")
+            return 
+
+        command = cfg.get("lstchain", "catB_calibration")
+        if cfg.getboolean("lstchain", "use_lstcam_env_for_CatB_calib"):
+            env_command = f"conda run -n lstcam-env {command}"
+        else:
+            env_command = command
+        options.filters = get_calib_filters(run_id) 
+        base_dir = Path(cfg.get(options.tel_id, "BASE")).resolve()
+        r0_dir = Path(cfg.get(options.tel_id, "R0_DIR")).resolve()
+        log_dir = Path(options.directory) / "log"
+        catA_calib_run = get_last_pedcalib(options.date)
+        slurm_account = cfg.get("SLURM", "ACCOUNT")
+        lstchain_version = get_major_version(get_lstchain_version())
+        analysis_dir = cfg.get("LST1", "ANALYSIS_DIR")
+        cmd = ["sbatch", f"--account={slurm_account}", "--parsable",
+            "-o", f"{log_dir}/catB_calibration_{run_id:05d}_%j.out",
+            "-e", f"{log_dir}/catB_calibration_{run_id:05d}_%j.err",
+            env_command,
+            f"-r {run_id:05d}",
+            f"--catA_calibration_run={catA_calib_run}",
+            "-b", base_dir,
+            f"--r0-dir={r0_dir}",
+            f"--filters={options.filters}",
+        ]
+        
+        if command=="onsite_create_cat_B_calibration_file":
+            cmd.append(f"--interleaved-dir={analysis_dir}")
+        elif command=="lstcam_calib_onsite_create_cat_B_calibration_file":
+            cmd.append(f"--dl1-dir={analysis_dir}")
+            cmd.append(f"--lstchain-version={lstchain_version[1:]}")
+
+        if not options.simulate:
+            job = sp.run(cmd, encoding="utf-8", capture_output=True, text=True, check=True)
+            job_id = job.stdout.strip()
+            log.debug(f"Launched Cat-B calibration job {job_id} for run {run_id}!")
+
+        else: 
+            log.info(f"Simulate launching of the {command} script.")
+            
+
+def launch_tailcuts_finder(run_id: int):
+    """
+    Launch the lstchain script to calculate the correct
+    tailcuts to use for a given run. 
+    """
+    command = cfg.get("lstchain", "tailcuts_finder")
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
+    input_dir = Path(options.directory)
+    output_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+    log_dir = Path(options.directory) / "log"
+    log_file = log_dir / f"tailcuts_finder_{run_id:05d}_%j.log"
+    cmd = [
+        "sbatch", "--parsable",
+        f"--account={slurm_account}",
+        "-o", log_file,
+        command,
+        f"--input-dir={input_dir}",
+        f"--run={run_id}",
+        f"--output-dir={output_dir}",
+    ]
+    if not options.simulate:
+        job = sp.run(cmd, encoding="utf-8", capture_output=True, text=True, check=True)
+        job_id = job.stdout.strip()
+        log.debug(f"Launched lstchain_find_tailcuts job {job_id} for run {run_id}!")
+
+    else: 
+        log.info(f"Simulate launching of the {command} script.")
+
+
+
+def tailcuts_config_file_exists(run_id: int) -> bool:
+    """Check if the config file created by the tailcuts finder script already exists."""
+    tailcuts_config_file = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR")) / f"dl1ab_Run{run_id:05d}.json"
+    return tailcuts_config_file.exists()
+    
+        
+def main():
+    """
+    Main script to be called as cron job. It launches the Cat-B calibration script 
+    and the tailcuts finder script for each run of the corresponding date, and creates
+    the catB_{run}.closed files if Cat-B calibration has finished successfully.
+    """ 
+    opts = parser.parse_args()
+    options.tel_id = opts.tel_id
+    options.simulate = opts.simulate
+    options.date = opts.date
+    options.date = set_default_date_if_needed()
+    options.configfile = opts.config.resolve()
+    options.directory = analysis_path(options.tel_id)
+
+    if opts.verbose:
+        log.setLevel(logging.DEBUG)
+    else:
+        log.setLevel(logging.INFO)
+
+    run_summary_dir = Path(cfg.get(options.tel_id, "RUN_SUMMARY_DIR"))
+    run_summary = Table.read(run_summary_dir / f"RunSummary_{date_to_dir(options.date)}.ecsv")
+    data_runs = run_summary[run_summary["run_type"]=="DATA"]
+    for run_id in data_runs["run_id"]:
+        # first check if the dl1a files are produced
+        if not r0_to_dl1_step_finished_for_run(run_id):
+            log.info(f"The r0_to_dl1 step did not finish yet for run {run_id:05d}. Please try again later.")
+        else:
+            # launch catB calibration and tailcut finder in parallel
+            if cfg.getboolean("lstchain", "apply_catB_calibration") and not catB_closed_file_exists(run_id):
+                launch_catB_calibration(run_id)
+            if not cfg.getboolean("lstchain", "apply_standard_dl1b_config") and not tailcuts_config_file_exists(run_id):
+                launch_tailcuts_finder(run_id)
+
+
+if __name__ == "__main__":
+    main()

osa/scripts/sequencer_webmaker.py

@@ -16,6 +16,7 @@ from osa.configs.config import cfg
 from osa.utils.cliopts import sequencer_webmaker_argparser
 from osa.utils.logging import myLogger
 from osa.utils.utils import is_day_closed, date_to_iso, date_to_dir
+from osa.paths import all_dl1ab_config_files_exist
 
 log = myLogger(logging.getLogger())
 
@@ -88,6 +89,9 @@ def get_sequencer_output(date: str, config: str, test=False, no_gainsel=False) -
     if test:
         commandargs.insert(-1, "-t")
 
+    if not all_dl1ab_config_files_exist(date):
+        commandargs.insert(-1, "--no-dl1ab")
+
     try:
         output = sp.run(commandargs, stdout=sp.PIPE, stderr=sp.STDOUT, encoding="utf-8", check=True)
     except sp.CalledProcessError as error:

osa/scripts/show_run_summary.py

@@ -26,8 +26,8 @@ parser.add_argument(
 parser.add_argument(
     "--r0-path",
     type=Path,
-    help="Path to the R0 files. Default is /fefs/aswg/data/real/R0",
-    default=Path("/fefs/aswg/data/real/R0"),
+    help="Path to the R0 files. Default is /fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events",
+    default=Path("/fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events"),
 )
 
 parser.add_argument(

osa/scripts/simulate_processing.py

@@ -8,7 +8,6 @@ python osa/scripts/simulate_processing.py"""
 import logging
 import multiprocessing as mp
 import subprocess
-from datetime import datetime
 from pathlib import Path
 
 import yaml
@@ -20,7 +19,8 @@ from osa.nightsummary.extract import build_sequences
 from osa.provenance.utils import get_log_config
 from osa.utils.cliopts import simprocparsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir
+from osa.utils.utils import date_to_iso
+from osa.paths import analysis_path
 
 __all__ = [
     "parse_template",
@@ -174,7 +174,7 @@ def simulate_processing():
                 drs4_pedestal_run_id,
                 pedcal_run_id,
                 sequence.run_str,
-                options.directory,
+                date_to_iso(options.date),
                 options.prod_id,
             ]
             log.info(f"Processing provenance for run {sequence.run_str}")
@@ -187,10 +187,7 @@ def main():
 
     simprocparsing()
 
-    # date and tel_id hardcoded for the moment
-    options.date = datetime.fromisoformat("2020-01-17")
-    options.tel_id = "LST1"
-    options.directory = date_to_dir(options.date)
+    options.directory = analysis_path(options.tel_id)
 
     log.info("Running simulate processing")