lstosa 0.10.18__py3-none-any.whl → 0.10.19__py3-none-any.whl

osa/scripts/datasequence.py

@@ -9,11 +9,13 @@ from osa.configs.config import cfg
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
+from osa.paths import get_catB_calibration_filename
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir
+from osa.paths import catB_closed_file_exists
 
-__all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"]
+__all__ = ["data_sequence", "r0_to_dl1", "dl1ab", "dl1_datacheck"]
 
 log = myLogger(logging.getLogger())
 
@@ -27,6 +29,8 @@ def data_sequence(
     run_summary: Path,
     pedestal_ids_file: Path,
     run_str: str,
+    dl1b_config: Path,
+    dl1_prod_id: str,
 ):
     """
     Performs all the steps to process a whole run.
@@ -50,10 +54,10 @@ def data_sequence(
     history_file = Path(options.directory) / f"sequence_{options.tel_id}_{run_str}.history"
     # Set the starting level and corresponding return code from last analysis step
     # registered in the history file.
-    level, rc = (4, 0) if options.simulate else historylevel(history_file, "DATA")
+    level, rc = (3, 0) if options.simulate else historylevel(history_file, "DATA")
     log.info(f"Going to level {level}")
 
-    if level == 4:
+    if level == 3:
         rc = r0_to_dl1(
             calibration_file,
             pedestal_file,
@@ -67,32 +71,23 @@ def data_sequence(
         level -= 1
         log.info(f"Going to level {level}")
 
-    if level == 3:
-        rc = dl1ab(run_str)
-        if cfg.getboolean("lstchain", "store_image_dl1ab"):
-            level -= 1
-            log.info(f"Going to level {level}")
-        else:
-            level -= 2
-            log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}")
-
     if level == 2:
-        rc = dl1_datacheck(run_str)
-        if options.no_dl2:
+        if options.no_dl1ab:
             level = 0
-            log.info(f"No DL2 are going to be produced. Going to level {level}")
+            log.info(f"No DL1B are going to be produced. Going to level {level}")
         else:
-            level -= 1
-            log.info(f"Going to level {level}")
+            rc = dl1ab(run_str, dl1b_config, dl1_prod_id)
+            if cfg.getboolean("lstchain", "store_image_dl1ab"):
+                level -= 1
+                log.info(f"Going to level {level}")
+            else:
+                level -= 2
+                log.info(f"No images stored in dl1ab. Going to level {level}")
 
     if level == 1:
-        if options.no_dl2:
-            level = 0
-            log.info(f"No DL2 are going to be produced. Going to level {level}")
-        else:
-            rc = dl1_to_dl2(run_str)
-            level -= 1
-            log.info(f"Going to level {level}")
+        rc = dl1_datacheck(run_str, dl1_prod_id)
+        level -= 1
+        log.info(f"Going to level {level}")
 
     if level == 0:
         log.info(f"Job for sequence {run_str} finished without fatal errors")
@@ -166,7 +161,7 @@ def r0_to_dl1(
 
 
 @trace
-def dl1ab(run_str: str) -> int:
+def dl1ab(run_str: str, dl1b_config: Path, dl1_prod_id: str) -> int:
     """
     Prepare and launch the actual lstchain script that is performing
     the image cleaning considering the interleaved pedestal information
@@ -181,26 +176,39 @@ def dl1ab(run_str: str) -> int:
     rc: int
         Return code of the executed command.
     """
+    
+    # Prepare and launch the actual lstchain script
+    command = cfg.get("lstchain", "dl1ab")
+    
     # Create a new subdirectory for the dl1ab output
-    dl1ab_subdirectory = Path(options.directory) / options.dl1_prod_id
+    dl1ab_subdirectory = Path(options.directory) / dl1_prod_id
     dl1ab_subdirectory.mkdir(parents=True, exist_ok=True)
-    dl1b_config = Path(cfg.get("lstchain", "dl1b_config"))
     # DL1a input file from base running_analysis directory
     input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5"
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
 
-    # Prepare and launch the actual lstchain script
-    command = cfg.get("lstchain", "dl1ab")
     cmd = [
         command,
         f"--input-file={input_dl1_datafile}",
         f"--output-file={output_dl1_datafile}",
         f"--config={dl1b_config}",
     ]
+    
     if not cfg.getboolean("lstchain", "store_image_dl1ab"):
         cmd.append("--no-image=True")
 
+    if cfg.getboolean("lstchain", "apply_catB_calibration"):
+        if catB_closed_file_exists(int(run_str[:5])):
+            catB_calibration_file = get_catB_calibration_filename(int(run_str[:5]))
+            cmd.append(f"--catB-calibration-file={catB_calibration_file}")
+        else:
+            log.info(
+                f"Cat-B calibration did not finish yet for run {run_str[:5]}. "
+                "Please try again later."
+            )
+            sys.exit(1)
+
     if options.simulate:
         return 0
 
@@ -210,7 +218,7 @@ def dl1ab(run_str: str) -> int:
 
 
 @trace
-def dl1_datacheck(run_str: str) -> int:
+def dl1_datacheck(run_str: str, dl1_prod_id: str) -> int:
     """
     Run datacheck script
 
@@ -223,9 +231,9 @@ def dl1_datacheck(run_str: str) -> int:
     rc: int
     """
     # Create a new subdirectory for the dl1ab output
-    dl1ab_subdirectory = Path(options.directory) / options.dl1_prod_id
+    dl1ab_subdirectory = Path(options.directory) / dl1_prod_id
     input_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
-    output_directory = Path(options.directory) / options.dl1_prod_id
+    output_directory = Path(options.directory) / dl1_prod_id
     output_directory.mkdir(parents=True, exist_ok=True)
 
     # Prepare and launch the actual lstchain script
@@ -247,46 +255,8 @@ def dl1_datacheck(run_str: str) -> int:
     return analysis_step.rc
 
 
-@trace
-def dl1_to_dl2(run_str: str) -> int:
-    """
-    It prepares and execute the dl1 to dl2 lstchain scripts that applies
-    the already trained RFs models to DL1 files. It identifies the
-    primary particle, reconstructs its energy and direction.
-
-    Parameters
-    ----------
-    run_str: str
-
-    Returns
-    -------
-    rc: int
-    """
-    dl1ab_subdirectory = Path(options.directory) / options.dl1_prod_id
-    dl2_subdirectory = Path(options.directory) / options.dl2_prod_id
-    dl2_config = Path(cfg.get("lstchain", "dl2_config"))
-    rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
-    dl1_file = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
-
-    command = cfg.get("lstchain", "dl1_to_dl2")
-    cmd = [
-        command,
-        f"--input-file={dl1_file}",
-        f"--output-dir={dl2_subdirectory}",
-        f"--path-models={rf_models_directory}",
-        f"--config={dl2_config}",
-    ]
-
-    if options.simulate:
-        return 0
-
-    analysis_step = AnalysisStage(run=run_str, command_args=cmd, config_file=dl2_config.name)
-    analysis_step.execute()
-    return analysis_step.rc
-
-
 def main():
-    """Performs the analysis steps to convert raw data into DL2 files."""
+    """Performs the analysis steps to convert raw data into DL1b files."""
     (
         calibration_file,
         drs4_ped_file,
@@ -296,6 +266,8 @@ def main():
         run_summary_file,
         pedestal_ids_file,
         run_number,
+        dl1b_config,
+        dl1_prod_id,
     ) = data_sequence_cli_parsing()
 
     if options.verbose:
@@ -313,6 +285,8 @@ def main():
         run_summary_file,
         pedestal_ids_file,
         run_number,
+        dl1b_config,
+        dl1_prod_id,
     )
     sys.exit(rc)
 

osa/scripts/provprocess.py

@@ -17,6 +17,8 @@ from osa.provenance.io import provdoc2graph, provdoc2json, provlist2provdoc, rea
 from osa.provenance.utils import get_log_config
 from osa.utils.cliopts import provprocessparsing
 from osa.utils.logging import myLogger
+from osa.utils.utils import date_to_dir
+from osa.paths import get_dl1_prod_id_and_config, get_dl2_prod_id
 
 __all__ = ["copy_used_file", "parse_lines_log", "parse_lines_run", "produce_provenance"]
 
@@ -110,7 +112,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number):
                 keep = True
             # make session starts with calibration
             if session_id and filter_cut == "all" and not filtered:
-                prov_dict["session_id"] = f"{options.date}{run_number}"
+                nightdir = date_to_dir(options.date)
+                prov_dict["session_id"] = f"{nightdir}{run_number}"
                 prov_dict["name"] = run_number
                 prov_dict["observation_run"] = run_number
                 line = f"{ll[0]}{PROV_PREFIX}{ll[1]}{PROV_PREFIX}{prov_dict}\n"
@@ -336,7 +339,8 @@ def define_paths(grain, start_path, end_path, base_filename):
     paths = {}
 
     # check destination folder exists
-    step_path = Path(start_path) / options.date / options.prod_id / end_path
+    nightdir = date_to_dir(options.date)
+    step_path = Path(start_path) / nightdir / options.prod_id / end_path
     if not step_path.exists():
         log.error(f"Path {step_path} does not exist")
 
@@ -381,8 +385,9 @@ def produce_provenance(session_log_filename, base_filename):
     """
 
     if options.filter == "calibration" or not options.filter:
+        dl1_prod_id = get_dl1_prod_id_and_config(int(options.run))[0]
         paths_calibration = define_paths(
-            "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename
+            "calibration_to_dl1", PATH_DL1, dl1_prod_id, base_filename
         )
         plines_drs4 = parse_lines_run(
             "drs4_pedestal",
@@ -402,7 +407,8 @@ def produce_provenance(session_log_filename, base_filename):
         pass
 
     if options.filter == "r0_to_dl1" or not options.filter:
-        paths_r0_dl1 = define_paths("r0_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename)
+        dl1_prod_id = get_dl1_prod_id_and_config(int(options.run))[0]
+        paths_r0_dl1 = define_paths("r0_to_dl1", PATH_DL1, dl1_prod_id, base_filename)
         plines_r0 = parse_lines_run(
             "r0_to_dl1",
             read_prov(filename=session_log_filename),
@@ -425,8 +431,9 @@ def produce_provenance(session_log_filename, base_filename):
         produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1)
 
     if options.filter == "dl1_to_dl2" or not options.filter:
+        dl2_prod_id = get_dl2_prod_id(int(options.run))
         if not options.no_dl2:
-            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
+            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, dl2_prod_id, base_filename)
             plines_dl2 = parse_lines_run(
                 "dl1_to_dl2",
                 read_prov(filename=session_log_filename),
@@ -441,16 +448,18 @@ def produce_provenance(session_log_filename, base_filename):
 
     # create calibration_to_dl1 and calibration_to_dl2 prov files
     if not options.filter:
+        dl1_prod_id = get_dl1_prod_id_and_config(int(options.run))[0]
         calibration_to_dl1 = define_paths(
-            "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename
+            "calibration_to_dl1", PATH_DL1, dl1_prod_id, base_filename
         )
         calibration_to_dl1_lines = calibration_lines + dl1_lines[1:]
         lines_dl1 = copy.deepcopy(calibration_to_dl1_lines)
         produce_provenance_files(lines_dl1, calibration_to_dl1)
 
         if not options.no_dl2:
+            dl2_prod_id = get_dl2_prod_id(int(options.run))
             calibration_to_dl2 = define_paths(
-                "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename
+                "calibration_to_dl2", PATH_DL2, dl2_prod_id, base_filename
             )
             calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:]
             lines_dl2 = copy.deepcopy(calibration_to_dl2_lines)

osa/scripts/sequencer.py

@@ -22,10 +22,15 @@ from osa.job import (
     get_squeue_output,
     run_sacct,
     run_squeue,
+    are_all_jobs_correctly_finished,
+)
+from osa.nightsummary.extract import (
+    build_sequences,
+    extract_runs,
+    extract_sequences
 )
-from osa.nightsummary.extract import build_sequences
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import analysis_path
+from osa.paths import analysis_path, destination_dir
 from osa.report import start
 from osa.utils.cliopts import sequencer_cli_parsing
 from osa.utils.logging import myLogger
@@ -115,7 +120,7 @@ def single_process(telescope):
             log.info(f"Sequencer is still running for date {date_to_iso(options.date)}. Try again later.")
             sys.exit(0)
 
-        elif is_sequencer_completed(options.date) and not options.force_submit:
+        if is_sequencer_completed(options.date) and not options.force_submit:
             log.info(f"Sequencer already finished for date {date_to_iso(options.date)}. Exiting")
             sys.exit(0)
 
@@ -192,7 +197,7 @@ def update_sequence_status(seq_list):
                 Decimal(get_status_for_sequence(seq, "DATACHECK") * 100) / seq.subruns
             )
             seq.muonstatus = int(Decimal(get_status_for_sequence(seq, "MUON") * 100) / seq.subruns)
-            seq.dl2status = int(Decimal(get_status_for_sequence(seq, "DL2") * 100) / seq.subruns)
+            seq.dl2status = int(Decimal(get_status_for_sequence(seq, "DL2") * 100))
 
 
 def get_status_for_sequence(sequence, data_level) -> int:
@@ -210,17 +215,26 @@ def get_status_for_sequence(sequence, data_level) -> int:
     number_of_files : int
     """
     if data_level == "DL1AB":
-        directory = options.directory / options.dl1_prod_id
-        files = list(directory.glob(f"dl1_LST-1*{sequence.run}*.h5"))
-
+        try:
+            directory = options.directory / sequence.dl1_prod_id
+            files = list(directory.glob(f"dl1_LST-1*{sequence.run}*.h5"))
+        except AttributeError:
+            return 0
+        
     elif data_level == "DL2":
-        directory = options.directory / options.dl2_prod_id
-        files = list(directory.glob(f"dl2_LST-1*{sequence.run}*.h5"))
-
+        try:
+            directory = destination_dir(concept="DL2", create_dir=False, dl2_prod_id=sequence.dl2_prod_id)
+            files = list(directory.glob(f"dl2_LST-1*{sequence.run}*.h5"))
+        except AttributeError:
+            return 0
+        
     elif data_level == "DATACHECK":
-        directory = options.directory / options.dl1_prod_id
-        files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
-
+        try:
+            directory = options.directory / sequence.dl1_prod_id
+            files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+        except AttributeError:
+            return 0
+        
     else:
         prefix = cfg.get("PATTERN", f"{data_level}PREFIX")
         suffix = cfg.get("PATTERN", f"{data_level}SUFFIX")
@@ -340,20 +354,14 @@ def is_sequencer_completed(date: datetime.datetime) -> bool:
     """Check if the jobs launched by sequencer are already completed."""
     summary_table = run_summary_table(date)
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
-    sacct_output = run_sacct()
-    sacct_info = get_sacct_output(sacct_output)
-
-    for run in data_runs["run_id"]:
-        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
-        if len(jobs_run["JobID"].unique())>1:
-            last_job_id = sorted(jobs_run["JobID"].unique())[-1]
-            jobs_run = sacct_info[sacct_info["JobID"]==last_job_id]
-        incomplete_jobs = jobs_run[(jobs_run["State"] != "COMPLETED")]
-        if len(jobs_run) == 0 or len(incomplete_jobs) != 0:
-            return False
-
-    return True
+    run_list = extract_runs(data_runs)
+    sequence_list = extract_sequences(options.date, run_list)
 
+    if are_all_jobs_correctly_finished(sequence_list):
+        return True
+    else:
+        log.info("Jobs did not correctly/yet finish")
+        return False
 
 def timeout_in_sequencer(date: datetime.datetime) -> bool:
     """Check if any of the jobs launched by sequencer finished in timeout."""

osa/scripts/sequencer_catB_tailcuts.py

@@ -0,0 +1,223 @@
+import glob
+import re
+import argparse
+import logging
+from pathlib import Path
+from astropy.table import Table
+import subprocess as sp
+
+from osa.configs import options
+from osa.configs.config import cfg
+from osa.nightsummary.extract import get_last_pedcalib
+from osa.utils.cliopts import valid_date, set_default_date_if_needed
+from osa.utils.logging import myLogger
+from osa.job import run_sacct, get_sacct_output
+from osa.utils.utils import date_to_dir, get_calib_filters
+from osa.paths import catB_closed_file_exists, catB_calibration_file_exists, analysis_path
+
+log = myLogger(logging.getLogger())
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "-c",                                                                                            
+    "--config",                                                                                      
+    action="store",
+    type=Path,
+    help="Configuration file",
+)
+parser.add_argument(                                                                                     
+    "-d",                                                                                            
+    "--date",
+    type=valid_date,
+    default=None,
+)
+parser.add_argument(
+    "-v",                                                                                      
+    "--verbose",
+    action="store_true",
+    default=False,
+    help="Activate debugging mode.",
+)
+parser.add_argument(
+    "-s",
+    "--simulate",
+    action="store_true",
+    default=False,
+    help="Simulate launching of the sequencer_catB_tailcuts script.",
+)
+parser.add_argument(
+    "tel_id",
+    choices=["ST", "LST1", "LST2", "all"],
+    help="telescope identifier LST1, LST2, ST or all.",
+)
+
+def are_all_history_files_created(run_id: int) -> bool:
+    """Check if all the history files (one per subrun) were created for a given run."""
+    run_summary_dir = Path(cfg.get(options.tel_id, "RUN_SUMMARY_DIR"))
+    run_summary_file = run_summary_dir / f"RunSummary_{date_to_dir(options.date)}.ecsv"
+    run_summary = Table.read(run_summary_file)
+    n_subruns = run_summary[run_summary["run_id"] == run_id]["n_subruns"]
+    analysis_dir = Path(options.directory)
+    history_files = glob.glob(f"{str(analysis_dir)}/sequence_LST1_{run_id:05d}.????.history")
+    if len(history_files) == n_subruns:
+        return True
+    else:
+        return False
+
+
+def r0_to_dl1_step_finished_for_run(run_id: int) -> bool:
+    """
+    Check if the step r0_to_dl1 finished successfully 
+    for a given run by looking the history files.
+    """
+    if not are_all_history_files_created(run_id):
+        log.debug(f"All history files for run {run_id:05d} were not created yet.")
+        return False
+    analysis_dir = Path(options.directory)
+    history_files = glob.glob(f"{str(analysis_dir)}/sequence_LST1_{run_id:05d}.????.history")
+    for file in history_files:
+        rc = Path(file).read_text().splitlines()[-1][-1]
+        if rc != "0":
+            print(f"r0_to_dl1 step did not finish successfully (check file {file})")
+            return False
+    return True
+
+
+def get_catB_last_job_id(run_id: int) -> int:
+    """Get job id of the last Cat-B calibration job that was launched for a given run."""
+    log_dir = Path(options.directory) / "log"
+    filenames = glob.glob(f"{log_dir}/catB_calibration_{run_id:05d}_*.err")
+    if filenames:
+        match = re.search(f"catB_calibration_{run_id:05d}_(\d+).err", sorted(filenames)[-1])
+        job_id = match.group(1)
+        return job_id
+
+
+def launch_catB_calibration(run_id: int):
+    """
+    Launch the Cat-B calibration script for a given run if the Cat-B calibration 
+    file has not been created yet. If the Cat-B calibration script was launched
+    before and it finished successfully, it creates a catB_{run}.closed file.
+    """
+    job_id = get_catB_last_job_id(run_id)
+    if job_id:
+        job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+        if job_status.item() in ["RUNNING", "PENDING"]:
+            log.debug(f"Job {job_id} (corresponding to run {run_id:05d}) is still running.")
+
+        elif job_status.item() == "COMPLETED":
+            catB_closed_file = Path(options.directory) / f"catB_{run_id:05d}.closed"
+            catB_closed_file.touch()
+            log.debug(
+                f"Cat-B job {job_id} (corresponding to run {run_id:05d}) finished "
+                f"successfully. Creating file {catB_closed_file}"
+            )
+
+        else: 
+            log.warning(f"Cat-B job {job_id} (corresponding to run {run_id:05d}) failed.")
+
+    else:
+        if catB_calibration_file_exists(run_id):
+            log.info(f"Cat-B calibration file already produced for run {run_id:05d}.")
+            return 
+
+        command = cfg.get("lstchain", "catB_calibration")
+        options.filters = get_calib_filters(run_id) 
+        base_dir = Path(cfg.get(options.tel_id, "BASE")).resolve()
+        r0_dir = Path(cfg.get(options.tel_id, "R0_DIR")).resolve()
+        interleaved_dir = Path(options.directory) / "interleaved"
+        log_dir = Path(options.directory) / "log"
+        catA_calib_run = get_last_pedcalib(options.date)
+        slurm_account = cfg.get("SLURM", "ACCOUNT")
+        cmd = ["sbatch", f"--account={slurm_account}", "--parsable",
+            "-o", f"{log_dir}/catB_calibration_{run_id:05d}_%j.out",
+            "-e", f"{log_dir}/catB_calibration_{run_id:05d}_%j.err",
+            command,
+            f"--run_number={run_id}",
+            f"--catA_calibration_run={catA_calib_run}",
+            f"--base_dir={base_dir}",
+            f"--r0-dir={r0_dir}",
+            f"--interleaved-dir={interleaved_dir}",
+            f"--filters={options.filters}",
+        ]
+        if not options.simulate:
+            job = sp.run(cmd, encoding="utf-8", capture_output=True, text=True, check=True)
+            job_id = job.stdout.strip()
+            log.debug(f"Launched Cat-B calibration job {job_id} for run {run_id}!")
+
+        else: 
+            log.info(f"Simulate launching of the {command} script.")
+            
+
+def launch_tailcuts_finder(run_id: int):
+    """
+    Launch the lstchain script to calculate the correct
+    tailcuts to use for a given run. 
+    """
+    command = cfg.get("lstchain", "tailcuts_finder")
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
+    input_dir = Path(options.directory)
+    output_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+    log_dir = Path(options.directory) / "log"
+    log_file = log_dir / f"tailcuts_finder_{run_id:05d}_%j.log"
+    cmd = [
+        "sbatch", "--parsable",
+        f"--account={slurm_account}",
+        "-o", log_file,
+        command,
+        f"--input-dir={input_dir}",
+        f"--run={run_id}",
+        f"--output-dir={output_dir}",
+    ]
+    if not options.simulate:
+        job = sp.run(cmd, encoding="utf-8", capture_output=True, text=True, check=True)
+        job_id = job.stdout.strip()
+        log.debug(f"Launched lstchain_find_tailcuts job {job_id} for run {run_id}!")
+
+    else: 
+        log.info(f"Simulate launching of the {command} script.")
+
+
+
+def tailcuts_config_file_exists(run_id: int) -> bool:
+    """Check if the config file created by the tailcuts finder script already exists."""
+    tailcuts_config_file = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR")) / f"dl1ab_Run{run_id:05d}.json"
+    return tailcuts_config_file.exists()
+    
+        
+def main():
+    """
+    Main script to be called as cron job. It launches the Cat-B calibration script 
+    and the tailcuts finder script for each run of the corresponding date, and creates
+    the catB_{run}.closed files if Cat-B calibration has finished successfully.
+    """ 
+    opts = parser.parse_args()
+    options.tel_id = opts.tel_id
+    options.simulate = opts.simulate
+    options.date = opts.date
+    options.date = set_default_date_if_needed()
+    options.configfile = opts.config.resolve()
+    options.directory = analysis_path(options.tel_id)
+
+    if opts.verbose:
+        log.setLevel(logging.DEBUG)
+    else:
+        log.setLevel(logging.INFO)
+
+    run_summary_dir = Path(cfg.get(options.tel_id, "RUN_SUMMARY_DIR"))
+    run_summary = Table.read(run_summary_dir / f"RunSummary_{date_to_dir(options.date)}.ecsv")
+    data_runs = run_summary[run_summary["run_type"]=="DATA"]
+    for run_id in data_runs["run_id"]:
+        # first check if the dl1a files are produced
+        if not r0_to_dl1_step_finished_for_run(run_id):
+            log.info(f"The r0_to_dl1 step did not finish yet for run {run_id:05d}. Please try again later.")
+        else:
+            # launch catB calibration and tailcut finder in parallel
+            if cfg.getboolean("lstchain", "apply_catB_calibration") and not catB_closed_file_exists(run_id):
+                launch_catB_calibration(run_id)
+            if not cfg.getboolean("lstchain", "apply_standard_dl1b_config") and not tailcuts_config_file_exists(run_id):
+                launch_tailcuts_finder(run_id)
+
+
+if __name__ == "__main__":
+    main()

osa/scripts/sequencer_webmaker.py

@@ -16,6 +16,7 @@ from osa.configs.config import cfg
 from osa.utils.cliopts import sequencer_webmaker_argparser
 from osa.utils.logging import myLogger
 from osa.utils.utils import is_day_closed, date_to_iso, date_to_dir
+from osa.paths import all_dl1ab_config_files_exist
 
 log = myLogger(logging.getLogger())
 
@@ -88,6 +89,9 @@ def get_sequencer_output(date: str, config: str, test=False, no_gainsel=False) -
     if test:
         commandargs.insert(-1, "-t")
 
+    if not all_dl1ab_config_files_exist(date):
+        commandargs.insert(-1, "--no-dl1ab")
+
     try:
         output = sp.run(commandargs, stdout=sp.PIPE, stderr=sp.STDOUT, encoding="utf-8", check=True)
     except sp.CalledProcessError as error:

osa/scripts/simulate_processing.py

@@ -8,7 +8,6 @@ python osa/scripts/simulate_processing.py"""
 import logging
 import multiprocessing as mp
 import subprocess
-from datetime import datetime
 from pathlib import Path
 
 import yaml
@@ -20,7 +19,8 @@ from osa.nightsummary.extract import build_sequences
 from osa.provenance.utils import get_log_config
 from osa.utils.cliopts import simprocparsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir
+from osa.utils.utils import date_to_iso
+from osa.paths import analysis_path
 
 __all__ = [
     "parse_template",
@@ -174,7 +174,7 @@ def simulate_processing():
                 drs4_pedestal_run_id,
                 pedcal_run_id,
                 sequence.run_str,
-                options.directory,
+                date_to_iso(options.date),
                 options.prod_id,
             ]
             log.info(f"Processing provenance for run {sequence.run_str}")
@@ -187,10 +187,7 @@ def main():
 
     simprocparsing()
 
-    # date and tel_id hardcoded for the moment
-    options.date = datetime.fromisoformat("2020-01-17")
-    options.tel_id = "LST1"
-    options.directory = date_to_dir(options.date)
+    options.directory = analysis_path(options.tel_id)
 
     log.info("Running simulate processing")