lstosa 0.10.14__py3-none-any.whl → 0.10.16__py3-none-any.whl

osa/scripts/gain_selection.py

@@ -1,24 +1,28 @@
 """Script to run the gain selection over a list of dates."""
 import logging
 import re
-import shutil
 import glob
 import pandas as pd
 import subprocess as sp
 from pathlib import Path
 from textwrap import dedent
-from io import StringIO
 import argparse
+import sys
 
 from astropy.table import Table
-from lstchain.paths import run_info_from_filename, parse_r0_filename
+from lstchain.paths import parse_r0_filename
+from datetime import datetime
 
 from osa.scripts.reprocessing import get_list_of_dates, check_job_status_and_wait
-from osa.utils.utils import wait_for_daytime
+from osa.utils.utils import wait_for_daytime, date_to_dir, date_to_iso
 from osa.utils.logging import myLogger
-from osa.job import get_sacct_output, FORMAT_SLURM
+from osa.utils.iofile import append_to_file
+from osa.utils.cliopts import valid_date
+from osa.job import get_sacct_output, run_sacct, job_finished_in_timeout
 from osa.configs.config import cfg
 from osa.paths import DEFAULT_CFG
+from osa.nightsummary.nightsummary import run_summary_table
+
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -49,23 +53,16 @@ parser.add_argument(
         "-d",                                                                                            
         "--date",                                                                                        
         default=None,
-        type=str,
-        help="Night to apply the gain selection in YYYYMMDD format",
+        type=valid_date,
+        help="Night to apply the gain selection in YYYY-MM-DD format",
 )                                                                                                        
 parser.add_argument(                                                                                     
         "-l",                                                                                            
         "--dates-file",
         default=None,
         help="List of dates to apply the gain selection. The input file should list"
-        "the dates in the format YYYYMMDD, one date per line.",
-)
-parser.add_argument(                                                                                     
-        "-o",                                                                                            
-        "--output-basedir",
-        type=Path,
-        default=Path("/fefs/aswg/data/real/R0G"),
-        help="Output directory of the gain selected files. Default is /fefs/aswg/data/real/R0G."
-)                                                                                                        
+        "the dates in the format YYYY-MM-DD, one date per line.",
+)                                                                                                       
 parser.add_argument(                                                                                     
         "-s",                                                                                            
         "--start-time",
@@ -86,67 +83,205 @@ parser.add_argument(
         type=str,
         default=None,
         help="Choose tool to apply the gain selection regardless the date. Possible options are: lst_dvr (by default used for dates "
-        "previous to 20231205) and lstchain_r0_to_r0g (by default used for dates later than 20231205).",
+        "previous to 2023-12-05) and lstchain_r0_to_r0g (by default used for dates later than 2023-12-05).",
+)
+parser.add_argument(                                                                                          
+        "--simulate",
+        action="store_true",
+        default=False,
+        help="Simulate launching of the gain selection script. Dry run.",
+)
+parser.add_argument(
+        "-v",                                                                                      
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Activate debugging mode.",
 )
 
 def get_sbatch_script(
-    run_id, subrun, input_file, output_dir, log_dir, log_file, ref_time, ref_counter, module, ref_source, tool
+    run_id: str,
+    subrun: str,
+    input_file: Path,
+    output_dir: Path,
+    log_dir: Path,
+    log_file: Path,
+    ref_time: int,
+    ref_counter: int,
+    module: int,
+    ref_source: str,
+    tool: str
 ):
     """Build the sbatch job pilot script for running the gain selection."""
-    if tool == "lst_dvr":
-        return dedent(
+    mem_per_job = cfg.get("SLURM", "MEMSIZE_GAINSEL")
+    sbatch_script = dedent(
             f"""\
         #!/bin/bash
 
         #SBATCH -D {log_dir}
         #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
         #SBATCH --job-name "gain_selection_{run_id:05d}"
-        #SBATCH --export {PATH}
         #SBATCH --partition=short,long
+        #SBATCH --mem={mem_per_job}
+        """
+        )
+    
+    if tool == "lst_dvr":
+        sbatch_script += dedent(
+            f"""   
+        #SBATCH --export {PATH}
 
         lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
         """
         )
+        
     elif tool == "lstchain_r0_to_r0g":
-        return dedent(
-            f"""\
-        #!/bin/bash
+        cmd = f"lstchain_r0_to_r0g --R0-file={input_file} --output-dir={output_dir} --log={log_file}"
+        if not cfg.getboolean("lstchain", "use_ff_heuristic_gain_selection"): 
+            cmd += " --no-flatfield-heuristic"
+        sbatch_script += dedent(cmd)
 
-        #SBATCH -D {log_dir}
-        #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
-        #SBATCH --job-name "gain_selection_{run_id:05d}"
-        #SBATCH --mem=40GB
-        #SBATCH --partition=short,long
+    return sbatch_script
 
-        lstchain_r0_to_r0g --R0-file={input_file} --output-dir={output_dir} --log={log_file} --no-flatfield-heuristic
-        """
-        )
 
-def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None, tool: str = None, no_queue_check: bool = False):
+def launch_gainsel_for_data_run(
+    date: datetime, run: Table, output_dir: Path, r0_dir: Path, log_dir: Path, tool: str, simulate: bool = False
+    ):
+    """
+    Create the gain selection sbatch script and launch it for a given run.
+    
+    Runs from before 20231205 without UCTS or TIB info are directly copied to the final directory.
+    Subruns that do not have four streams are also directly copied.
+    """
+    run_id = run["run_id"]
+    ref_time = run["dragon_reference_time"]
+    ref_counter = run["dragon_reference_counter"]
+    module = run["dragon_reference_module_index"]
+    ref_source = run["dragon_reference_source"].upper()
+
+    files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    subrun_numbers = [int(file[-12:-8]) for file in files]
+    
+    if tool == "lst_dvr" and ref_source not in ["UCTS", "TIB"]:
+        input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+        
+        if is_run_already_copied(date, run_id):
+            log.info(f"The R0 files corresponding to run {run_id} have already been copied to the R0G directory.")
+        else:
+            if not simulate:
+                for file in input_files:
+                    log.debug(
+                        f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
+                        f"be applied. Copying directly the R0 files to {output_dir}."
+                    )
+                    sp.run(["cp", file, output_dir])
+
+            else:
+                log.info(
+                    f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
+                    f"be applied. Simulate copy of the R0 files directly to {output_dir}."
+                )
+
+    else:
+        n_subruns = max(subrun_numbers) 
+
+        for subrun in range(n_subruns + 1):
+
+            r0_files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.{subrun:04d}.fits.fz")
+
+            if len(r0_files) != 4:
+                if not simulate and not is_run_already_copied(date, run_id):
+                    log.debug(f"Run {run_id:05d}.{subrun:04d} does not have 4 streams of R0 files, so gain"
+                        f"selection cannot be applied. Copying directly the R0 files to {output_dir}.")
+                    for file in r0_files:
+                        sp.run(["cp", file, output_dir])
+                elif is_run_already_copied(date, run_id):
+                    log.debug(f"Run {run_id:05d}.{subrun:04d} does not have 4 streams of R0 files. The R0 files"
+                        f"have already been copied to {output_dir}.")
+                elif simulate:
+                    log.debug(f"Run {run_id:05d}.{subrun:04d} does not have 4 streams of R0 files, so gain"
+                        f"selection cannot be applied. Simulate copy of the R0 files directly to {output_dir}.")
+
+            else:
+                history_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.history"
+                if history_file.exists():
+                    if not simulate:
+                        update_history_file(run_id, subrun, log_dir, history_file)
+
+                    if history_file.read_text() == "":   # history_file is empty
+                        log.debug(f"Gain selection is still running for run {run_id:05d}.{subrun:04d}")
+                        continue
+                    else:
+                        gainsel_rc = history_file.read_text().splitlines()[-1][-1]
+                        if gainsel_rc == "1":
+                            job_id = get_last_job_id(run_id, subrun, log_dir)
+                            if job_finished_in_timeout(job_id) and not simulate:
+                                # Relaunch the job that finished in TIMEOUT
+                                job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"
+                                sp.run(["sbatch", job_file], stdout=sp.PIPE, stderr=sp.STDOUT, check=True)
+                            else:
+                                log.warning(f"Gain selection failed for run {run_id:05d}.{subrun:04d}")
+                        elif gainsel_rc == "0":
+                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d},"
+                                        "no additional jobs will be submitted for this subrun.") 
+                else:
+                    log.debug("Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
+                    if not simulate:
+                        log_file = log_dir / f"r0_to_r0g_{run_id:05d}.{subrun:04d}.log"
+                        job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"
+                        r0_files.sort()
+                        with open(job_file, "w") as f:
+                            f.write(
+                                get_sbatch_script(
+                                    run_id,
+                                    subrun,
+                                    r0_files[0],
+                                    output_dir,
+                                    log_dir,
+                                    log_file,
+                                    ref_time,
+                                    ref_counter,
+                                    module,
+                                    ref_source,
+                                    tool,
+                                )
+                            )
+
+                        #submit job
+                        history_file.touch()
+                        sp.run(["sbatch", job_file], stdout=sp.PIPE, stderr=sp.STDOUT, check=True)
+
+
+def apply_gain_selection(date: datetime, start: int, end: int, tool: str = None, no_queue_check: bool = False, simulate: bool = False):
     """
     Submit the jobs to apply the gain selection to the data for a given date
     on a subrun-by-subrun basis.
     """
 
     if not tool:
-        if date < "20231205":
+        if date_to_dir(date) < "20231205":
             tool = "lst_dvr"
         else:
             tool = "lstchain_r0_to_r0g"
 
-    run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
-    run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
-    summary_table = Table.read(run_summary_file)
+    summary_table = run_summary_table(date)
+
+    if len(summary_table) == 0:
+        log.warning(f"No runs are found in the run summary of {date_to_iso(date)}. Nothing to do. Exiting.")
+        sys.exit(0)
+
     # Apply gain selection only to DATA runs
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
     log.info(f"Found {len(data_runs)} DATA runs to which apply the gain selection")
 
-    output_dir = output_basedir / date
-    log_dir = output_basedir / "log" / date
-    output_dir.mkdir(parents=True, exist_ok=True)
-    log_dir.mkdir(parents=True, exist_ok=True)
-    log_file = log_dir / f"r0_to_r0g_{date}.log"
-    r0_dir = Path(f"/fefs/aswg/data/real/R0/{date}")
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    date_str = date_to_dir(date)
+    r0_dir = base_dir / "R0" / date_str
+    output_dir = base_dir / f"R0G/{date_str}"
+    log_dir = base_dir / f"R0G/log/{date_str}"
+    if not simulate:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        log_dir.mkdir(parents=True, exist_ok=True)
 
     for run in data_runs:
         if not no_queue_check:
@@ -156,171 +291,210 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
         # Avoid running jobs while it is still night time
         wait_for_daytime(start, end)
 
-        run_id = run["run_id"]
-        ref_time = run["dragon_reference_time"]
-        ref_counter = run["dragon_reference_counter"]
-        module = run["dragon_reference_module_index"]
-        ref_source = run["dragon_reference_source"].upper()
-
-        files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.????.fits.fz")
-        subrun_numbers = [int(file[-12:-8]) for file in files]
-        input_files = []
-
-        if tool == "lst_dvr" and ref_source not in ["UCTS", "TIB"]:
-            input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
-            log.info(
-                f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
-                f"be applied. Copying directly the R0 files to {output_dir}."
-            )
-            for file in input_files:
-                sp.run(["cp", file, output_dir])
+        if not is_closed(date, run["run_id"]):
+            launch_gainsel_for_data_run(date, run, output_dir, r0_dir, log_dir, tool, simulate)
+
+    calib_runs = summary_table[summary_table["run_type"] != "DATA"]
+    log.info(f"Found {len(calib_runs)} NO-DATA runs")
 
+    for run in calib_runs:
+        run_id = run["run_id"]
+        
+        if is_run_already_copied(date, run_id):
+            log.info(f"The R0 files corresponding to run {run_id:05d} have already been copied, nothing to do.")
         else:
-            n_subruns = max(subrun_numbers)
+            log.info(f"Copying R0 files corresponding to run {run_id} directly to {output_dir}")
+       	    if not simulate:
+            	# Avoid copying files while it is still night time
+            	wait_for_daytime(start, end)
 
-            for subrun in range(n_subruns + 1):
-                new_files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.{subrun:04d}.fits.fz")
+            	r0_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
 
-                if len(new_files) != 4:
-                    log.info(f"Run {run_id}.{subrun:05d} does not have 4 streams of R0 files, so gain"
-                        f"selection cannot be applied. Copying directly the R0 files to {output_dir}."
-                    )
-                    for file in new_files:
-                        sp.run(["cp", file, output_dir])
+            	for file in r0_files:
+                    sp.run(["cp", file, output_dir])
 
-                else:
-                    new_files.sort()
-                    input_files.append(new_files[0])
-
-            log.info("Creating and launching the sbatch scripts for the rest of the runs to apply gain selection")
-            for file in input_files:
-                run_info = run_info_from_filename(file)
-                job_file = log_dir / f"gain_selection_{run_info.run:05d}.{run_info.subrun:04d}.sh"
-                with open(job_file, "w") as f:
-                    f.write(
-                        get_sbatch_script(
-                            run_id,
-                            run_info.subrun,
-                            file,
-                            output_dir,
-                            log_dir,
-                            log_file,
-                            ref_time,
-                            ref_counter,
-                            module,
-                            ref_source,
-                            tool,
-                        )
-                    )
-                sp.run(["sbatch", job_file], check=True)
 
-    calib_runs = summary_table[summary_table["run_type"] != "DATA"]
-    log.info(f"Found {len(calib_runs)} NO-DATA runs")
+def get_last_job_id(run_id: str, subrun: str, log_dir: Path) -> int:
+    """Get job id of the last gain selection job that was launched for a given subrun."""
+    filenames = glob.glob(f"{log_dir}/gain_selection_{run_id:05d}_{subrun:04d}_*.log")
+    if filenames:
+        match = re.search(f'gain_selection_{run_id:05d}_{subrun:04d}_(\d+).log', sorted(filenames)[-1])
+        job_id = match.group(1)
+        return job_id
 
-    for run in calib_runs:
-        run_id = run["run_id"]
-        log.info(f"Copying R0 files corresponding to run {run_id} directly to {output_dir}")
-        # Avoid copying files while it is still night time
-        wait_for_daytime(start, end)
 
-        run_id = run["run_id"]
-        r0_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+def update_history_file(run_id: str, subrun: str, log_dir: Path, history_file: Path):
+    """
+    Update the gain selection history file with the result 
+    of the last job launched for a given subrun.
+    """
+    job_id = get_last_job_id(run_id, subrun, log_dir)
+    if not job_id:
+        log.debug(f"Cannot find a job_id for the run {run_id:05d}.{subrun:04d}")
+    else:
+        job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+        if job_status.item() in ["RUNNING", "PENDING"]:
+            log.info(f"Job {job_id} is still running.")
+            return
+            
+        elif job_status.item() == "COMPLETED":
+            log.debug(f"Job {job_id} finished successfully, updating history file.")
+            string_to_write = (
+                f"{run_id:05d}.{subrun:04d} gain_selection 0\n"
+            )
+            append_to_file(history_file, string_to_write)
+        
+        else:
+            log.info(f"Job {job_id} failed, updating history file.")
+            string_to_write = (
+                f"{run_id:05d}.{subrun:04d} gain_selection 1\n"
+            )
+            append_to_file(history_file, string_to_write)
 
-        for file in r0_files:
-            sp.run(["cp", file, output_dir])
 
-def run_sacct_j(job) -> StringIO:
-    """Run sacct to obtain the job information."""
-    if shutil.which("sacct") is None:
-        log.warning("No job info available since sacct command is not available")
-        return StringIO()
+def is_run_already_copied(date: datetime, run_id: int) -> bool:
+    """Check if the R0 files of a given run have already been copied to the R0G directory."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    r0_files = glob.glob(f"{base_dir}/R0/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    r0g_files = glob.glob(f"{base_dir}/R0G/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    return len(r0_files)==len(r0g_files)
 
-    sacct_cmd = [
-        "sacct",
-        "-n",
-        "--parsable2",
-        "--delimiter=,",
-        "--units=G",
-        "-o",
-        ",".join(FORMAT_SLURM),
-        "-j",
-        job,
-    ]
 
-    return StringIO(sp.check_output(sacct_cmd).decode())
+def is_closed(date: datetime, run_id: str) -> bool:
+    """Check if run is already closed."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
+    closed_run_file = log_dir / f"gain_selection_{run_id:05d}.closed"
+    return closed_run_file.exists()
 
 
-def GainSel_flag_file(date: str) -> Path:
+def GainSel_flag_file(date: datetime) -> Path:
+    """Return the path to the file indicating the completion of the gain selection stage."""
     filename = cfg.get("LSTOSA", "gain_selection_check")
     GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
-    flagfile = GainSel_dir / date / filename
+    flagfile = GainSel_dir / date_to_dir(date) / filename
     return flagfile.resolve()
 
 
-def GainSel_finished(date: str) -> bool:
+def GainSel_finished(date: datetime) -> bool:
     """Check if gain selection finished successfully."""
     flagfile = GainSel_flag_file(date)
     return flagfile.exists()
+   
 
-
-def check_failed_jobs(date: str, output_basedir: Path = None):
+def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool:
     """Search for failed jobs in the log directory."""
-    failed_jobs = []
-    log_dir = output_basedir / "log" / date
-    filenames = glob.glob(f"{log_dir}/gain_selection*.log")
-    jobs = [re.search(r'(?<=_)(.[0-9.]+?)(?=.log)', i).group(0) for i in filenames]
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
+    history_files = list(log_dir.glob(f"gain_selection_{run_id:05d}.????.history"))
+    summary_table = run_summary_table(date)
+    n_subruns = summary_table[summary_table["run_id"] == run_id]["n_subruns"]
+    
+    if len(history_files) != n_subruns:
+        log.debug(f"All history files of run {run_id} were not created yet")
+        return False
+
+    failed_subruns = []
+    log.info(f"Checking all history files of run {run_id}")
+    
+    for file in history_files:
+        match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file))
+        subrun = match.group(1)
+        if file.read_text() != "":
+            gainsel_rc = file.read_text().splitlines()[-1][-1]
+
+            if gainsel_rc == "1":
+                log.warning(f"Gain selection failed for run {run_id}.{subrun}")
+                failed_subruns.append(file)
+
+            elif gainsel_rc == "0":
+                log.debug(f"Gain selection finished successfully for run {run_id}.{subrun}")
+        else:
+            log.info(f"Gain selection is still running for run {run_id}.{subrun}")
+            return False
+            
+    if failed_subruns:
+        log.warning(f"{date_to_iso(date)}: Some gain selection jobs did not finish successfully for run {run_id}")
+        return False
+    else:
+        log.info(f"{date_to_iso(date)}: All jobs finished successfully for run {run_id}, creating the corresponding .closed file")
+        closed_run_file = log_dir / f"gain_selection_{run_id:05d}.closed"
+        closed_run_file.touch()
+        return True
 
-    for job in jobs:
-        output = run_sacct_j(job)
-        df = get_sacct_output(output)
 
-        if not df.iloc[0]["State"] == "COMPLETED":
-            log.warning(f"Job {job} did not finish successfully")
-            failed_jobs.append(job)
+def check_warnings_in_logs(date: datetime, run_id: int):
+    """Look for warnings in the log files created by lstchain_r0_to_r0g."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
+    log_files = log_dir.glob(f"r0_to_r0g_{run_id:05d}.*.log")
+    for file in log_files:
+        content = file.read_text().splitlines()
+        for line in content:
+            if "FlatField(FF)-like events are not tagged as FF" in line:
+                log.warning(f"Warning for run {run_id}: {line}")
 
-    if failed_jobs:
-        log.warning(f"{date}: some jobs did not finish successfully")
 
-    else:
-        log.info(f"{date}: all jobs finished successfully")
-
-
-        run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
-        run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
-        summary_table = Table.read(run_summary_file)
-        runs = summary_table["run_id"]
-        missing_runs = []
-
-        r0_files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run?????.????.fits.fz")
-        r0g_files = glob.glob(f"/fefs/aswg/data/real/R0G/{date}/LST-1.?.Run?????.????.fits.fz")
-        all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
-        all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
-
-        for run in all_r0_runs:
-            if run not in runs:
-                if run not in all_r0g_runs:
-                    missing_runs.append(run)
-
-        missing_runs.sort()
-        if missing_runs:
-            log.info(
-                f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
-                f"directly to /fefs/aswg/data/real/R0G/{date}"
-            )
+def check_failed_jobs(date: datetime):
+    """Search for failed jobs in the log directory."""
 
-            for run in missing_runs:
-                output_dir = Path(f"/fefs/aswg/data/real/R0G/{date}/")
-                files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run{run:05d}.????.fits.fz")
-                for file in files:
-                    sp.run(["cp", file, output_dir])
+    summary_table = run_summary_table(date)
 
-        GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
-        flagfile_dir = GainSel_dir / date
-        flagfile_dir.mkdir(parents=True, exist_ok=True)
+    if len(summary_table) == 0:
+        log.warning(f"No runs are found in the run summary of {date_to_iso(date)}. Nothing to do. Exiting.")
+        sys.exit(0)
         
-        flagfile = GainSel_flag_file(date)
-        log.info(f"Gain selection finished successfully, creating flag file for date {date} ({flagfile})")
-        flagfile.touch()
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    failed_runs = []
+
+    for run in data_runs:
+        run_id = run["run_id"]
+        check_warnings_in_logs(date, run_id)
+        if not is_closed(date, run_id):
+            if not check_gainsel_jobs_runwise(date, run_id):
+                log.warning(f"Gain selection did not finish successfully for run {run_id}.")
+                failed_runs.append(run)
+
+    if failed_runs:
+        log.warning(f"Gain selection did not finish successfully for {date_to_iso(date)}, cannot create the flag file.")
+        return
+
+    runs = summary_table["run_id"]
+    missing_runs = []
+
+    date_str = date_to_dir(date)
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    r0_files = glob.glob(f"{base_dir}/R0/{date_str}/LST-1.?.Run?????.????.fits.fz")
+    r0g_files = glob.glob(f"{base_dir}/R0G/{date_str}/LST-1.?.Run?????.????.fits.fz")
+    all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
+    all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
+
+    for run in all_r0_runs:
+        if run not in runs:
+            if run not in all_r0g_runs:
+                missing_runs.append(run)
+
+    missing_runs.sort()
+    if missing_runs:
+        output_dir = base_dir / f"R0G/{date_str}/"
+        log.info(
+            f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
+            f"directly to {output_dir}"
+        )
+
+        for run in missing_runs:
+            
+            files = base_dir.glob(f"R0/{date_str}/LST-1.?.Run{run:05d}.????.fits.fz")
+            for file in files:
+                sp.run(["cp", file, output_dir])
+
+    GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
+    flagfile_dir = GainSel_dir / date_str
+    flagfile_dir.mkdir(parents=True, exist_ok=True)
+    
+    flagfile = GainSel_flag_file(date)
+    log.info(f"Gain selection finished successfully, creating flag file for date {date_to_iso(date)} ({flagfile})")
+    flagfile.touch()
 
 
 def main():
@@ -329,22 +503,29 @@ def main():
     script for each of them. The input file should list the dates in the format
     YYYYMMDD one date per line.
     """
-    log.setLevel(logging.INFO)
     args = parser.parse_args()
+    
+    if args.verbose:
+        log.setLevel(logging.DEBUG)
+    else:
+        log.setLevel(logging.INFO)
 
     if args.date:
-        if args.check:
-            log.info(f"Checking gain selection status for date {args.date}")
-            check_failed_jobs(args.date, args.output_basedir)
+        if GainSel_finished(args.date):
+            log.warning(f"Gain selection already done for date {date_to_iso(args.date)}. Exiting.")
+            sys.exit(0)
+        elif args.check:
+            log.info(f"Checking gain selection status for date {date_to_iso(args.date)}")
+            check_failed_jobs(args.date)
         else:
-            log.info(f"Applying gain selection to date {args.date}")
+            log.info(f"\nApplying gain selection to date {date_to_iso(args.date)}")
             apply_gain_selection(
                 args.date, 
                 args.start_time, 
-                args.end_time, 
-                args.output_basedir,
+                args.end_time,
                 args.tool,
                 no_queue_check=args.no_queue_check, 
+                simulate=args.simulate,
             )
 
 
@@ -355,7 +536,7 @@ def main():
         if args.check:
             for date in list_of_dates:
                 log.info(f"Checking gain selection status for date {date}")
-                check_failed_jobs(date, args.output_basedir)
+                check_failed_jobs(date)
         else:
             for date in list_of_dates:
                 log.info(f"Applying gain selection to date {date}")
@@ -363,9 +544,9 @@ def main():
                     date, 
                     args.start_time, 
                     args.end_time,
-                    args.output_basedir,
                     args.tool,
                     no_queue_check=args.no_queue_check,
+                    simulate=args.simulate,
                 )
             log.info("Done! No more dates to process.")