lstosa 0.10.10__tar.gz → 0.10.12__tar.gz

{lstosa-0.10.10 → lstosa-0.10.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.10
+Version: 0.10.12
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.10 → lstosa-0.10.12}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.10
+Version: 0.10.12
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.10 → lstosa-0.10.12}/src/lstosa.egg-info/SOURCES.txt

@@ -95,7 +95,6 @@ src/osa/scripts/reprocessing.py
 src/osa/scripts/sequencer.py
 src/osa/scripts/sequencer_webmaker.py
 src/osa/scripts/show_run_summary.py
-src/osa/scripts/show_run_summary_tcu.py
 src/osa/scripts/simulate_processing.py
 src/osa/scripts/update_source_catalog.py
 src/osa/scripts/tests/__init__.py

{lstosa-0.10.10 → lstosa-0.10.12}/src/osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.10'
-__version_tuple__ = version_tuple = (0, 10, 10)
+__version__ = version = '0.10.12'
+__version_tuple__ = version_tuple = (0, 10, 12)

{lstosa-0.10.10 → lstosa-0.10.12}/src/osa/configs/sequencer.cfg

@@ -28,6 +28,7 @@ HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
 LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
+GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
@@ -138,6 +139,7 @@ WORKFLOWPREFIX: Workflow
 GRAPHSUFFIX: .dot
 SVGSUFFIX: .svg
 end_of_activity: NightFinished.txt
+gain_selection_check: GainSelFinished.txt
 
 [OUTPUT]
 # REPORTWIDTH is the width in characters of the heading frame for the output

{lstosa-0.10.10 → lstosa-0.10.12}/src/osa/conftest.py

@@ -48,6 +48,13 @@ def monitoring_dir(base_test_dir):
     return monitoring_dir
 
 
+@pytest.fixture(scope="session")
+def osa_dir(base_test_dir):
+    osa_dir = base_test_dir / "OSA"
+    osa_dir.mkdir(parents=True, exist_ok=True)
+    return osa_dir
+
+
 @pytest.fixture(scope="session")
 def run_summary_dir(monitoring_dir):
     summary_dir = monitoring_dir / "RunSummary"
@@ -436,6 +443,8 @@ def sequence_file_list(
     drs4_time_calibration_files,
     systematic_correction_files,
     r0_data,
+    gain_selection_flag_file,
+    merged_run_summary,
 ):
     for r0_file in r0_data:
         assert r0_file.exists()
@@ -448,6 +457,8 @@ def sequence_file_list(
 
     assert run_summary_file.exists()
     assert run_catalog.exists()
+    assert gain_selection_flag_file.exists()
+    assert merged_run_summary.exists()
 
     run_program("sequencer", "-d", "2020-01-17", "--no-submit", "-t", "LST1")
     # First sequence in the list corresponds to the calibration run 1809
@@ -548,11 +559,9 @@ def run_catalog(run_catalog_dir):
 
 
 @pytest.fixture(scope="session")
-def database(base_test_dir):
+def database(osa_dir):
     import sqlite3
 
-    osa_dir = base_test_dir / "OSA"
-    osa_dir.mkdir(parents=True, exist_ok=True)
     db_file = osa_dir / "osa.db"
     with sqlite3.connect(db_file) as connection:
         cursor = connection.cursor()
@@ -562,3 +571,13 @@ def database(base_test_dir):
         )
         cursor.connection.commit()
         yield cursor
+
+
+@pytest.fixture(scope="session")
+def gain_selection_flag_file(osa_dir):
+
+    GainSel_dir = osa_dir / "GainSel" / "20200117"
+    GainSel_dir.mkdir(parents=True, exist_ok=True)
+    file = GainSel_dir / "GainSelFinished.txt"
+    file.touch()
+    return file

{lstosa-0.10.10 → lstosa-0.10.12}/src/osa/paths.py

@@ -5,6 +5,8 @@ import re
 from datetime import datetime
 from pathlib import Path
 from typing import List
+import subprocess
+import time
 
 import lstchain
 from astropy.table import Table
@@ -359,15 +361,46 @@ def get_latest_version_file(longterm_files: List[str]) -> Path:
     )
 
 
-def create_longterm_symlink():
+def is_job_completed(job_id: str):
+    """
+    Check whether SLURM job `job_id` has finished.
+    
+    It keeps checking every 10 minutes for one our.
+    """
+    n_max = 10
+    n = 0
+    while n < n_max:
+        # Check if the status of the SLURM job is "COMPLETED"
+        status = subprocess.run(["sacct", "--format=state", "--jobs", job_id], capture_output=True, text=True)
+        if "COMPLETED" in status.stdout:
+            log.debug(f"Job {job_id} finished successfully!")
+            return True
+        n += 1
+        log.debug(f"Job {job_id} is not completed yet, checking again in 10 minutes...")
+        time.sleep(600)  # wait 10 minutes to check again
+    log.info(f"The maximum number of checks of job {job_id} was reached, job {job_id} did not finish succesfully yet.")
+    return False
+
+
+def create_longterm_symlink(cherenkov_job_id: str = None):
     """If the created longterm DL1 datacheck file corresponds to the latest 
     version available, make symlink to it in the "all" common directory."""
+    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
+        nightdir = utils.date_to_dir(options.date)
+        longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+        linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
+        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+        latest_version_file = get_latest_version_file(all_longterm_files)
+        log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
+        linked_longterm_file.unlink(missing_ok=True)
+        linked_longterm_file.symlink_to(latest_version_file)
+    else:
+        log.warning(f"Job {cherenkov_job_id} (lstchain_cherenkov_transparency) did not finish successfully.")
+
+def dl1_datacheck_longterm_file_exits() -> bool:
+    """Return true if the longterm DL1 datacheck file was already produced."""
     nightdir = utils.date_to_dir(options.date)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
-    linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
-    all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
-    latest_version_file = get_latest_version_file(all_longterm_files)
+    longterm_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5"
+    return longterm_file.exists()
 
-    log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
-    linked_longterm_file.unlink(missing_ok=True)
-    linked_longterm_file.symlink_to(latest_version_file)

{lstosa-0.10.10 → lstosa-0.10.12}/src/osa/scripts/autocloser.py

@@ -41,6 +41,7 @@ class Telescope:
         config_file: Path,
         ignore_cronlock: bool = False,
         test: bool = False,
+        no_gainsel: bool = False,
     ):
         """
         Parameters
@@ -82,7 +83,7 @@ class Telescope:
         if not self.lock_automatic_sequencer() and not ignore_cronlock:
             log.warning(f"{self.telescope} already locked! Ignoring {self.telescope}")
             return
-        if not self.simulate_sequencer(date, config_file, test):
+        if not self.simulate_sequencer(date, config_file, test, no_gainsel):
             log.warning(
                 f"Simulation of the sequencer failed "
                 f"for {self.telescope}! Ignoring {self.telescope}"
@@ -121,7 +122,7 @@ class Telescope:
         self.locked = True
         return True
 
-    def simulate_sequencer(self, date: str, config_file: Path, test: bool):
+    def simulate_sequencer(self, date: str, config_file: Path, test: bool, no_gainsel: bool):
         """Launch the sequencer in simulation mode."""
         if test:
             self.read_file()
@@ -135,6 +136,9 @@ class Telescope:
                 date,
                 self.telescope,
             ]
+            if no_gainsel:
+                sequencer_cmd.insert(1, "--no-gainsel")
+
             log.debug(f"Executing {' '.join(sequencer_cmd)}")
             sequencer = subprocess.Popen(
                 sequencer_cmd,
@@ -445,7 +449,7 @@ def main():
     # create telescope and sequence objects
     log.info("Simulating sequencer...")
 
-    telescope = Telescope(args.tel_id, date, args.config)
+    telescope = Telescope(args.tel_id, date, args.config, no_gainsel=args.no_gainsel)
 
     log.info(f"Processing {args.tel_id}...")
 

{lstosa-0.10.10 → lstosa-0.10.12}/src/osa/scripts/closer.py

@@ -24,7 +24,11 @@ from osa.job import (
 )
 from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import destination_dir, create_longterm_symlink
+from osa.paths import (
+    destination_dir,
+    create_longterm_symlink,
+    dl1_datacheck_longterm_file_exits
+)
 from osa.raw import is_raw_data_available
 from osa.report import start
 from osa.utils.cliopts import closercliparsing
@@ -154,38 +158,42 @@ def ask_for_closing():
 def post_process(seq_tuple):
     """Set of last instructions."""
     seq_list = seq_tuple[1]
+    
+    if dl1_datacheck_longterm_file_exits() and not options.test:
+        create_longterm_symlink()
 
-    # Close the sequences
-    post_process_files(seq_list)
+    else:
+        # Close the sequences
+        post_process_files(seq_list)
 
-    # Merge DL1 datacheck files and produce PDFs. It also produces
-    # the daily datacheck report using the longterm script, and updates
-    # the longterm DL1 datacheck file with the cherenkov_transparency script.
-    if cfg.getboolean("lstchain", "merge_dl1_datacheck"):
-        list_job_id = merge_dl1_datacheck(seq_list)
-        longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
-        cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
-        create_longterm_symlink()
+        # Extract the provenance info
+        extract_provenance(seq_list)
 
-    # Extract the provenance info
-    extract_provenance(seq_list)
+        # Merge DL1b files run-wise
+        merge_files(seq_list, data_level="DL1AB")
 
-    # Merge DL1b files run-wise
-    merge_files(seq_list, data_level="DL1AB")
+        merge_muon_files(seq_list)
 
-    merge_muon_files(seq_list)
+        # Merge DL2 files run-wise
+        if not options.no_dl2:
+            merge_files(seq_list, data_level="DL2")
 
-    # Merge DL2 files run-wise
-    if not options.no_dl2:
-        merge_files(seq_list, data_level="DL2")
+        # Merge DL1 datacheck files and produce PDFs. It also produces
+        # the daily datacheck report using the longterm script, and updates
+        # the longterm DL1 datacheck file with the cherenkov_transparency script.
+        if cfg.getboolean("lstchain", "merge_dl1_datacheck"):
+            list_job_id = merge_dl1_datacheck(seq_list)
+            longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
+            cherenkov_job_id = cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
+            create_longterm_symlink(cherenkov_job_id)
 
-    time.sleep(600)
+        time.sleep(600)
 
     # Check if all jobs launched by autocloser finished correctly 
     # before creating the NightFinished.txt file
     n_max = 6
     n = 0
-    while not all_closer_jobs_finished_correctly() & n <= n_max:
+    while not all_closer_jobs_finished_correctly() and n <= n_max:
         log.info(
             "All jobs launched by autocloser did not finished correctly yet. "
             "Checking again in 10 minutes..."
@@ -194,7 +202,7 @@ def post_process(seq_tuple):
         n += 1
 
     if n > n_max:
-        send_warning_mail(date=options.date)
+        send_warning_mail(date=date_to_iso(options.date))
         return False
 
     if options.seqtoclose is None:
@@ -536,6 +544,7 @@ def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
 
     return [
         "sbatch",
+        "--parsable",
         "-D",
         options.directory,
         "-o",
@@ -553,7 +562,16 @@ def cherenkov_transparency(cmd: List[str]):
     log.debug(f"Executing {stringify(cmd)}")
 
     if not options.simulate and not options.test and shutil.which("sbatch") is not None:
-        subprocess.run(cmd, check=True)
+        job = subprocess.run(
+            cmd,
+            encoding="utf-8",
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        job_id = job.stdout.strip()
+        return job_id
+
     else:
         log.debug("Simulate launching scripts")
 

lstosa-0.10.12/src/osa/scripts/gain_selection.py

@@ -0,0 +1,349 @@
+"""Script to run the gain selection over a list of dates."""
+import logging
+import re
+import shutil
+import glob
+import pandas as pd
+import subprocess as sp
+from pathlib import Path
+from textwrap import dedent
+from io import StringIO
+import argparse
+
+from astropy.table import Table
+from lstchain.paths import run_info_from_filename, parse_r0_filename
+
+from osa.scripts.reprocessing import get_list_of_dates, check_job_status_and_wait
+from osa.utils.utils import wait_for_daytime
+from osa.utils.logging import myLogger
+from osa.job import get_sacct_output, FORMAT_SLURM
+from osa.configs.config import cfg
+from osa.paths import DEFAULT_CFG
+
+log = myLogger(logging.getLogger(__name__))
+
+PATH = "PATH=/fefs/aswg/software/offline_dvr/bin:$PATH"
+
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument(
+        "--check",                                                                                       
+        action="store_true",
+        default=False,
+        help="Check if any job failed",
+)
+parser.add_argument(
+        "--no-queue-check",
+        action="store_true",
+        default=False,
+        help="Do not wait until the number of jobs in the slurm queue is < 1500",
+)
+parser.add_argument(
+        "-c",                                                                                            
+        "--config",                                                                                      
+        action="store",
+        type=Path,
+        default=DEFAULT_CFG,
+        help="Configuration file",
+)
+parser.add_argument(                                                                                     
+        "-d",                                                                                            
+        "--date",                                                                                        
+        default=None,
+        type=str,
+        help="Night to apply the gain selection",
+)                                                                                                        
+parser.add_argument(                                                                                     
+        "-l",                                                                                            
+        "--dates-file",
+        default=None,
+        help="List of dates to apply the gain selection",
+)
+parser.add_argument(                                                                                     
+        "-o",                                                                                            
+        "--output-basedir",
+        type=Path,
+        default=Path("/fefs/aswg/data/real/R0G"),
+        help="Output directory of the gain selected files. Default is /fefs/aswg/data/real/R0G."
+)                                                                                                        
+parser.add_argument(                                                                                     
+        "-s",                                                                                            
+        "--start-time",
+        type=int,
+        default=10,
+        help="Time to (re)start gain selection in HH format. Default is 10.",
+)                                                                                                       
+parser.add_argument(                                                                                     
+        "-e",                                                                                            
+        "--end-time",
+        type=int,
+        default=18,
+        help="Time to stop gain selection in HH format. Default is 18.",
+)
+
+def get_sbatch_script(
+    run_id, subrun, input_file, output_dir, log_dir, log_file, ref_time, ref_counter, module, ref_source, script
+):
+    """Build the sbatch job pilot script for running the gain selection."""
+    if script=="old":
+        return dedent(
+            f"""\
+        #!/bin/bash
+
+        #SBATCH -D {log_dir}
+        #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
+        #SBATCH --job-name "gain_selection_{run_id:05d}"
+        #SBATCH --export {PATH}
+        #SBATCH --partition=short,long
+
+        lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
+        """
+        )
+    elif script=="new":
+        return dedent(
+            f"""\
+        #!/bin/bash
+
+        #SBATCH -D {log_dir}
+        #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
+        #SBATCH --job-name "gain_selection_{run_id:05d}"
+        #SBATCH --mem=40GB
+        #SBATCH --partition=short,long
+
+        lstchain_r0_to_r0g --R0-file={input_file} --output-dir={output_dir} --log={log_file} --no-flatfield-heuristic
+        """
+        )
+
+def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None, no_queue_check: bool = False):
+    """
+    Submit the jobs to apply the gain selection to the data for a given date
+    on a subrun-by-subrun basis.
+    """
+
+    if date < "20231205":
+        script = "old"
+    else:
+        script = "new"
+
+    run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
+    run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
+    summary_table = Table.read(run_summary_file)
+    # Apply gain selection only to DATA runs
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    log.info(f"Found {len(data_runs)} DATA runs to which apply the gain selection")
+
+    output_dir = output_basedir / date
+    log_dir = output_basedir / "log" / date
+    output_dir.mkdir(parents=True, exist_ok=True)
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / f"r0_to_r0g_{date}.log"
+    r0_dir = Path(f"/fefs/aswg/data/real/R0/{date}")
+
+    for run in data_runs:
+        if not no_queue_check:
+            # Check slurm queue status and sleep for a while to avoid overwhelming the queue
+            check_job_status_and_wait(max_jobs=1500)
+
+        # Avoid running jobs while it is still night time
+        wait_for_daytime(start, end)
+
+        run_id = run["run_id"]
+        ref_time = run["dragon_reference_time"]
+        ref_counter = run["dragon_reference_counter"]
+        module = run["dragon_reference_module_index"]
+        ref_source = run["dragon_reference_source"].upper()
+
+        files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+        subrun_numbers = [int(file[-12:-8]) for file in files]
+        input_files = []
+
+        if date < "20231205" and ref_source not in ["UCTS", "TIB"]:
+            input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+            log.info(
+                f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
+                f"be applied. Copying directly the R0 files to {output_dir}."
+            )
+            for file in input_files:
+                sp.run(["cp", file, output_dir])
+
+        else:
+            n_subruns = max(subrun_numbers)
+
+            for subrun in range(n_subruns + 1):
+                new_files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.{subrun:04d}.fits.fz")
+
+                if len(new_files) != 4:
+                    log.info(f"Run {run_id}.{subrun:05d} does not have 4 streams of R0 files, so gain"
+                        f"selection cannot be applied. Copying directly the R0 files to {output_dir}."
+                    )
+                    for file in new_files:
+                        sp.run(["cp", file, output_dir])
+
+                else:
+                    new_files.sort()
+                    input_files.append(new_files[0])
+
+            log.info("Creating and launching the sbatch scripts for the rest of the runs to apply gain selection")
+            for file in input_files:
+                run_info = run_info_from_filename(file)
+                job_file = log_dir / f"gain_selection_{run_info.run:05d}.{run_info.subrun:04d}.sh"
+                with open(job_file, "w") as f:
+                    f.write(
+                        get_sbatch_script(
+                            run_id,
+                            run_info.subrun,
+                            file,
+                            output_dir,
+                            log_dir,
+                            log_file,
+                            ref_time,
+                            ref_counter,
+                            module,
+                            ref_source,
+                            script,
+                        )
+                    )
+                sp.run(["sbatch", job_file], check=True)
+
+    calib_runs = summary_table[summary_table["run_type"] != "DATA"]
+    log.info(f"Found {len(calib_runs)} NO-DATA runs")
+
+    for run in calib_runs:
+        log.info(f"Copying R0 files corresponding to run {run} directly to {output_dir}")
+        # Avoid copying files while it is still night time
+        wait_for_daytime(start, end)
+
+        run_id = run["run_id"]
+        r0_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+
+        for file in r0_files:
+            sp.run(["cp", file, output_dir])
+
+def run_sacct_j(job) -> StringIO:
+    """Run sacct to obtain the job information."""
+    if shutil.which("sacct") is None:
+        log.warning("No job info available since sacct command is not available")
+        return StringIO()
+
+    sacct_cmd = [
+        "sacct",
+        "-n",
+        "--parsable2",
+        "--delimiter=,",
+        "--units=G",
+        "-o",
+        ",".join(FORMAT_SLURM),
+        "-j",
+        job,
+    ]
+
+    return StringIO(sp.check_output(sacct_cmd).decode())
+
+
+def GainSel_flag_file(date: str) -> Path:
+    filename = cfg.get("LSTOSA", "gain_selection_check")
+    GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
+    flagfile = GainSel_dir / date / filename
+    return flagfile.resolve()
+
+
+def GainSel_finished(date: str) -> bool:
+    """Check if gain selection finished successfully."""
+    flagfile = GainSel_flag_file(date)
+    return flagfile.exists()
+
+
+def check_failed_jobs(date: str, output_basedir: Path = None):
+    """Search for failed jobs in the log directory."""
+    failed_jobs = []
+    log_dir = output_basedir / "log" / date
+    filenames = glob.glob(f"{log_dir}/gain_selection*.log")
+    jobs = [re.search(r'(?<=_)(.[0-9.]+?)(?=.log)', i).group(0) for i in filenames]
+
+    for job in jobs:
+        output = run_sacct_j(job)
+        df = get_sacct_output(output)
+
+        if not df.iloc[0]["State"] == "COMPLETED":
+            log.warning(f"Job {job} did not finish successfully")
+            failed_jobs.append(job)
+
+    if failed_jobs:
+        log.warning(f"{date}: some jobs did not finish successfully")
+
+    else:
+        log.info(f"{date}: all jobs finished successfully")
+
+
+        run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
+        run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
+        summary_table = Table.read(run_summary_file)
+        runs = summary_table["run_id"]
+        missing_runs = []
+
+        r0_files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run?????.????.fits.fz")
+        r0g_files = glob.glob(f"/fefs/aswg/data/real/R0G/{date}/LST-1.?.Run?????.????.fits.fz")
+        all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
+        all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
+
+        for run in all_r0_runs:
+            if run not in runs:
+                if run not in all_r0g_runs:
+                    missing_runs.append(run)
+
+        missing_runs.sort()
+        if missing_runs:
+            log.info(
+                f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
+                f"directly to /fefs/aswg/data/real/R0G/{date}"
+            )
+
+            for run in missing_runs:
+                output_dir = Path(f"/fefs/aswg/data/real/R0G/{date}/")
+                files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run{run:05d}.????.fits.fz")
+                for file in files:
+                    sp.run(["cp", file, output_dir])
+
+        GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
+        flagfile_dir = GainSel_dir / date
+        flagfile_dir.mkdir(parents=True, exist_ok=True)
+        
+        flagfile = GainSel_flag_file(date)
+        log.info(f"Gain selection finished successfully, creating flag file for date {date} ({flagfile})")
+        flagfile.touch()
+
+
+def main():
+    """
+    Loop over the dates listed in the input file and launch the gain selection
+    script for each of them. The input file should list the dates in the format
+    YYYYMMDD one date per line.
+    """
+    log.setLevel(logging.INFO)
+    args = parser.parse_args()
+
+    if args.date:
+        if args.check:
+            log.info(f"Checking gain selection status for date {args.date}")
+            check_failed_jobs(args.date, args.output_basedir)
+        else:
+            log.info(f"Applying gain selection to date {args.date}")
+            apply_gain_selection(args.date, args.start_time, args.end_time, args.output_basedir, no_queue_check=args.no_queue_check)
+
+
+    elif args.dates_file:
+        list_of_dates = get_list_of_dates(args.dates_file)
+        log.info(f"Found {len(list_of_dates)} dates to apply or check gain selection")
+
+        if args.check:
+            for date in list_of_dates:
+                log.info(f"Checking gain selection status for date {date}")
+                check_failed_jobs(date, args.output_basedir)
+        else:
+            for date in list_of_dates:
+                log.info(f"Applying gain selection to date {date}")
+                apply_gain_selection(date, args.start_time, args.end_time, args.output_basedir, no_queue_check=args.no_queue_check)
+            log.info("Done! No more dates to process.")
+
+
+if __name__ == "__main__":
+    main()