lstosa 0.10.11__py3-none-any.whl → 0.10.12__py3-none-any.whl

{lstosa-0.10.11.dist-info → lstosa-0.10.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.11
+Version: 0.10.12
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.11.dist-info → lstosa-0.10.12.dist-info}/RECORD

@@ -1,6 +1,6 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=BHIdn73vwwgskUs3cpzDgJte1GN_HGZgzXDzfpJ_m6c,415
-osa/conftest.py,sha256=_NERtB9t-Oi7ykW4QDYy9ZbgsrYxXMmQ3QkDScH5JyE,19601
+osa/_version.py,sha256=iptzZkijfZOuLJEgzcWGVRKqYSdbRz1HyJHUd9sgNhw,415
+osa/conftest.py,sha256=NBeGqTUBRqCPirDSDPny4bf1e_OJXbiePazHwaoQPY4,20072
 osa/job.py,sha256=OnjF88kTVdURcrIR9iPenATNx2HteDFlAKtOX4fD144,26603
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
 osa/paths.py,sha256=R-LwBlyoIJ-PuTJ8vcSYWMwzQY7YrgXq7gan0zhVzPY,14399
@@ -12,7 +12,7 @@ osa/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/configs/config.py,sha256=cX0Vr4sorBVQ2KRqPIRSEGENKH_uC5lrgVHx1hp6YTk,1148
 osa/configs/datamodel.py,sha256=L_WRM91PBlMrtuE30akh7YR-56P0g9D994qzKSfhNJc,1950
 osa/configs/options.py,sha256=CyL7WnHiC_pvB3mnjRF7Wg43uPzQgmwlbvIqkRzlDLA,524
-osa/configs/sequencer.cfg,sha256=cysPy6q8vShh0V0LiEBfip-rUmwyNlNVDKhGPMiKH5g,4784
+osa/configs/sequencer.cfg,sha256=vG-6VS2osCccEsY58JgaqqADUDz-k8-9tD3NDitscR4,4871
 osa/high_level/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/high_level/selection_cuts.toml,sha256=ReSmcKtOPZY5JsZ9ExnxYdz7OrJEB8gghCbzHmeOyFg,128
 osa/high_level/significance.py,sha256=Y1jokkHCo-D_qSqxKiQzc6KJSmivznaJRS2xY-txNIo,9039
@@ -34,22 +34,22 @@ osa/provenance/config/definition.yaml,sha256=DSwqNln1jEXV8aUh7ca2r7ArMkpaMJi1xbH
 osa/provenance/config/environment.yaml,sha256=kPM6ucPyLZLDFzkwFWoY0C6vmAArG98U-P1UAl89bgE,246
 osa/provenance/config/logger.yaml,sha256=hy_lH3DfbRFh2VM_iawI-c-3wE0cjTRHy465C2eFfnQ,510
 osa/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/autocloser.py,sha256=j7nRvglIv_Ol_GJzbcfqPHquBIWwQU515GkDRfc6_ho,14540
+osa/scripts/autocloser.py,sha256=Z4x4yfDA2Xkr-6ld57L4VjWcVse-sLqkGuE6ZFgSXps,14716
 osa/scripts/calibration_pipeline.py,sha256=g9o1chqCRRSo7GNuQZRABjGnJYjZyfhkPgRLPmuV994,5703
 osa/scripts/closer.py,sha256=p11tq2YJLwsRK_CBWXU1465ArzLJ0qWlUCnubGqLPUo,18277
 osa/scripts/copy_datacheck.py,sha256=tfDs6oTdPbii4BOXp6bTHuED0xNJeqaPFrv6Ed7ZnWc,3104
 osa/scripts/datasequence.py,sha256=gXAp8arbLPEK-sca9VnME6-2XfUzBFIoEFchlUZYrXI,9260
-osa/scripts/gain_selection.py,sha256=yaz2fKicuCDgLbD3fhb3l9LBlXLEEO3yuZO-oDc2IPo,8084
+osa/scripts/gain_selection.py,sha256=0ZOFgX-ZaJFoe_B0UI5NYRr7CmeHL1_bxjlB0n8E5hs,13775
 osa/scripts/provprocess.py,sha256=mufkZe6_qwH3DGqTFxINIc01hciF5RMpw3n_Mp7vwXU,18629
 osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
 osa/scripts/reprocessing.py,sha256=D-J8Rl3GrkWpxYkk6ci79oJOMewgGdxLkQgaHCAZuqs,3417
-osa/scripts/sequencer.py,sha256=6Cg-eIExk0eN8-HXkO6DBeFEWOI-FznDueqJEpbEzos,8163
+osa/scripts/sequencer.py,sha256=GM6B17YQNJfBOosWx0gpXBy4fIwga3DC4T2nGapGQEU,8763
 osa/scripts/sequencer_webmaker.py,sha256=99P1rJiBlvNidDeppvtw4kOoAfsOxSMJ2sm4xlgKCE8,4640
 osa/scripts/show_run_summary.py,sha256=SoDLVKdQHOJkfenFguBOfXf10Gyv7heXSQAFnDVZqMs,2468
 osa/scripts/simulate_processing.py,sha256=NiRVYiwZENt_mnKncytgJT23_-tJMb1B5PswM12nnX4,6941
 osa/scripts/update_source_catalog.py,sha256=GHwWFc-y6S4KkUJxUVM5drdAnVDD0-n3D-Tv3CCmh4E,7218
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=1B2UFc3idLGSyqeuzf4IEvb6sEbtfaV8ogj1LoEoEyQ,12851
+osa/scripts/tests/test_osa_scripts.py,sha256=7xms2Jq2ZalU5kiHBBbtFOxBzXhrybsNlT16nEtFTxU,12926
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/tests/test_jobs.py,sha256=F0jsHZ9BYB_cCHHxlXyO9v1E5_-mBJhuFtshtsAtnXo,15260
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
@@ -59,7 +59,7 @@ osa/tests/test_raw.py,sha256=WkgwEc_vY0D6nREo-BSm6F-5xDpqidMC0DkS86pXlRU,1058
 osa/tests/test_report.py,sha256=OY-EsrXytoS6esfjUeLnIAmCMIw9EzoGD-elySafyhE,1365
 osa/tests/test_veto.py,sha256=UIsooji_5Z8TtAhc0UlD2VqheVd9DBufuMxinJ3e0w8,1066
 osa/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/utils/cliopts.py,sha256=Xqh-9w2o1iSHfwO0jHEW0AXQ9VeEw6X6IGLNAPKoNyc,13699
+osa/utils/cliopts.py,sha256=XeVu6TjrIEoqQEqf4meIKSiRyobMqYTLuggj-iGU4SQ,14126
 osa/utils/iofile.py,sha256=kJ7KB1suynhS2cTf7EeHwhMXq3delC_ls2HFpCzvsZo,2021
 osa/utils/logging.py,sha256=1WcNPjjslo3y25jcEY_fe0yXOeJ6frZrGLAy1GJpu_k,1491
 osa/utils/mail.py,sha256=uQfqPQdiOVHTvEAXr9H15a7-g9DtYVNKjMEb9GnI0oY,554
@@ -74,9 +74,9 @@ osa/workflow/dl3.py,sha256=kz7L5jcKHFJ--UdQ8HQKLzWO6nxc2LLOTz42ExcqzTk,9921
 osa/workflow/stages.py,sha256=WYgUM2XDIaUjCc4_Zs_VSGW6gk73EaKcHk6ZMnPds74,6692
 osa/workflow/tests/test_dl3.py,sha256=aY5bb-8OcZGAXG3JPCZihChzkA_GsWjRIa31BHZn3Dg,299
 osa/workflow/tests/test_stages.py,sha256=TmC00XFACWZp740TQeFaokWi3C50ovj_XGiySWrrdZk,3944
-lstosa-0.10.11.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.10.11.dist-info/METADATA,sha256=X6RF9uQOjldNtXq3Fn6yOTSaiXpxqdQmIrtg0HDdat0,7349
-lstosa-0.10.11.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-lstosa-0.10.11.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
-lstosa-0.10.11.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.10.11.dist-info/RECORD,,
+lstosa-0.10.12.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.10.12.dist-info/METADATA,sha256=HmGZoRfvvKndHKpKPPxBbsC85Rj7Eel1Q9lQhbZyJoM,7349
+lstosa-0.10.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+lstosa-0.10.12.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
+lstosa-0.10.12.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.10.12.dist-info/RECORD,,

osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.11'
-__version_tuple__ = version_tuple = (0, 10, 11)
+__version__ = version = '0.10.12'
+__version_tuple__ = version_tuple = (0, 10, 12)

osa/configs/sequencer.cfg

@@ -28,6 +28,7 @@ HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
 LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
+GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
@@ -138,6 +139,7 @@ WORKFLOWPREFIX: Workflow
 GRAPHSUFFIX: .dot
 SVGSUFFIX: .svg
 end_of_activity: NightFinished.txt
+gain_selection_check: GainSelFinished.txt
 
 [OUTPUT]
 # REPORTWIDTH is the width in characters of the heading frame for the output

osa/conftest.py

@@ -48,6 +48,13 @@ def monitoring_dir(base_test_dir):
     return monitoring_dir
 
 
+@pytest.fixture(scope="session")
+def osa_dir(base_test_dir):
+    osa_dir = base_test_dir / "OSA"
+    osa_dir.mkdir(parents=True, exist_ok=True)
+    return osa_dir
+
+
 @pytest.fixture(scope="session")
 def run_summary_dir(monitoring_dir):
     summary_dir = monitoring_dir / "RunSummary"
@@ -436,6 +443,8 @@ def sequence_file_list(
     drs4_time_calibration_files,
     systematic_correction_files,
     r0_data,
+    gain_selection_flag_file,
+    merged_run_summary,
 ):
     for r0_file in r0_data:
         assert r0_file.exists()
@@ -448,6 +457,8 @@ def sequence_file_list(
 
     assert run_summary_file.exists()
     assert run_catalog.exists()
+    assert gain_selection_flag_file.exists()
+    assert merged_run_summary.exists()
 
     run_program("sequencer", "-d", "2020-01-17", "--no-submit", "-t", "LST1")
     # First sequence in the list corresponds to the calibration run 1809
@@ -548,11 +559,9 @@ def run_catalog(run_catalog_dir):
 
 
 @pytest.fixture(scope="session")
-def database(base_test_dir):
+def database(osa_dir):
     import sqlite3
 
-    osa_dir = base_test_dir / "OSA"
-    osa_dir.mkdir(parents=True, exist_ok=True)
     db_file = osa_dir / "osa.db"
     with sqlite3.connect(db_file) as connection:
         cursor = connection.cursor()
@@ -562,3 +571,13 @@ def database(base_test_dir):
         )
         cursor.connection.commit()
         yield cursor
+
+
+@pytest.fixture(scope="session")
+def gain_selection_flag_file(osa_dir):
+
+    GainSel_dir = osa_dir / "GainSel" / "20200117"
+    GainSel_dir.mkdir(parents=True, exist_ok=True)
+    file = GainSel_dir / "GainSelFinished.txt"
+    file.touch()
+    return file

osa/scripts/autocloser.py

@@ -41,6 +41,7 @@ class Telescope:
         config_file: Path,
         ignore_cronlock: bool = False,
         test: bool = False,
+        no_gainsel: bool = False,
     ):
         """
         Parameters
@@ -82,7 +83,7 @@ class Telescope:
         if not self.lock_automatic_sequencer() and not ignore_cronlock:
             log.warning(f"{self.telescope} already locked! Ignoring {self.telescope}")
             return
-        if not self.simulate_sequencer(date, config_file, test):
+        if not self.simulate_sequencer(date, config_file, test, no_gainsel):
             log.warning(
                 f"Simulation of the sequencer failed "
                 f"for {self.telescope}! Ignoring {self.telescope}"
@@ -121,7 +122,7 @@ class Telescope:
         self.locked = True
         return True
 
-    def simulate_sequencer(self, date: str, config_file: Path, test: bool):
+    def simulate_sequencer(self, date: str, config_file: Path, test: bool, no_gainsel: bool):
         """Launch the sequencer in simulation mode."""
         if test:
             self.read_file()
@@ -135,6 +136,9 @@ class Telescope:
                 date,
                 self.telescope,
             ]
+            if no_gainsel:
+                sequencer_cmd.insert(1, "--no-gainsel")
+
             log.debug(f"Executing {' '.join(sequencer_cmd)}")
             sequencer = subprocess.Popen(
                 sequencer_cmd,
@@ -445,7 +449,7 @@ def main():
     # create telescope and sequence objects
     log.info("Simulating sequencer...")
 
-    telescope = Telescope(args.tel_id, date, args.config)
+    telescope = Telescope(args.tel_id, date, args.config, no_gainsel=args.no_gainsel)
 
     log.info(f"Processing {args.tel_id}...")
 

osa/scripts/gain_selection.py

@@ -8,8 +8,8 @@ import subprocess as sp
 from pathlib import Path
 from textwrap import dedent
 from io import StringIO
+import argparse
 
-import click
 from astropy.table import Table
 from lstchain.paths import run_info_from_filename, parse_r0_filename
 
@@ -17,51 +17,131 @@ from osa.scripts.reprocessing import get_list_of_dates, check_job_status_and_wai
 from osa.utils.utils import wait_for_daytime
 from osa.utils.logging import myLogger
 from osa.job import get_sacct_output, FORMAT_SLURM
+from osa.configs.config import cfg
+from osa.paths import DEFAULT_CFG
 
 log = myLogger(logging.getLogger(__name__))
 
 PATH = "PATH=/fefs/aswg/software/offline_dvr/bin:$PATH"
 
+parser = argparse.ArgumentParser(add_help=False)
+parser.add_argument(
+        "--check",                                                                                       
+        action="store_true",
+        default=False,
+        help="Check if any job failed",
+)
+parser.add_argument(
+        "--no-queue-check",
+        action="store_true",
+        default=False,
+        help="Do not wait until the number of jobs in the slurm queue is < 1500",
+)
+parser.add_argument(
+        "-c",                                                                                            
+        "--config",                                                                                      
+        action="store",
+        type=Path,
+        default=DEFAULT_CFG,
+        help="Configuration file",
+)
+parser.add_argument(                                                                                     
+        "-d",                                                                                            
+        "--date",                                                                                        
+        default=None,
+        type=str,
+        help="Night to apply the gain selection",
+)                                                                                                        
+parser.add_argument(                                                                                     
+        "-l",                                                                                            
+        "--dates-file",
+        default=None,
+        help="List of dates to apply the gain selection",
+)
+parser.add_argument(                                                                                     
+        "-o",                                                                                            
+        "--output-basedir",
+        type=Path,
+        default=Path("/fefs/aswg/data/real/R0G"),
+        help="Output directory of the gain selected files. Default is /fefs/aswg/data/real/R0G."
+)                                                                                                        
+parser.add_argument(                                                                                     
+        "-s",                                                                                            
+        "--start-time",
+        type=int,
+        default=10,
+        help="Time to (re)start gain selection in HH format. Default is 10.",
+)                                                                                                       
+parser.add_argument(                                                                                     
+        "-e",                                                                                            
+        "--end-time",
+        type=int,
+        default=18,
+        help="Time to stop gain selection in HH format. Default is 18.",
+)
 
 def get_sbatch_script(
-    run_id, subrun, input_file, output_dir, log_dir, ref_time, ref_counter, module, ref_source
+    run_id, subrun, input_file, output_dir, log_dir, log_file, ref_time, ref_counter, module, ref_source, script
 ):
     """Build the sbatch job pilot script for running the gain selection."""
-    return dedent(
-        f"""\
-    #!/bin/bash
-
-    #SBATCH -D {log_dir}
-    #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
-    #SBATCH --job-name "gain_selection_{run_id:05d}"
-    #SBATCH --export {PATH}
-
-    lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
-    """
-    )
-
+    if script=="old":
+        return dedent(
+            f"""\
+        #!/bin/bash
+
+        #SBATCH -D {log_dir}
+        #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
+        #SBATCH --job-name "gain_selection_{run_id:05d}"
+        #SBATCH --export {PATH}
+        #SBATCH --partition=short,long
+
+        lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
+        """
+        )
+    elif script=="new":
+        return dedent(
+            f"""\
+        #!/bin/bash
+
+        #SBATCH -D {log_dir}
+        #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
+        #SBATCH --job-name "gain_selection_{run_id:05d}"
+        #SBATCH --mem=40GB
+        #SBATCH --partition=short,long
+
+        lstchain_r0_to_r0g --R0-file={input_file} --output-dir={output_dir} --log={log_file} --no-flatfield-heuristic
+        """
+        )
 
-def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None):
+def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None, no_queue_check: bool = False):
     """
     Submit the jobs to apply the gain selection to the data for a given date
     on a subrun-by-subrun basis.
     """
 
+    if date < "20231205":
+        script = "old"
+    else:
+        script = "new"
+
     run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
     run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
     summary_table = Table.read(run_summary_file)
     # Apply gain selection only to DATA runs
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    log.info(f"Found {len(data_runs)} DATA runs to which apply the gain selection")
 
     output_dir = output_basedir / date
     log_dir = output_basedir / "log" / date
     output_dir.mkdir(parents=True, exist_ok=True)
     log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = log_dir / f"r0_to_r0g_{date}.log"
     r0_dir = Path(f"/fefs/aswg/data/real/R0/{date}")
 
     for run in data_runs:
-        # Check slurm queue status and sleep for a while to avoid overwhelming the queue
-        check_job_status_and_wait(max_jobs=1500)
+        if not no_queue_check:
+            # Check slurm queue status and sleep for a while to avoid overwhelming the queue
+            check_job_status_and_wait(max_jobs=1500)
 
         # Avoid running jobs while it is still night time
         wait_for_daytime(start, end)
@@ -76,14 +156,25 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
         subrun_numbers = [int(file[-12:-8]) for file in files]
         input_files = []
 
-        if ref_source in ["UCTS", "TIB"]:
+        if date < "20231205" and ref_source not in ["UCTS", "TIB"]:
+            input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+            log.info(
+                f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
+                f"be applied. Copying directly the R0 files to {output_dir}."
+            )
+            for file in input_files:
+                sp.run(["cp", file, output_dir])
 
+        else:
             n_subruns = max(subrun_numbers)
 
             for subrun in range(n_subruns + 1):
                 new_files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.{subrun:04d}.fits.fz")
 
                 if len(new_files) != 4:
+                    log.info(f"Run {run_id}.{subrun:05d} does not have 4 streams of R0 files, so gain"
+                        f"selection cannot be applied. Copying directly the R0 files to {output_dir}."
+                    )
                     for file in new_files:
                         sp.run(["cp", file, output_dir])
 
@@ -91,6 +182,7 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
                     new_files.sort()
                     input_files.append(new_files[0])
 
+            log.info("Creating and launching the sbatch scripts for the rest of the runs to apply gain selection")
             for file in input_files:
                 run_info = run_info_from_filename(file)
                 job_file = log_dir / f"gain_selection_{run_info.run:05d}.{run_info.subrun:04d}.sh"
@@ -102,24 +194,21 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
                             file,
                             output_dir,
                             log_dir,
+                            log_file,
                             ref_time,
                             ref_counter,
                             module,
                             ref_source,
+                            script,
                         )
                     )
                 sp.run(["sbatch", job_file], check=True)
 
-        else:
-
-            input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
-
-            for file in input_files:
-                sp.run(["cp", file, output_dir])
-
     calib_runs = summary_table[summary_table["run_type"] != "DATA"]
+    log.info(f"Found {len(calib_runs)} NO-DATA runs")
 
     for run in calib_runs:
+        log.info(f"Copying R0 files corresponding to run {run} directly to {output_dir}")
         # Avoid copying files while it is still night time
         wait_for_daytime(start, end)
 
@@ -150,6 +239,19 @@ def run_sacct_j(job) -> StringIO:
     return StringIO(sp.check_output(sacct_cmd).decode())
 
 
+def GainSel_flag_file(date: str) -> Path:
+    filename = cfg.get("LSTOSA", "gain_selection_check")
+    GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
+    flagfile = GainSel_dir / date / filename
+    return flagfile.resolve()
+
+
+def GainSel_finished(date: str) -> bool:
+    """Check if gain selection finished successfully."""
+    flagfile = GainSel_flag_file(date)
+    return flagfile.exists()
+
+
 def check_failed_jobs(date: str, output_basedir: Path = None):
     """Search for failed jobs in the log directory."""
     failed_jobs = []
@@ -165,72 +267,82 @@ def check_failed_jobs(date: str, output_basedir: Path = None):
             log.warning(f"Job {job} did not finish successfully")
             failed_jobs.append(job)
 
-    if not failed_jobs:
-        log.info(f"{date}: all jobs finished successfully")
-    else:
+    if failed_jobs:
         log.warning(f"{date}: some jobs did not finish successfully")
 
-
-    run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
-    run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
-    summary_table = Table.read(run_summary_file)
-    runs = summary_table["run_id"]
-    missing_runs = []
-
-    r0_files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run?????.????.fits.fz")
-    r0g_files = glob.glob(f"/fefs/aswg/data/real/R0G/{date}/LST-1.?.Run?????.????.fits.fz")
-    all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
-    all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
-
-    for run in all_r0_runs:
-        if run not in runs:
-            if run not in all_r0g_runs:
-                missing_runs.append(run)
-
-    missing_runs.sort()
-    if missing_runs:
-        log.info(
-            f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
-            f"directly to /fefs/aswg/data/real/R0G/{date}"
-        )
-
-        for run in missing_runs:
-            output_dir = Path(f"/fefs/aswg/data/real/R0G/{date}/")
-            files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run{run:05d}.????.fits.fz")
-            for file in files:
-                sp.run(["cp", file, output_dir])
-
+    else:
+        log.info(f"{date}: all jobs finished successfully")
 
 
-@click.command()
-@click.option("--check", is_flag=True, default=False, help="Check for failed jobs.")
-@click.argument("dates-file", type=click.Path(exists=True, path_type=Path))
-@click.argument("output-basedir", type=click.Path(path_type=Path))
-@click.option("-s", "--start-time", type=int, default=10, help="Time to (re)start gain selection in HH format.")
-@click.option("-e", "--end-time", type=int, default=18, help="Time to stop gain selection in HH format.")
-def main(
-    dates_file: Path = None, 
-    output_basedir: Path = None, 
-    check: bool = False, 
-    start_time: int = 10, 
-    end_time: int = 18
-):
+        run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
+        run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
+        summary_table = Table.read(run_summary_file)
+        runs = summary_table["run_id"]
+        missing_runs = []
+
+        r0_files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run?????.????.fits.fz")
+        r0g_files = glob.glob(f"/fefs/aswg/data/real/R0G/{date}/LST-1.?.Run?????.????.fits.fz")
+        all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
+        all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
+
+        for run in all_r0_runs:
+            if run not in runs:
+                if run not in all_r0g_runs:
+                    missing_runs.append(run)
+
+        missing_runs.sort()
+        if missing_runs:
+            log.info(
+                f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
+                f"directly to /fefs/aswg/data/real/R0G/{date}"
+            )
+
+            for run in missing_runs:
+                output_dir = Path(f"/fefs/aswg/data/real/R0G/{date}/")
+                files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run{run:05d}.????.fits.fz")
+                for file in files:
+                    sp.run(["cp", file, output_dir])
+
+        GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
+        flagfile_dir = GainSel_dir / date
+        flagfile_dir.mkdir(parents=True, exist_ok=True)
+        
+        flagfile = GainSel_flag_file(date)
+        log.info(f"Gain selection finished successfully, creating flag file for date {date} ({flagfile})")
+        flagfile.touch()
+
+
+def main():
     """
     Loop over the dates listed in the input file and launch the gain selection
     script for each of them. The input file should list the dates in the format
     YYYYMMDD one date per line.
     """
     log.setLevel(logging.INFO)
+    args = parser.parse_args()
 
-    list_of_dates = get_list_of_dates(dates_file)
+    if args.date:
+        if args.check:
+            log.info(f"Checking gain selection status for date {args.date}")
+            check_failed_jobs(args.date, args.output_basedir)
+        else:
+            log.info(f"Applying gain selection to date {args.date}")
+            apply_gain_selection(args.date, args.start_time, args.end_time, args.output_basedir, no_queue_check=args.no_queue_check)
 
-    if check:
-        for date in list_of_dates:
-            check_failed_jobs(date, output_basedir)
-    else:
-        for date in list_of_dates:
-            apply_gain_selection(date, start_time, end_time, output_basedir)
-        log.info("Done! No more dates to process.")
+
+    elif args.dates_file:
+        list_of_dates = get_list_of_dates(args.dates_file)
+        log.info(f"Found {len(list_of_dates)} dates to apply or check gain selection")
+
+        if args.check:
+            for date in list_of_dates:
+                log.info(f"Checking gain selection status for date {date}")
+                check_failed_jobs(date, args.output_basedir)
+        else:
+            for date in list_of_dates:
+                log.info(f"Applying gain selection to date {date}")
+                apply_gain_selection(date, args.start_time, args.end_time, args.output_basedir, no_queue_check=args.no_queue_check)
+            log.info("Done! No more dates to process.")
 
 
 if __name__ == "__main__":

osa/scripts/sequencer.py

@@ -7,6 +7,7 @@ prepares a SLURM job array which launches the data sequences for every subrun.
 
 import logging
 import os
+import sys
 from decimal import Decimal
 
 from osa import osadb
@@ -22,12 +23,14 @@ from osa.job import (
     run_squeue,
 )
 from osa.nightsummary.extract import build_sequences
+from osa.nightsummary.nightsummary import run_summary_table
 from osa.paths import analysis_path
 from osa.report import start
 from osa.utils.cliopts import sequencer_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import is_day_closed, gettag, date_to_iso
+from osa.utils.utils import is_day_closed, gettag, date_to_iso, date_to_dir
 from osa.veto import get_closed_list, get_veto_list
+from osa.scripts.gain_selection import GainSel_finished
 
 __all__ = [
     "single_process",
@@ -90,6 +93,18 @@ def single_process(telescope):
     if not options.simulate:
         os.makedirs(options.log_directory, exist_ok=True)
 
+    summary_table = run_summary_table(options.date)
+    if len(summary_table) == 0:
+        log.warning("No runs found for this date. Nothing to do. Exiting.")
+        sys.exit(0)
+
+    if not options.no_gainsel and not GainSel_finished(date_to_dir(options.date)):
+        log.info(
+            f"Gain selection did not finish successfully for date {options.date}."
+            "Try again later, once gain selection has finished."
+        )
+        sys.exit()
+
     if is_day_closed():
         log.info(f"Date {date_to_iso(options.date)} is already closed for {options.tel_id}")
         return sequence_list

osa/scripts/tests/test_osa_scripts.py

@@ -126,9 +126,11 @@ def test_simulated_sequencer(
     run_catalog,
     r0_data,
     merged_run_summary,
+    gain_selection_flag_file,
 ):
     assert run_summary_file.exists()
     assert run_catalog.exists()
+    assert gain_selection_flag_file.exists()
 
     for r0_file in r0_data:
         assert r0_file.exists()

osa/utils/cliopts.py

@@ -274,6 +274,12 @@ def sequencer_argparser():
         default=False,
         help="Do not produce DL2 files (default False)",
     )
+    parser.add_argument(
+        "--no-gainsel",
+        action="store_true",
+        default=False,
+        help="Do not check if the gain selection finished correctly (default False)",
+    )
     parser.add_argument(
         "tel_id",
         choices=["ST", "LST1", "LST2", "all"],
@@ -292,6 +298,7 @@ def sequencer_cli_parsing():
     options.no_submit = opts.no_submit
     options.no_calib = opts.no_calib
     options.no_dl2 = opts.no_dl2
+    options.no_gainsel = opts.no_gainsel
 
     log.debug(f"the options are {opts}")
 
@@ -476,6 +483,12 @@ def autocloser_cli_parser():
         default=False,
         help="Disregard the production of DL2 files",
     )
+    parser.add_argument(
+        "--no-gainsel",
+        action="store_true",
+        default=False,
+        help="Do not check if the gain selection finished correctly (default False)",
+    )
     parser.add_argument("-r", "--runwise", action="store_true", help="Close the day run-wise.")
     parser.add_argument("-l", "--log", type=Path, default=None, help="Write log to a file.")
     parser.add_argument("tel_id", type=str, choices=["LST1"])