lstosa 0.10.14__py3-none-any.whl → 0.10.15__py3-none-any.whl

{lstosa-0.10.14.dist-info → lstosa-0.10.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.14
+Version: 0.10.15
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.14.dist-info → lstosa-0.10.15.dist-info}/RECORD

@@ -1,7 +1,7 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=76UZ8OAmh_iNsrP-vbUrVEfcRxo13WrlquAZql0gVVg,415
+osa/_version.py,sha256=zLN3_5vILM2SpYp9rDAVJoNlgXr6RmLnoGoKrqnkdX8,415
 osa/conftest.py,sha256=NBeGqTUBRqCPirDSDPny4bf1e_OJXbiePazHwaoQPY4,20072
-osa/job.py,sha256=OnjF88kTVdURcrIR9iPenATNx2HteDFlAKtOX4fD144,26603
+osa/job.py,sha256=vl7kBRM8Oe5YK2onUWGtGmHWX0AEktS2sbOuS5fsbgU,27074
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
 osa/paths.py,sha256=R-LwBlyoIJ-PuTJ8vcSYWMwzQY7YrgXq7gan0zhVzPY,14399
 osa/raw.py,sha256=ZNIsuqfx5ljoz_hwhSuafdKf-wr8-cxRJmel-A2endg,1337
@@ -12,7 +12,7 @@ osa/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/configs/config.py,sha256=cX0Vr4sorBVQ2KRqPIRSEGENKH_uC5lrgVHx1hp6YTk,1148
 osa/configs/datamodel.py,sha256=L_WRM91PBlMrtuE30akh7YR-56P0g9D994qzKSfhNJc,1950
 osa/configs/options.py,sha256=CyL7WnHiC_pvB3mnjRF7Wg43uPzQgmwlbvIqkRzlDLA,524
-osa/configs/sequencer.cfg,sha256=vG-6VS2osCccEsY58JgaqqADUDz-k8-9tD3NDitscR4,4871
+osa/configs/sequencer.cfg,sha256=ZHgY1GvIHkdQIg0GtR5iSNXb1C0giDObxL1SH77G1Io,4991
 osa/high_level/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/high_level/selection_cuts.toml,sha256=ReSmcKtOPZY5JsZ9ExnxYdz7OrJEB8gghCbzHmeOyFg,128
 osa/high_level/significance.py,sha256=Y1jokkHCo-D_qSqxKiQzc6KJSmivznaJRS2xY-txNIo,9039
@@ -39,19 +39,20 @@ osa/scripts/calibration_pipeline.py,sha256=g9o1chqCRRSo7GNuQZRABjGnJYjZyfhkPgRLP
 osa/scripts/closer.py,sha256=p11tq2YJLwsRK_CBWXU1465ArzLJ0qWlUCnubGqLPUo,18277
 osa/scripts/copy_datacheck.py,sha256=tfDs6oTdPbii4BOXp6bTHuED0xNJeqaPFrv6Ed7ZnWc,3104
 osa/scripts/datasequence.py,sha256=gXAp8arbLPEK-sca9VnME6-2XfUzBFIoEFchlUZYrXI,9260
-osa/scripts/gain_selection.py,sha256=2aYmvKSW2ELHvf-4eW_AwiynvrbUeFCsvG2A2qmqvTQ,14775
+osa/scripts/gain_selection.py,sha256=YIxSSkROox9Xqdgtz9yTFzhWg5F0Qx1ZEJkauaQF2Y8,22871
+osa/scripts/gainsel_webmaker.py,sha256=40_DX7RUmImX-31iqRTFPEUvdVXXPSKp0xMEJWuAu80,5006
 osa/scripts/provprocess.py,sha256=mufkZe6_qwH3DGqTFxINIc01hciF5RMpw3n_Mp7vwXU,18629
 osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
 osa/scripts/reprocessing.py,sha256=A97kqX7QtD7ewoUvFhmSaQVVIfplViGhzTtOenTx37w,3630
-osa/scripts/sequencer.py,sha256=GM6B17YQNJfBOosWx0gpXBy4fIwga3DC4T2nGapGQEU,8763
-osa/scripts/sequencer_webmaker.py,sha256=YticifQRicxCp5YMeOaP4L8ZRWl7KF2raceVrCf2Ras,4765
+osa/scripts/sequencer.py,sha256=zLG_lZ4Nh0oOhTr3TeAG3RsG-iDBOwvt68Vemn-KpGI,10572
+osa/scripts/sequencer_webmaker.py,sha256=gdBYxB85wZIH01ZZKAnlygMiqBEChR1gTHmCGdA08Xo,4792
 osa/scripts/show_run_summary.py,sha256=SoDLVKdQHOJkfenFguBOfXf10Gyv7heXSQAFnDVZqMs,2468
 osa/scripts/simulate_processing.py,sha256=NiRVYiwZENt_mnKncytgJT23_-tJMb1B5PswM12nnX4,6941
 osa/scripts/update_source_catalog.py,sha256=GHwWFc-y6S4KkUJxUVM5drdAnVDD0-n3D-Tv3CCmh4E,7218
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=7xms2Jq2ZalU5kiHBBbtFOxBzXhrybsNlT16nEtFTxU,12926
+osa/scripts/tests/test_osa_scripts.py,sha256=8rOHbujP-eutTM4QOdDvKbQEPon3gSzteua1PAgKO1E,13790
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/tests/test_jobs.py,sha256=F0jsHZ9BYB_cCHHxlXyO9v1E5_-mBJhuFtshtsAtnXo,15260
+osa/tests/test_jobs.py,sha256=tdgZE21jisCvtlB6oAqXbV4sM1KPMG5tD-Rmcaf5OWo,15488
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
 osa/tests/test_osadb.py,sha256=pJHV1dxxblGH2sjS-JPDPTkMn-ew1MzbioCFyg7wbB8,1599
 osa/tests/test_paths.py,sha256=sFCxG5uPGLcto76E7X1I26-kRx5faxgHGh9z8LvHz2M,3173
@@ -59,7 +60,7 @@ osa/tests/test_raw.py,sha256=WkgwEc_vY0D6nREo-BSm6F-5xDpqidMC0DkS86pXlRU,1058
 osa/tests/test_report.py,sha256=OY-EsrXytoS6esfjUeLnIAmCMIw9EzoGD-elySafyhE,1365
 osa/tests/test_veto.py,sha256=UIsooji_5Z8TtAhc0UlD2VqheVd9DBufuMxinJ3e0w8,1066
 osa/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/utils/cliopts.py,sha256=PFxGh89SJBWjvqrWit05JFUeXnNz4w_ZXqVp4uZwlcc,14319
+osa/utils/cliopts.py,sha256=4uBf4gfUDARWzhzlobPERGn6Pv1ONWdpAXOKxh_Dtm4,14533
 osa/utils/iofile.py,sha256=kJ7KB1suynhS2cTf7EeHwhMXq3delC_ls2HFpCzvsZo,2021
 osa/utils/logging.py,sha256=1WcNPjjslo3y25jcEY_fe0yXOeJ6frZrGLAy1GJpu_k,1491
 osa/utils/mail.py,sha256=uQfqPQdiOVHTvEAXr9H15a7-g9DtYVNKjMEb9GnI0oY,554
@@ -74,9 +75,9 @@ osa/workflow/dl3.py,sha256=kz7L5jcKHFJ--UdQ8HQKLzWO6nxc2LLOTz42ExcqzTk,9921
 osa/workflow/stages.py,sha256=WYgUM2XDIaUjCc4_Zs_VSGW6gk73EaKcHk6ZMnPds74,6692
 osa/workflow/tests/test_dl3.py,sha256=aY5bb-8OcZGAXG3JPCZihChzkA_GsWjRIa31BHZn3Dg,299
 osa/workflow/tests/test_stages.py,sha256=TmC00XFACWZp740TQeFaokWi3C50ovj_XGiySWrrdZk,3944
-lstosa-0.10.14.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.10.14.dist-info/METADATA,sha256=qy3G8stmx7C9eAbLAECr6VgWqDn7wKEuSw4kklsmdPU,7349
-lstosa-0.10.14.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-lstosa-0.10.14.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
-lstosa-0.10.14.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.10.14.dist-info/RECORD,,
+lstosa-0.10.15.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.10.15.dist-info/METADATA,sha256=MMb8yy_4losD6R4MOaW8kHk0R5xz_C-q-kby_yVzKNk,7349
+lstosa-0.10.15.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+lstosa-0.10.15.dist-info/entry_points.txt,sha256=qfARj13Vqt3I-E0tXuJwyC79cv84bjCwEj5uK67vWts,981
+lstosa-0.10.15.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.10.15.dist-info/RECORD,,

{lstosa-0.10.14.dist-info → lstosa-0.10.15.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{lstosa-0.10.14.dist-info → lstosa-0.10.15.dist-info}/entry_points.txt

@@ -6,6 +6,7 @@ copy_datacheck = osa.scripts.copy_datacheck:main
 datasequence = osa.scripts.datasequence:main
 dl3_stage = osa.workflow.dl3:main
 gain_selection = osa.scripts.gain_selection:main
+gainsel_webmaker = osa.scripts.gainsel_webmaker:main
 provprocess = osa.scripts.provprocess:main
 reprocess_longterm = osa.scripts.reprocess_longterm:main
 reprocessing = osa.scripts.reprocessing:main

osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.14'
-__version_tuple__ = version_tuple = (0, 10, 14)
+__version__ = version = '0.10.15'
+__version_tuple__ = version_tuple = (0, 10, 15)

osa/configs/sequencer.cfg

@@ -29,6 +29,7 @@ LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
 GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
+GAIN_SELECTION_WEB_DIR: %(OSA_DIR)s/GainSelWeb
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
@@ -54,6 +55,7 @@ dl1_to_dl2: lstchain_dl1_to_dl2
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+use_ff_heuristic_gain_selection: False
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
 rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3
@@ -70,11 +72,13 @@ electron: /path/to/DL2/electron_mc_testing.h5
 PARTITION_PEDCALIB: short, long
 PARTITION_DATA: short, long
 MEMSIZE_PEDCALIB: 3GB
-MEMSIZE_DATA: 16GB
+MEMSIZE_DATA: 6GB
+MEMSIZE_GAINSEL: 2GB
 WALLTIME: 1:15:00
 # Days from current day up to which the jobs are fetched from the queue.
 # Default is None (left empty).
 STARTTIME_DAYS_SACCT:
+ACCOUNT: dpps
 
 [WEBSERVER]
 # Set the server address and port to transfer the datacheck plots

osa/job.py

@@ -342,6 +342,7 @@ def scheduler_env_variables(sequence, scheduler="slurm"):
 
     sbatch_parameters.append(f"--partition={cfg.get('SLURM', f'PARTITION_{sequence.type}')}")
     sbatch_parameters.append(f"--mem-per-cpu={cfg.get('SLURM', f'MEMSIZE_{sequence.type}')}")
+    sbatch_parameters.append(f"--account={cfg.get('SLURM', 'ACCOUNT')}")
 
     return ["#SBATCH " + line for line in sbatch_parameters]
 
@@ -652,7 +653,7 @@ def get_squeue_output(squeue_output: StringIO) -> pd.DataFrame:
     return df
 
 
-def run_sacct() -> StringIO:
+def run_sacct(job_id: str = None) -> StringIO:
     """Run sacct to obtain the job information."""
     if shutil.which("sacct") is None:
         log.warning("No job info available since sacct command is not available")
@@ -667,13 +668,18 @@ def run_sacct() -> StringIO:
         "-o",
         ",".join(FORMAT_SLURM),
     ]
+
+    if job_id:
+        sacct_cmd.append("--jobs")
+        sacct_cmd.append(job_id)
+
     if cfg.get("SLURM", "STARTTIME_DAYS_SACCT"):
         days = int(cfg.get("SLURM", "STARTTIME_DAYS_SACCT"))
         start_date = (datetime.date.today() - datetime.timedelta(days=days)).isoformat()
         sacct_cmd.extend(["--starttime", start_date])
 
     return StringIO(sp.check_output(sacct_cmd).decode())
-
+    
 
 def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
     """
@@ -808,3 +814,12 @@ def update_sequence_state(sequence, filtered_job_info: pd.DataFrame) -> None:
         sequence.exit = "0:15"
     elif any("RUNNING" in job for job in filtered_job_info.State):
         sequence.state = "RUNNING"
+
+
+def job_finished_in_timeout(job_id: str) -> bool:
+    """Return True if the input job_id finished in TIMEOUT state."""
+    job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+    if job_id and job_status.item() == "TIMEOUT":
+        return True
+    else:
+        return False

osa/scripts/gain_selection.py

@@ -1,24 +1,28 @@
 """Script to run the gain selection over a list of dates."""
 import logging
 import re
-import shutil
 import glob
 import pandas as pd
 import subprocess as sp
 from pathlib import Path
 from textwrap import dedent
-from io import StringIO
 import argparse
+import sys
 
 from astropy.table import Table
-from lstchain.paths import run_info_from_filename, parse_r0_filename
+from lstchain.paths import parse_r0_filename
+from datetime import datetime
 
 from osa.scripts.reprocessing import get_list_of_dates, check_job_status_and_wait
-from osa.utils.utils import wait_for_daytime
+from osa.utils.utils import wait_for_daytime, date_to_dir, date_to_iso
 from osa.utils.logging import myLogger
-from osa.job import get_sacct_output, FORMAT_SLURM
+from osa.utils.iofile import append_to_file
+from osa.utils.cliopts import valid_date
+from osa.job import get_sacct_output, run_sacct, job_finished_in_timeout
 from osa.configs.config import cfg
 from osa.paths import DEFAULT_CFG
+from osa.nightsummary.nightsummary import run_summary_table
+
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -49,23 +53,16 @@ parser.add_argument(
         "-d",                                                                                            
         "--date",                                                                                        
         default=None,
-        type=str,
-        help="Night to apply the gain selection in YYYYMMDD format",
+        type=valid_date,
+        help="Night to apply the gain selection in YYYY-MM-DD format",
 )                                                                                                        
 parser.add_argument(                                                                                     
         "-l",                                                                                            
         "--dates-file",
         default=None,
         help="List of dates to apply the gain selection. The input file should list"
-        "the dates in the format YYYYMMDD, one date per line.",
-)
-parser.add_argument(                                                                                     
-        "-o",                                                                                            
-        "--output-basedir",
-        type=Path,
-        default=Path("/fefs/aswg/data/real/R0G"),
-        help="Output directory of the gain selected files. Default is /fefs/aswg/data/real/R0G."
-)                                                                                                        
+        "the dates in the format YYYY-MM-DD, one date per line.",
+)                                                                                                       
 parser.add_argument(                                                                                     
         "-s",                                                                                            
         "--start-time",
@@ -86,67 +83,205 @@ parser.add_argument(
         type=str,
         default=None,
         help="Choose tool to apply the gain selection regardless the date. Possible options are: lst_dvr (by default used for dates "
-        "previous to 20231205) and lstchain_r0_to_r0g (by default used for dates later than 20231205).",
+        "previous to 2023-12-05) and lstchain_r0_to_r0g (by default used for dates later than 2023-12-05).",
+)
+parser.add_argument(                                                                                          
+        "--simulate",
+        action="store_true",
+        default=False,
+        help="Simulate launching of the gain selection script. Dry run.",
+)
+parser.add_argument(
+        "-v",                                                                                      
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="Activate debugging mode.",
 )
 
 def get_sbatch_script(
-    run_id, subrun, input_file, output_dir, log_dir, log_file, ref_time, ref_counter, module, ref_source, tool
+    run_id: str,
+    subrun: str,
+    input_file: Path,
+    output_dir: Path,
+    log_dir: Path,
+    log_file: Path,
+    ref_time: int,
+    ref_counter: int,
+    module: int,
+    ref_source: str,
+    tool: str
 ):
     """Build the sbatch job pilot script for running the gain selection."""
-    if tool == "lst_dvr":
-        return dedent(
+    mem_per_job = cfg.get("SLURM", "MEMSIZE_GAINSEL")
+    sbatch_script = dedent(
             f"""\
         #!/bin/bash
 
         #SBATCH -D {log_dir}
         #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
         #SBATCH --job-name "gain_selection_{run_id:05d}"
-        #SBATCH --export {PATH}
         #SBATCH --partition=short,long
+        #SBATCH --mem={mem_per_job}
+        """
+        )
+    
+    if tool == "lst_dvr":
+        sbatch_script += dedent(
+            f"""   
+        #SBATCH --export {PATH}
 
         lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
         """
         )
+        
     elif tool == "lstchain_r0_to_r0g":
-        return dedent(
-            f"""\
-        #!/bin/bash
+        cmd = f"lstchain_r0_to_r0g --R0-file={input_file} --output-dir={output_dir} --log={log_file}"
+        if not cfg.getboolean("lstchain", "use_ff_heuristic_gain_selection"): 
+            cmd += " --no-flatfield-heuristic"
+        sbatch_script += dedent(cmd)
 
-        #SBATCH -D {log_dir}
-        #SBATCH -o "gain_selection_{run_id:05d}_{subrun:04d}_%j.log"
-        #SBATCH --job-name "gain_selection_{run_id:05d}"
-        #SBATCH --mem=40GB
-        #SBATCH --partition=short,long
+    return sbatch_script
 
-        lstchain_r0_to_r0g --R0-file={input_file} --output-dir={output_dir} --log={log_file} --no-flatfield-heuristic
-        """
-        )
 
-def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None, tool: str = None, no_queue_check: bool = False):
+def launch_gainsel_for_data_run(
+    date: datetime, run: Table, output_dir: Path, r0_dir: Path, log_dir: Path, tool: str, simulate: bool = False
+    ):
+    """
+    Create the gain selection sbatch script and launch it for a given run.
+    
+    Runs from before 20231205 without UCTS or TIB info are directly copied to the final directory.
+    Subruns that do not have four streams are also directly copied.
+    """
+    run_id = run["run_id"]
+    ref_time = run["dragon_reference_time"]
+    ref_counter = run["dragon_reference_counter"]
+    module = run["dragon_reference_module_index"]
+    ref_source = run["dragon_reference_source"].upper()
+
+    files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    subrun_numbers = [int(file[-12:-8]) for file in files]
+    
+    if tool == "lst_dvr" and ref_source not in ["UCTS", "TIB"]:
+        input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+        
+        if is_run_already_copied(date, run_id):
+            log.info(f"The R0 files corresponding to run {run_id} have already been copied to the R0G directory.")
+        else:
+            if not simulate:
+                for file in input_files:
+                    log.debug(
+                        f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
+                        f"be applied. Copying directly the R0 files to {output_dir}."
+                    )
+                    sp.run(["cp", file, output_dir])
+
+            else:
+                log.info(
+                    f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
+                    f"be applied. Simulate copy of the R0 files directly to {output_dir}."
+                )
+
+    else:
+        n_subruns = max(subrun_numbers) 
+
+        for subrun in range(n_subruns + 1):
+
+            r0_files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.{subrun:04d}.fits.fz")
+
+            if len(r0_files) != 4:
+                if not simulate and not is_run_already_copied(date, run_id):
+                    log.debug(f"Run {run_id:05d}.{subrun:04d} does not have 4 streams of R0 files, so gain"
+                        f"selection cannot be applied. Copying directly the R0 files to {output_dir}.")
+                    for file in r0_files:
+                        sp.run(["cp", file, output_dir])
+                elif is_run_already_copied(date, run_id):
+                    log.debug(f"Run {run_id:05d}.{subrun:04d} does not have 4 streams of R0 files. The R0 files"
+                        f"have already been copied to {output_dir}.")
+                elif simulate:
+                    log.debug(f"Run {run_id:05d}.{subrun:04d} does not have 4 streams of R0 files, so gain"
+                        f"selection cannot be applied. Simulate copy of the R0 files directly to {output_dir}.")
+
+            else:
+                history_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.history"
+                if history_file.exists():
+                    if not simulate:
+                        update_history_file(run_id, subrun, log_dir, history_file)
+
+                    if history_file.read_text() == "":   # history_file is empty
+                        log.debug(f"Gain selection is still running for run {run_id:05d}.{subrun:04d}")
+                        continue
+                    else:
+                        gainsel_rc = history_file.read_text().splitlines()[-1][-1]
+                        if gainsel_rc == "1":
+                            job_id = get_last_job_id(run_id, subrun, log_dir)
+                            if job_finished_in_timeout(job_id) and not simulate:
+                                # Relaunch the job that finished in TIMEOUT
+                                job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"
+                                sp.run(["sbatch", job_file], stdout=sp.PIPE, stderr=sp.STDOUT, check=True)
+                            else:
+                                log.warning(f"Gain selection failed for run {run_id:05d}.{subrun:04d}")
+                        elif gainsel_rc == "0":
+                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d},"
+                                        "no additional jobs will be submitted for this subrun.") 
+                else:
+                    log.debug("Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
+                    if not simulate:
+                        log_file = log_dir / f"r0_to_r0g_{run_id:05d}.{subrun:04d}.log"
+                        job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"
+                        r0_files.sort()
+                        with open(job_file, "w") as f:
+                            f.write(
+                                get_sbatch_script(
+                                    run_id,
+                                    subrun,
+                                    r0_files[0],
+                                    output_dir,
+                                    log_dir,
+                                    log_file,
+                                    ref_time,
+                                    ref_counter,
+                                    module,
+                                    ref_source,
+                                    tool,
+                                )
+                            )
+
+                        #submit job
+                        history_file.touch()
+                        sp.run(["sbatch", job_file], stdout=sp.PIPE, stderr=sp.STDOUT, check=True)
+
+
+def apply_gain_selection(date: datetime, start: int, end: int, tool: str = None, no_queue_check: bool = False, simulate: bool = False):
     """
     Submit the jobs to apply the gain selection to the data for a given date
     on a subrun-by-subrun basis.
     """
 
     if not tool:
-        if date < "20231205":
+        if date_to_dir(date) < "20231205":
             tool = "lst_dvr"
         else:
             tool = "lstchain_r0_to_r0g"
 
-    run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
-    run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
-    summary_table = Table.read(run_summary_file)
+    summary_table = run_summary_table(date)
+
+    if len(summary_table) == 0:
+        log.warning(f"No runs are found in the run summary of {date_to_iso(date)}. Nothing to do. Exiting.")
+        sys.exit(0)
+
     # Apply gain selection only to DATA runs
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
     log.info(f"Found {len(data_runs)} DATA runs to which apply the gain selection")
 
-    output_dir = output_basedir / date
-    log_dir = output_basedir / "log" / date
-    output_dir.mkdir(parents=True, exist_ok=True)
-    log_dir.mkdir(parents=True, exist_ok=True)
-    log_file = log_dir / f"r0_to_r0g_{date}.log"
-    r0_dir = Path(f"/fefs/aswg/data/real/R0/{date}")
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    date_str = date_to_dir(date)
+    r0_dir = base_dir / "R0" / date_str
+    output_dir = base_dir / f"R0G/{date_str}"
+    log_dir = base_dir / f"R0G/log/{date_str}"
+    if not simulate:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        log_dir.mkdir(parents=True, exist_ok=True)
 
     for run in data_runs:
         if not no_queue_check:
@@ -156,171 +291,203 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
         # Avoid running jobs while it is still night time
         wait_for_daytime(start, end)
 
-        run_id = run["run_id"]
-        ref_time = run["dragon_reference_time"]
-        ref_counter = run["dragon_reference_counter"]
-        module = run["dragon_reference_module_index"]
-        ref_source = run["dragon_reference_source"].upper()
-
-        files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.????.fits.fz")
-        subrun_numbers = [int(file[-12:-8]) for file in files]
-        input_files = []
-
-        if tool == "lst_dvr" and ref_source not in ["UCTS", "TIB"]:
-            input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
-            log.info(
-                f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
-                f"be applied. Copying directly the R0 files to {output_dir}."
-            )
-            for file in input_files:
-                sp.run(["cp", file, output_dir])
+        if not is_closed(date, run["run_id"]):
+            launch_gainsel_for_data_run(date, run, output_dir, r0_dir, log_dir, tool, simulate)
+
+    calib_runs = summary_table[summary_table["run_type"] != "DATA"]
+    log.info(f"Found {len(calib_runs)} NO-DATA runs")
 
+    for run in calib_runs:
+        run_id = run["run_id"]
+        
+        if is_run_already_copied(date, run_id):
+            log.info(f"The R0 files corresponding to run {run_id:05d} have already been copied, nothing to do.")
         else:
-            n_subruns = max(subrun_numbers)
+            log.info(f"Copying R0 files corresponding to run {run_id} directly to {output_dir}")
+       	    if not simulate:
+            	# Avoid copying files while it is still night time
+            	wait_for_daytime(start, end)
 
-            for subrun in range(n_subruns + 1):
-                new_files = glob.glob(f"{r0_dir}/LST-1.?.Run{run_id:05d}.{subrun:04d}.fits.fz")
+            	r0_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
 
-                if len(new_files) != 4:
-                    log.info(f"Run {run_id}.{subrun:05d} does not have 4 streams of R0 files, so gain"
-                        f"selection cannot be applied. Copying directly the R0 files to {output_dir}."
-                    )
-                    for file in new_files:
-                        sp.run(["cp", file, output_dir])
+            	for file in r0_files:
+                    sp.run(["cp", file, output_dir])
 
-                else:
-                    new_files.sort()
-                    input_files.append(new_files[0])
-
-            log.info("Creating and launching the sbatch scripts for the rest of the runs to apply gain selection")
-            for file in input_files:
-                run_info = run_info_from_filename(file)
-                job_file = log_dir / f"gain_selection_{run_info.run:05d}.{run_info.subrun:04d}.sh"
-                with open(job_file, "w") as f:
-                    f.write(
-                        get_sbatch_script(
-                            run_id,
-                            run_info.subrun,
-                            file,
-                            output_dir,
-                            log_dir,
-                            log_file,
-                            ref_time,
-                            ref_counter,
-                            module,
-                            ref_source,
-                            tool,
-                        )
-                    )
-                sp.run(["sbatch", job_file], check=True)
 
-    calib_runs = summary_table[summary_table["run_type"] != "DATA"]
-    log.info(f"Found {len(calib_runs)} NO-DATA runs")
+def get_last_job_id(run_id: str, subrun: str, log_dir: Path) -> int:
+    """Get job id of the last gain selection job that was launched for a given subrun."""
+    filenames = glob.glob(f"{log_dir}/gain_selection_{run_id:05d}_{subrun:04d}_*.log")
+    if filenames:
+        match = re.search(f'gain_selection_{run_id:05d}_{subrun:04d}_(\d+).log', sorted(filenames)[-1])
+        job_id = match.group(1)
+        return job_id
 
-    for run in calib_runs:
-        run_id = run["run_id"]
-        log.info(f"Copying R0 files corresponding to run {run_id} directly to {output_dir}")
-        # Avoid copying files while it is still night time
-        wait_for_daytime(start, end)
 
-        run_id = run["run_id"]
-        r0_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
+def update_history_file(run_id: str, subrun: str, log_dir: Path, history_file: Path):
+    """
+    Update the gain selection history file with the result 
+    of the last job launched for a given subrun.
+    """
+    job_id = get_last_job_id(run_id, subrun, log_dir)
+    if not job_id:
+        log.debug(f"Cannot find a job_id for the run {run_id:05d}.{subrun:04d}")
+    else:
+        job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+        if job_status.item() in ["RUNNING", "PENDING"]:
+            log.info(f"Job {job_id} is still running.")
+            return
+            
+        elif job_status.item() == "COMPLETED":
+            log.debug(f"Job {job_id} finished successfully, updating history file.")
+            string_to_write = (
+                f"{run_id:05d}.{subrun:04d} gain_selection 0\n"
+            )
+            append_to_file(history_file, string_to_write)
+        
+        else:
+            log.info(f"Job {job_id} failed, updating history file.")
+            string_to_write = (
+                f"{run_id:05d}.{subrun:04d} gain_selection 1\n"
+            )
+            append_to_file(history_file, string_to_write)
 
-        for file in r0_files:
-            sp.run(["cp", file, output_dir])
 
-def run_sacct_j(job) -> StringIO:
-    """Run sacct to obtain the job information."""
-    if shutil.which("sacct") is None:
-        log.warning("No job info available since sacct command is not available")
-        return StringIO()
+def is_run_already_copied(date: datetime, run_id: int) -> bool:
+    """Check if the R0 files of a given run have already been copied to the R0G directory."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    r0_files = glob.glob(f"{base_dir}/R0/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    r0g_files = glob.glob(f"{base_dir}/R0G/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    return len(r0_files)==len(r0g_files)
 
-    sacct_cmd = [
-        "sacct",
-        "-n",
-        "--parsable2",
-        "--delimiter=,",
-        "--units=G",
-        "-o",
-        ",".join(FORMAT_SLURM),
-        "-j",
-        job,
-    ]
 
-    return StringIO(sp.check_output(sacct_cmd).decode())
+def is_closed(date: datetime, run_id: str) -> bool:
+    """Check if run is already closed."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
+    closed_run_file = log_dir / f"gain_selection_{run_id:05d}.closed"
+    return closed_run_file.exists()
 
 
-def GainSel_flag_file(date: str) -> Path:
+def GainSel_flag_file(date: datetime) -> Path:
+    """Return the path to the file indicating the completion of the gain selection stage."""
     filename = cfg.get("LSTOSA", "gain_selection_check")
     GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
-    flagfile = GainSel_dir / date / filename
+    flagfile = GainSel_dir / date_to_dir(date) / filename
     return flagfile.resolve()
 
 
-def GainSel_finished(date: str) -> bool:
+def GainSel_finished(date: datetime) -> bool:
     """Check if gain selection finished successfully."""
     flagfile = GainSel_flag_file(date)
     return flagfile.exists()
 
 
-def check_failed_jobs(date: str, output_basedir: Path = None):
+def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool:
     """Search for failed jobs in the log directory."""
-    failed_jobs = []
-    log_dir = output_basedir / "log" / date
-    filenames = glob.glob(f"{log_dir}/gain_selection*.log")
-    jobs = [re.search(r'(?<=_)(.[0-9.]+?)(?=.log)', i).group(0) for i in filenames]
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
+    history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history")
+    failed_subruns = []
+    log.info(f"Checking all history files of run {run_id}")
+    
+    for file in history_files:
+        match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file))
+        subrun = match.group(1)
+        if file.read_text() != "":
+            gainsel_rc = file.read_text().splitlines()[-1][-1]
+
+            if gainsel_rc == "1":
+                log.warning(f"Gain selection failed for run {run_id}.{subrun}")
+                failed_subruns.append(file)
+
+            elif gainsel_rc == "0":
+                log.debug(f"Gain selection finished successfully for run {run_id}.{subrun}")
+        else:
+            log.info(f"Gain selection is still running for run {run_id}.{subrun}")
+            return False
+            
+    if failed_subruns:
+        log.warning(f"{date_to_iso(date)}: Some gain selection jobs did not finish successfully for run {run_id}")
+        return False
+    else:
+        log.info(f"{date_to_iso(date)}: All jobs finished successfully for run {run_id}, creating the corresponding .closed file")
+        closed_run_file = log_dir / f"gain_selection_{run_id:05d}.closed"
+        closed_run_file.touch()
+        return True
 
-    for job in jobs:
-        output = run_sacct_j(job)
-        df = get_sacct_output(output)
 
-        if not df.iloc[0]["State"] == "COMPLETED":
-            log.warning(f"Job {job} did not finish successfully")
-            failed_jobs.append(job)
+def check_warnings_in_logs(date: datetime, run_id: int):
+    """Look for warnings in the log files created by lstchain_r0_to_r0g."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
+    log_files = log_dir.glob(f"r0_to_r0g_{run_id:05d}.*.log")
+    for file in log_files:
+        content = file.read_text().splitlines()
+        for line in content:
+            if "FlatField(FF)-like events are not tagged as FF" in line:
+                log.warning(f"Warning for run {run_id}: {line}")
 
-    if failed_jobs:
-        log.warning(f"{date}: some jobs did not finish successfully")
 
-    else:
-        log.info(f"{date}: all jobs finished successfully")
-
-
-        run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
-        run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
-        summary_table = Table.read(run_summary_file)
-        runs = summary_table["run_id"]
-        missing_runs = []
-
-        r0_files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run?????.????.fits.fz")
-        r0g_files = glob.glob(f"/fefs/aswg/data/real/R0G/{date}/LST-1.?.Run?????.????.fits.fz")
-        all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
-        all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
-
-        for run in all_r0_runs:
-            if run not in runs:
-                if run not in all_r0g_runs:
-                    missing_runs.append(run)
-
-        missing_runs.sort()
-        if missing_runs:
-            log.info(
-                f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
-                f"directly to /fefs/aswg/data/real/R0G/{date}"
-            )
+def check_failed_jobs(date: datetime):
+    """Search for failed jobs in the log directory."""
 
-            for run in missing_runs:
-                output_dir = Path(f"/fefs/aswg/data/real/R0G/{date}/")
-                files = glob.glob(f"/fefs/aswg/data/real/R0/{date}/LST-1.?.Run{run:05d}.????.fits.fz")
-                for file in files:
-                    sp.run(["cp", file, output_dir])
+    summary_table = run_summary_table(date)
 
-        GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
-        flagfile_dir = GainSel_dir / date
-        flagfile_dir.mkdir(parents=True, exist_ok=True)
+    if len(summary_table) == 0:
+        log.warning(f"No runs are found in the run summary of {date_to_iso(date)}. Nothing to do. Exiting.")
+        sys.exit(0)
         
-        flagfile = GainSel_flag_file(date)
-        log.info(f"Gain selection finished successfully, creating flag file for date {date} ({flagfile})")
-        flagfile.touch()
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    failed_runs = []
+
+    for run in data_runs:
+        run_id = run["run_id"]
+        check_warnings_in_logs(date, run_id)
+        if not is_closed(date, run_id):
+            if not check_gainsel_jobs_runwise(date, run_id):
+                log.warning(f"Gain selection did not finish successfully for run {run_id}.")
+                failed_runs.append(run)
+
+    if failed_runs:
+        log.warning(f"Gain selection did not finish successfully for {date_to_iso(date)}, cannot create the flag file.")
+        return
+
+    runs = summary_table["run_id"]
+    missing_runs = []
+
+    date_str = date_to_dir(date)
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    r0_files = glob.glob(f"{base_dir}/R0/{date_str}/LST-1.?.Run?????.????.fits.fz")
+    r0g_files = glob.glob(f"{base_dir}/R0G/{date_str}/LST-1.?.Run?????.????.fits.fz")
+    all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
+    all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
+
+    for run in all_r0_runs:
+        if run not in runs:
+            if run not in all_r0g_runs:
+                missing_runs.append(run)
+
+    missing_runs.sort()
+    if missing_runs:
+        output_dir = base_dir / f"R0G/{date_str}/"
+        log.info(
+            f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
+            f"directly to {output_dir}"
+        )
+
+        for run in missing_runs:
+            
+            files = base_dir.glob(f"R0/{date_str}/LST-1.?.Run{run:05d}.????.fits.fz")
+            for file in files:
+                sp.run(["cp", file, output_dir])
+
+    GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
+    flagfile_dir = GainSel_dir / date_str
+    flagfile_dir.mkdir(parents=True, exist_ok=True)
+    
+    flagfile = GainSel_flag_file(date)
+    log.info(f"Gain selection finished successfully, creating flag file for date {date_to_iso(date)} ({flagfile})")
+    flagfile.touch()
 
 
 def main():
@@ -329,22 +496,29 @@ def main():
     script for each of them. The input file should list the dates in the format
     YYYYMMDD one date per line.
     """
-    log.setLevel(logging.INFO)
     args = parser.parse_args()
+    
+    if args.verbose:
+        log.setLevel(logging.DEBUG)
+    else:
+        log.setLevel(logging.INFO)
 
     if args.date:
-        if args.check:
-            log.info(f"Checking gain selection status for date {args.date}")
-            check_failed_jobs(args.date, args.output_basedir)
+        if GainSel_finished(args.date):
+            log.warning(f"Gain selection already done for date {date_to_iso(args.date)}. Exiting.")
+            sys.exit(0)
+        elif args.check:
+            log.info(f"Checking gain selection status for date {date_to_iso(args.date)}")
+            check_failed_jobs(args.date)
         else:
-            log.info(f"Applying gain selection to date {args.date}")
+            log.info(f"\nApplying gain selection to date {date_to_iso(args.date)}")
             apply_gain_selection(
                 args.date, 
                 args.start_time, 
-                args.end_time, 
-                args.output_basedir,
+                args.end_time,
                 args.tool,
                 no_queue_check=args.no_queue_check, 
+                simulate=args.simulate,
             )
 
 
@@ -355,7 +529,7 @@ def main():
         if args.check:
             for date in list_of_dates:
                 log.info(f"Checking gain selection status for date {date}")
-                check_failed_jobs(date, args.output_basedir)
+                check_failed_jobs(date)
         else:
             for date in list_of_dates:
                 log.info(f"Applying gain selection to date {date}")
@@ -363,9 +537,9 @@ def main():
                     date, 
                     args.start_time, 
                     args.end_time,
-                    args.output_basedir,
                     args.tool,
                     no_queue_check=args.no_queue_check,
+                    simulate=args.simulate,
                 )
             log.info("Done! No more dates to process.")
 

osa/scripts/gainsel_webmaker.py

@@ -0,0 +1,157 @@
+import logging
+from argparse import ArgumentParser
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pandas as pd
+from astropy.table import Table
+
+from osa.configs import options
+from osa.configs.config import cfg
+from osa.nightsummary.nightsummary import run_summary_table
+from osa.paths import DEFAULT_CFG
+from osa.scripts.sequencer_webmaker import html_content
+from osa.utils.utils import date_to_dir, date_to_iso
+
+log = logging.getLogger(__name__)
+
+
+def valid_date(string):
+    """Check if the string is a valid date and return a datetime object."""
+    return datetime.strptime(string, "%Y-%m-%d")
+
+
+common_parser = ArgumentParser(add_help=False)
+common_parser.add_argument(
+    "-c",
+    "--config",
+    type=Path,
+    default=DEFAULT_CFG,
+    help="Use specific config file [default configs/sequencer.cfg]",
+)
+common_parser.add_argument(
+    "-d",
+    "--date",
+    help="Date of the start of the night in ISO format (YYYY-MM-DD). Defaults to yesterday",
+    type=valid_date,
+)
+
+
+def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool:
+    """Search for failed jobs in the log directory."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    flat_date = date_to_dir(date)
+    log_dir = base_dir / f"R0G/log/{flat_date}"
+    history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history")
+
+    success_subruns = 0
+    failed_subruns = 0
+    pending_subruns = 0
+
+    for file in history_files:
+        if file.read_text() != "":
+            gainsel_rc = file.read_text().splitlines()[-1][-1]
+
+            if gainsel_rc == "1":
+                failed_subruns += 1
+
+            elif gainsel_rc == "0":
+                success_subruns += 1
+
+        else:
+            pending_subruns += 1
+
+    return {"pending": pending_subruns, "success": success_subruns, "failed": failed_subruns}
+
+
+def check_failed_jobs(date: datetime) -> pd.DataFrame:
+    """Search for failed jobs in the log directory."""
+    summary_table = run_summary_table(date)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+
+    gainsel_status_dict = {}
+    for run in data_runs:
+        run_id = run["run_id"]
+        gainsel_job_status = check_gainsel_jobs_runwise(date, run_id)
+        gainsel_status_dict[run_id] = gainsel_job_status
+
+    gainsel_df = pd.DataFrame(gainsel_status_dict.values(), index=gainsel_status_dict.keys())
+    gainsel_df.reset_index(inplace=True)
+    gainsel_df.rename(columns={"index": "run_id"}, inplace=True)
+    summary_table = summary_table.to_pandas()
+
+    final_table = pd.merge(summary_table, gainsel_df, on="run_id")[
+        [
+            "run_id",
+            "n_subruns",
+            "pending",
+            "success",
+            "failed",
+        ]
+    ]
+
+    def determine_status(row):
+        if row["failed"] > 0:
+            return "FAILED"
+        elif row["pending"] == row["n_subruns"]:
+            return "PENDING"
+        elif row["success"] == row["n_subruns"]:
+            return "COMPLETED"
+        elif row["pending"] > 0:
+            return "RUNNING"
+        else:
+            return "NOT STARTED"
+
+    final_table["GainSel%"] = round(final_table["success"] * 100 / final_table["n_subruns"])
+    final_table["GainSelStatus"] = final_table.apply(determine_status, axis=1)
+
+    return final_table
+
+
+def main():
+    """Produce the html file with the processing OSA Gain Selection status.
+
+    It creates an HTML file osa_gainsel_status_YYYY-MM-DD.html
+    """
+    args = ArgumentParser(
+        description=(
+            "Script to create an HTML file with the gain selection status "
+            "(osa_gainsel_status_YYYY-MM-DD.html)"
+        ),
+        parents=[common_parser],
+    ).parse_args()
+
+    if args.date:
+        flat_date = date_to_dir(args.date)
+        options.date = args.date
+
+    else:
+        # yesterday by default
+        yesterday = datetime.now() - timedelta(days=1)
+        options.date = yesterday
+        flat_date = date_to_dir(yesterday)
+
+    date = date_to_iso(options.date)
+    run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR"))
+    run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv"
+
+    gain_selection_web_directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR"))
+    gain_selection_web_directory.mkdir(parents=True, exist_ok=True)
+    html_file = gain_selection_web_directory / f"osa_gainsel_status_{date}.html"
+
+    # Create and save the HTML file
+    if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0:
+        content = "<p>No data found</p>"
+        log.warning(f"No data found for date {date}, creating an empty HTML file.")
+
+    else:
+        # Get the table with the gain selection check report in HTML format:
+        table_gain_selection_jobs = check_failed_jobs(options.date)
+        content = table_gain_selection_jobs.to_html(justify="left")
+
+    html_file.write_text(html_content(content, date, "OSA Gain Selection"))
+    log.info(f"Created HTML file {html_file}")
+
+
+if __name__ == "__main__":
+    main()

osa/scripts/sequencer.py

@@ -9,6 +9,7 @@ import logging
 import os
 import sys
 from decimal import Decimal
+import datetime
 
 from osa import osadb
 from osa.configs import options
@@ -28,7 +29,7 @@ from osa.paths import analysis_path
 from osa.report import start
 from osa.utils.cliopts import sequencer_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import is_day_closed, gettag, date_to_iso, date_to_dir
+from osa.utils.utils import is_day_closed, gettag, date_to_iso
 from osa.veto import get_closed_list, get_veto_list
 from osa.scripts.gain_selection import GainSel_finished
 
@@ -98,9 +99,9 @@ def single_process(telescope):
         log.warning("No runs found for this date. Nothing to do. Exiting.")
         sys.exit(0)
 
-    if not options.no_gainsel and not GainSel_finished(date_to_dir(options.date)):
+    if not options.no_gainsel and not GainSel_finished(options.date):
         log.info(
-            f"Gain selection did not finish successfully for date {options.date}."
+            f"Gain selection did not finish successfully for date {date_to_iso(options.date)}. "
             "Try again later, once gain selection has finished."
         )
         sys.exit()
@@ -109,6 +110,15 @@ def single_process(telescope):
         log.info(f"Date {date_to_iso(options.date)} is already closed for {options.tel_id}")
         return sequence_list
 
+    if not options.test and not options.simulate:
+        if is_sequencer_running(options.date):
+            log.info(f"Sequencer is still running for date {date_to_iso(options.date)}. Try again later.")
+            sys.exit(0)
+
+        elif is_sequencer_completed(options.date) and not options.force_submit:
+            log.info(f"Sequencer already finished for date {date_to_iso(options.date)}. Exiting")
+            sys.exit(0)
+
     # Build the sequences
     sequence_list = build_sequences(options.date)
 
@@ -306,5 +316,39 @@ def output_matrix(matrix: list, padding_space: int):
         log.info(stringrow)
 
 
+def is_sequencer_running(date: datetime.datetime) -> bool:
+    """Check if the jobs launched by sequencer are running or pending for the given date."""
+    summary_table = run_summary_table(date)
+    sacct_output = run_sacct()
+    sacct_info = get_sacct_output(sacct_output)
+
+    for run in summary_table["run_id"]:
+        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
+        queued_jobs = jobs_run[(jobs_run["State"] == "RUNNING") | (jobs_run["State"] == "PENDING")]
+        if len(queued_jobs) != 0:
+            return True
+
+    return False
+
+
+def is_sequencer_completed(date: datetime.datetime) -> bool:
+    """Check if the jobs launched by sequencer are running or pending for the given date."""
+    summary_table = run_summary_table(date)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    sacct_output = run_sacct()
+    sacct_info = get_sacct_output(sacct_output)
+
+    for run in data_runs["run_id"]:
+        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
+        if len(jobs_run["JobID"].unique())>1:
+            last_job_id = sorted(jobs_run["JobID"].unique())[-1]
+            jobs_run = sacct_info[sacct_info["JobID"]==last_job_id]
+        incomplete_jobs = jobs_run[(jobs_run["State"] != "COMPLETED")]
+        if len(jobs_run) == 0 or len(incomplete_jobs) != 0:
+            return False
+
+    return True
+
+
 if __name__ == "__main__":
     main()

osa/scripts/sequencer_webmaker.py

@@ -20,7 +20,7 @@ from osa.utils.utils import is_day_closed, date_to_iso, date_to_dir
 log = myLogger(logging.getLogger())
 
 
-def html_content(body: str, date: str) -> str:
+def html_content(body: str, date: str, title: str) -> str:
     """Build the HTML content.
 
     Parameters
@@ -43,11 +43,11 @@ def html_content(body: str, date: str) -> str:
         <html xmlns="http://www.w3.org/1999/xhtml">
          <head>
           <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-          <title>OSA Sequencer status</title><link href="osa.css" rel="stylesheet"
+          <title>{title} status</title><link href="osa.css" rel="stylesheet"
           type="text/css" /><style>table{{width:152ex;}}</style>
          </head>
          <body>
-         <h1>OSA processing status</h1>
+         <h1>{title} processing status</h1>
          <p>Processing data from: {date}. Last updated: {time_update} UTC</p>
          {body}
          </body>
@@ -159,7 +159,7 @@ def main():
     directory.mkdir(parents=True, exist_ok=True)
 
     html_file = directory / Path(f"osa_status_{flat_date}.html")
-    html_file.write_text(html_content(html_table, date), encoding="utf-8")
+    html_file.write_text(html_content(html_table, date, "OSA Sequencer"), encoding="utf-8")
 
     log.info("Done")
 

osa/scripts/tests/test_osa_scripts.py

@@ -23,6 +23,7 @@ ALL_SCRIPTS = [
     "theta2_significance",
     "source_coordinates",
     "sequencer_webmaker",
+    "gainsel_webmaker",
 ]
 
 options.date = datetime.datetime.fromisoformat("2020-01-17")
@@ -397,3 +398,29 @@ def test_sequencer_webmaker(
     # Running without test option will make the script fail
     output = sp.run(["sequencer_webmaker", "-d", "2020-01-17"])
     assert output.returncode != 0
+
+
+def test_gainsel_webmaker(
+    base_test_dir,
+):
+
+    output = sp.run(["gainsel_webmaker", "-d", "2020-01-17"])
+    assert output.returncode == 0
+    directory = base_test_dir / "OSA" / "GainSelWeb"
+    expected_file = directory / "osa_gainsel_status_2020-01-17.html"
+    assert expected_file.exists()
+
+    # Test a date with non-existing run summary
+    output = sp.run(["gainsel_webmaker", "-d", "2024-01-12"])
+    assert output.returncode == 0
+    directory = base_test_dir / "OSA" / "GainSelWeb"
+    expected_file = directory / "osa_gainsel_status_2024-01-12.html"
+    assert expected_file.exists()
+
+
+def test_gainsel_web_content():
+    from osa.scripts.gainsel_webmaker import check_failed_jobs
+
+    table = check_failed_jobs(options.date)
+    assert table["GainSelStatus"][0] == "NOT STARTED"
+    assert table["GainSel%"][0] == 0.0

osa/tests/test_jobs.py

@@ -71,6 +71,7 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
         "#SBATCH --error=log/Run01809.%4a_jobid_%A.err",
         f'#SBATCH --partition={cfg.get("SLURM", "PARTITION_PEDCALIB")}',
         "#SBATCH --mem-per-cpu=3GB",
+        "#SBATCH --account=dpps",
     ]
     # Extract the second sequence
     second_sequence = sequence_list[1]
@@ -83,7 +84,8 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
         "#SBATCH --error=log/Run01807.%4a_jobid_%A.err",
         "#SBATCH --array=0-10",
         f'#SBATCH --partition={cfg.get("SLURM", "PARTITION_DATA")}',
-        "#SBATCH --mem-per-cpu=16GB",
+        "#SBATCH --mem-per-cpu=6GB",
+        "#SBATCH --account=dpps",
     ]
 
 
@@ -104,7 +106,8 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #SBATCH --output=log/Run01809.%4a_jobid_%A.out
     #SBATCH --error=log/Run01809.%4a_jobid_%A.err
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_PEDCALIB')}
-    #SBATCH --mem-per-cpu=3GB"""
+    #SBATCH --mem-per-cpu=3GB
+    #SBATCH --account=dpps"""
     )
     assert header == output_string1
 
@@ -122,7 +125,8 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #SBATCH --error=log/Run01807.%4a_jobid_%A.err
     #SBATCH --array=0-10
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
-    #SBATCH --mem-per-cpu=16GB"""
+    #SBATCH --mem-per-cpu=6GB
+    #SBATCH --account=dpps"""
     )
     assert header == output_string2
 
@@ -154,6 +158,7 @@ def test_create_job_template_scheduler(
     #SBATCH --array=0-10
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
     #SBATCH --mem-per-cpu={cfg.get('SLURM', 'MEMSIZE_DATA')}
+    #SBATCH --account={cfg.get('SLURM', 'ACCOUNT')}
 
     import os
     import subprocess
@@ -199,6 +204,7 @@ def test_create_job_template_scheduler(
         #SBATCH --array=0-8
         #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
         #SBATCH --mem-per-cpu={cfg.get('SLURM', 'MEMSIZE_DATA')}
+        #SBATCH --account={cfg.get('SLURM', 'ACCOUNT')}
 
         import os
         import subprocess

osa/utils/cliopts.py

@@ -280,6 +280,13 @@ def sequencer_argparser():
         default=False,
         help="Do not check if the gain selection finished correctly (default False)",
     )
+    parser.add_argument(
+        "-f",
+        "--force-submit",
+        action="store_true",
+        default=False,
+        help="Force sequencer to submit jobs"
+    )
     parser.add_argument(
         "tel_id",
         choices=["ST", "LST1", "LST2", "all"],
@@ -299,6 +306,7 @@ def sequencer_cli_parsing():
     options.no_calib = opts.no_calib
     options.no_dl2 = opts.no_dl2
     options.no_gainsel = opts.no_gainsel
+    options.force_submit = opts.force_submit
 
     log.debug(f"the options are {opts}")