lstosa 0.10.14__py3-none-any.whl → 0.10.16__py3-none-any.whl

osa/scripts/gainsel_webmaker.py

@@ -0,0 +1,157 @@
+import logging
+from argparse import ArgumentParser
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import pandas as pd
+from astropy.table import Table
+
+from osa.configs import options
+from osa.configs.config import cfg
+from osa.nightsummary.nightsummary import run_summary_table
+from osa.paths import DEFAULT_CFG
+from osa.scripts.sequencer_webmaker import html_content
+from osa.utils.utils import date_to_dir, date_to_iso
+
+log = logging.getLogger(__name__)
+
+
+def valid_date(string):
+    """Check if the string is a valid date and return a datetime object."""
+    return datetime.strptime(string, "%Y-%m-%d")
+
+
+common_parser = ArgumentParser(add_help=False)
+common_parser.add_argument(
+    "-c",
+    "--config",
+    type=Path,
+    default=DEFAULT_CFG,
+    help="Use specific config file [default configs/sequencer.cfg]",
+)
+common_parser.add_argument(
+    "-d",
+    "--date",
+    help="Date of the start of the night in ISO format (YYYY-MM-DD). Defaults to yesterday",
+    type=valid_date,
+)
+
+
+def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool:
+    """Search for failed jobs in the log directory."""
+    base_dir = Path(cfg.get("LST1", "BASE"))
+    flat_date = date_to_dir(date)
+    log_dir = base_dir / f"R0G/log/{flat_date}"
+    history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history")
+
+    success_subruns = 0
+    failed_subruns = 0
+    pending_subruns = 0
+
+    for file in history_files:
+        if file.read_text() != "":
+            gainsel_rc = file.read_text().splitlines()[-1][-1]
+
+            if gainsel_rc == "1":
+                failed_subruns += 1
+
+            elif gainsel_rc == "0":
+                success_subruns += 1
+
+        else:
+            pending_subruns += 1
+
+    return {"pending": pending_subruns, "success": success_subruns, "failed": failed_subruns}
+
+
+def check_failed_jobs(date: datetime) -> pd.DataFrame:
+    """Search for failed jobs in the log directory."""
+    summary_table = run_summary_table(date)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+
+    gainsel_status_dict = {}
+    for run in data_runs:
+        run_id = run["run_id"]
+        gainsel_job_status = check_gainsel_jobs_runwise(date, run_id)
+        gainsel_status_dict[run_id] = gainsel_job_status
+
+    gainsel_df = pd.DataFrame(gainsel_status_dict.values(), index=gainsel_status_dict.keys())
+    gainsel_df.reset_index(inplace=True)
+    gainsel_df.rename(columns={"index": "run_id"}, inplace=True)
+    summary_table = summary_table.to_pandas()
+
+    final_table = pd.merge(summary_table, gainsel_df, on="run_id")[
+        [
+            "run_id",
+            "n_subruns",
+            "pending",
+            "success",
+            "failed",
+        ]
+    ]
+
+    def determine_status(row):
+        if row["failed"] > 0:
+            return "FAILED"
+        elif row["pending"] == row["n_subruns"]:
+            return "PENDING"
+        elif row["success"] == row["n_subruns"]:
+            return "COMPLETED"
+        elif row["pending"] > 0:
+            return "RUNNING"
+        else:
+            return "NOT STARTED"
+
+    final_table["GainSel%"] = round(final_table["success"] * 100 / final_table["n_subruns"])
+    final_table["GainSelStatus"] = final_table.apply(determine_status, axis=1)
+
+    return final_table
+
+
+def main():
+    """Produce the html file with the processing OSA Gain Selection status.
+
+    It creates an HTML file osa_gainsel_status_YYYY-MM-DD.html
+    """
+    args = ArgumentParser(
+        description=(
+            "Script to create an HTML file with the gain selection status "
+            "(osa_gainsel_status_YYYY-MM-DD.html)"
+        ),
+        parents=[common_parser],
+    ).parse_args()
+
+    if args.date:
+        flat_date = date_to_dir(args.date)
+        options.date = args.date
+
+    else:
+        # yesterday by default
+        yesterday = datetime.now() - timedelta(days=1)
+        options.date = yesterday
+        flat_date = date_to_dir(yesterday)
+
+    date = date_to_iso(options.date)
+    run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR"))
+    run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv"
+
+    gain_selection_web_directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR"))
+    gain_selection_web_directory.mkdir(parents=True, exist_ok=True)
+    html_file = gain_selection_web_directory / f"osa_gainsel_status_{date}.html"
+
+    # Create and save the HTML file
+    if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0:
+        content = "<p>No data found</p>"
+        log.warning(f"No data found for date {date}, creating an empty HTML file.")
+
+    else:
+        # Get the table with the gain selection check report in HTML format:
+        table_gain_selection_jobs = check_failed_jobs(options.date)
+        content = table_gain_selection_jobs.to_html(justify="left")
+
+    html_file.write_text(html_content(content, date, "OSA Gain Selection"))
+    log.info(f"Created HTML file {html_file}")
+
+
+if __name__ == "__main__":
+    main()

osa/scripts/sequencer.py

@@ -9,6 +9,7 @@ import logging
 import os
 import sys
 from decimal import Decimal
+import datetime
 
 from osa import osadb
 from osa.configs import options
@@ -28,7 +29,7 @@ from osa.paths import analysis_path
 from osa.report import start
 from osa.utils.cliopts import sequencer_cli_parsing
 from osa.utils.logging import myLogger
-from osa.utils.utils import is_day_closed, gettag, date_to_iso, date_to_dir
+from osa.utils.utils import is_day_closed, gettag, date_to_iso
 from osa.veto import get_closed_list, get_veto_list
 from osa.scripts.gain_selection import GainSel_finished
 
@@ -98,9 +99,9 @@ def single_process(telescope):
         log.warning("No runs found for this date. Nothing to do. Exiting.")
         sys.exit(0)
 
-    if not options.no_gainsel and not GainSel_finished(date_to_dir(options.date)):
+    if not options.no_gainsel and not GainSel_finished(options.date):
         log.info(
-            f"Gain selection did not finish successfully for date {options.date}."
+            f"Gain selection did not finish successfully for date {date_to_iso(options.date)}. "
             "Try again later, once gain selection has finished."
         )
         sys.exit()
@@ -109,6 +110,20 @@ def single_process(telescope):
         log.info(f"Date {date_to_iso(options.date)} is already closed for {options.tel_id}")
         return sequence_list
 
+    if not options.test and not options.simulate:
+        if is_sequencer_running(options.date):
+            log.info(f"Sequencer is still running for date {date_to_iso(options.date)}. Try again later.")
+            sys.exit(0)
+
+        elif is_sequencer_completed(options.date) and not options.force_submit:
+            log.info(f"Sequencer already finished for date {date_to_iso(options.date)}. Exiting")
+            sys.exit(0)
+
+        elif timeout_in_sequencer(options.date) and not options.force_submit:
+            log.info(f"Some jobs of sequencer finished in TIMEOUT for date {date_to_iso(options.date)}."
+                " Please relaunch the affected sequences manually.")
+            sys.exit(0)
+
     # Build the sequences
     sequence_list = build_sequences(options.date)
 
@@ -306,5 +321,58 @@ def output_matrix(matrix: list, padding_space: int):
         log.info(stringrow)
 
 
+def is_sequencer_running(date: datetime.datetime) -> bool:
+    """Check if the jobs launched by sequencer are running or pending for the given date."""
+    summary_table = run_summary_table(date)
+    sacct_output = run_sacct()
+    sacct_info = get_sacct_output(sacct_output)
+
+    for run in summary_table["run_id"]:
+        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
+        queued_jobs = jobs_run[(jobs_run["State"] == "RUNNING") | (jobs_run["State"] == "PENDING")]
+        if len(queued_jobs) != 0:
+            return True
+
+    return False
+
+
+def is_sequencer_completed(date: datetime.datetime) -> bool:
+    """Check if the jobs launched by sequencer are already completed."""
+    summary_table = run_summary_table(date)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    sacct_output = run_sacct()
+    sacct_info = get_sacct_output(sacct_output)
+
+    for run in data_runs["run_id"]:
+        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
+        if len(jobs_run["JobID"].unique())>1:
+            last_job_id = sorted(jobs_run["JobID"].unique())[-1]
+            jobs_run = sacct_info[sacct_info["JobID"]==last_job_id]
+        incomplete_jobs = jobs_run[(jobs_run["State"] != "COMPLETED")]
+        if len(jobs_run) == 0 or len(incomplete_jobs) != 0:
+            return False
+
+    return True
+
+
+def timeout_in_sequencer(date: datetime.datetime) -> bool:
+    """Check if any of the jobs launched by sequencer finished in timeout."""
+    summary_table = run_summary_table(date)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    sacct_output = run_sacct()
+    sacct_info = get_sacct_output(sacct_output)
+
+    for run in data_runs["run_id"]:
+        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
+        if len(jobs_run["JobID"].unique())>1:
+            last_job_id = sorted(jobs_run["JobID"].unique())[-1]
+            jobs_run = sacct_info[sacct_info["JobID"]==last_job_id]
+        timeout_jobs = jobs_run[(jobs_run["State"] == "TIMEOUT")]
+        if len(timeout_jobs) != 0:
+            return True
+
+    return False
+
+
 if __name__ == "__main__":
     main()

osa/scripts/sequencer_webmaker.py

@@ -20,7 +20,7 @@ from osa.utils.utils import is_day_closed, date_to_iso, date_to_dir
 log = myLogger(logging.getLogger())
 
 
-def html_content(body: str, date: str) -> str:
+def html_content(body: str, date: str, title: str) -> str:
     """Build the HTML content.
 
     Parameters
@@ -43,11 +43,11 @@ def html_content(body: str, date: str) -> str:
         <html xmlns="http://www.w3.org/1999/xhtml">
          <head>
           <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-          <title>OSA Sequencer status</title><link href="osa.css" rel="stylesheet"
+          <title>{title} status</title><link href="osa.css" rel="stylesheet"
           type="text/css" /><style>table{{width:152ex;}}</style>
          </head>
          <body>
-         <h1>OSA processing status</h1>
+         <h1>{title} processing status</h1>
          <p>Processing data from: {date}. Last updated: {time_update} UTC</p>
          {body}
          </body>
@@ -159,7 +159,7 @@ def main():
     directory.mkdir(parents=True, exist_ok=True)
 
     html_file = directory / Path(f"osa_status_{flat_date}.html")
-    html_file.write_text(html_content(html_table, date), encoding="utf-8")
+    html_file.write_text(html_content(html_table, date, "OSA Sequencer"), encoding="utf-8")
 
     log.info("Done")
 

osa/scripts/tests/test_osa_scripts.py

@@ -23,6 +23,7 @@ ALL_SCRIPTS = [
     "theta2_significance",
     "source_coordinates",
     "sequencer_webmaker",
+    "gainsel_webmaker",
 ]
 
 options.date = datetime.datetime.fromisoformat("2020-01-17")
@@ -397,3 +398,29 @@ def test_sequencer_webmaker(
     # Running without test option will make the script fail
     output = sp.run(["sequencer_webmaker", "-d", "2020-01-17"])
     assert output.returncode != 0
+
+
+def test_gainsel_webmaker(
+    base_test_dir,
+):
+
+    output = sp.run(["gainsel_webmaker", "-d", "2020-01-17"])
+    assert output.returncode == 0
+    directory = base_test_dir / "OSA" / "GainSelWeb"
+    expected_file = directory / "osa_gainsel_status_2020-01-17.html"
+    assert expected_file.exists()
+
+    # Test a date with non-existing run summary
+    output = sp.run(["gainsel_webmaker", "-d", "2024-01-12"])
+    assert output.returncode == 0
+    directory = base_test_dir / "OSA" / "GainSelWeb"
+    expected_file = directory / "osa_gainsel_status_2024-01-12.html"
+    assert expected_file.exists()
+
+
+def test_gainsel_web_content():
+    from osa.scripts.gainsel_webmaker import check_failed_jobs
+
+    table = check_failed_jobs(options.date)
+    assert table["GainSelStatus"][0] == "NOT STARTED"
+    assert table["GainSel%"][0] == 0.0

osa/tests/test_jobs.py

@@ -71,6 +71,7 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
         "#SBATCH --error=log/Run01809.%4a_jobid_%A.err",
         f'#SBATCH --partition={cfg.get("SLURM", "PARTITION_PEDCALIB")}',
         "#SBATCH --mem-per-cpu=3GB",
+        "#SBATCH --account=dpps",
     ]
     # Extract the second sequence
     second_sequence = sequence_list[1]
@@ -83,7 +84,8 @@ def test_scheduler_env_variables(sequence_list, running_analysis_dir):
         "#SBATCH --error=log/Run01807.%4a_jobid_%A.err",
         "#SBATCH --array=0-10",
         f'#SBATCH --partition={cfg.get("SLURM", "PARTITION_DATA")}',
-        "#SBATCH --mem-per-cpu=16GB",
+        "#SBATCH --mem-per-cpu=6GB",
+        "#SBATCH --account=dpps",
     ]
 
 
@@ -104,7 +106,8 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #SBATCH --output=log/Run01809.%4a_jobid_%A.out
     #SBATCH --error=log/Run01809.%4a_jobid_%A.err
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_PEDCALIB')}
-    #SBATCH --mem-per-cpu=3GB"""
+    #SBATCH --mem-per-cpu=3GB
+    #SBATCH --account=dpps"""
     )
     assert header == output_string1
 
@@ -122,7 +125,8 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     #SBATCH --error=log/Run01807.%4a_jobid_%A.err
     #SBATCH --array=0-10
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
-    #SBATCH --mem-per-cpu=16GB"""
+    #SBATCH --mem-per-cpu=6GB
+    #SBATCH --account=dpps"""
     )
     assert header == output_string2
 
@@ -154,6 +158,7 @@ def test_create_job_template_scheduler(
     #SBATCH --array=0-10
     #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
     #SBATCH --mem-per-cpu={cfg.get('SLURM', 'MEMSIZE_DATA')}
+    #SBATCH --account={cfg.get('SLURM', 'ACCOUNT')}
 
     import os
     import subprocess
@@ -199,6 +204,7 @@ def test_create_job_template_scheduler(
         #SBATCH --array=0-8
         #SBATCH --partition={cfg.get('SLURM', 'PARTITION_DATA')}
         #SBATCH --mem-per-cpu={cfg.get('SLURM', 'MEMSIZE_DATA')}
+        #SBATCH --account={cfg.get('SLURM', 'ACCOUNT')}
 
         import os
         import subprocess

osa/utils/cliopts.py

@@ -280,6 +280,13 @@ def sequencer_argparser():
         default=False,
         help="Do not check if the gain selection finished correctly (default False)",
     )
+    parser.add_argument(
+        "-f",
+        "--force-submit",
+        action="store_true",
+        default=False,
+        help="Force sequencer to submit jobs"
+    )
     parser.add_argument(
         "tel_id",
         choices=["ST", "LST1", "LST2", "all"],
@@ -299,6 +306,7 @@ def sequencer_cli_parsing():
     options.no_calib = opts.no_calib
     options.no_dl2 = opts.no_dl2
     options.no_gainsel = opts.no_gainsel
+    options.force_submit = opts.force_submit
 
     log.debug(f"the options are {opts}")
 

osa/workflow/stages.py

@@ -75,15 +75,15 @@ class AnalysisStage:
         reproduced in subsequent trials.
         """
 
-        if self.command == "lstchain_data_r0_to_dl1":
+        if self.command == cfg.get("lstchain", "r0_to_dl1"):
             self._remove_dl1a_output()
-        elif self.command == "lstchain_dl1ab":
+        elif self.command == cfg.get("lstchain", "dl1ab"):
             self._remove_dl1b_output('dl1_LST-1.Run')
-        elif self.command == "lstchain_check_dl1":
+        elif self.command == cfg.get("lstchain", "check_dl1"):
             self._remove_dl1b_output('datacheck_dl1_LST-1.Run')
-        elif self.command == "onsite_create_calibration_file":
+        elif self.command == cfg.get("lstchain", "charge_calibration"):
             self._remove_calibration()
-        elif self.command == "onsite_create_drs4_pedestal_file":
+        elif self.command == cfg.get("lstchain", "drs4_baseline"):
             self._remove_drs4_baseline()
 
     def _remove_drs4_baseline(self):
@@ -121,9 +121,16 @@ class AnalysisStage:
 
     def _write_checkpoint(self):
         """Write the checkpoint in the history file."""
+        command_to_prod_id = {
+            cfg.get("lstchain", "r0_to_dl1"): options.prod_id,
+            cfg.get("lstchain", "dl1ab"): options.dl1_prod_id,
+            cfg.get("lstchain", "check_dl1"): options.dl1_prod_id,
+            cfg.get("lstchain", "dl1_to_dl2"): options.dl2_prod_id
+        }
+        prod_id = command_to_prod_id.get(self.command)
         history(
             run=self.run,
-            prod_id=options.prod_id,
+            prod_id=prod_id,
             stage=self.command,
             return_code=self.rc,
             history_file=self.history_file,