lstosa 0.10.15__tar.gz → 0.10.17__tar.gz

{lstosa-0.10.15 → lstosa-0.10.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.15
+Version: 0.10.17
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.15 → lstosa-0.10.17}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.15
+Version: 0.10.17
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.15 → lstosa-0.10.17}/src/osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.15'
-__version_tuple__ = version_tuple = (0, 10, 15)
+__version__ = version = '0.10.17'
+__version_tuple__ = version_tuple = (0, 10, 17)

{lstosa-0.10.15 → lstosa-0.10.17}/src/osa/scripts/closer.py

@@ -336,6 +336,7 @@ def merge_dl1_datacheck(seq_list) -> List[str]:
 
     muons_dir = destination_dir("MUON", create_dir=False)
     datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     list_job_id = []
 
@@ -344,6 +345,7 @@ def merge_dl1_datacheck(seq_list) -> List[str]:
             cmd = [
                 "sbatch",
                 "--parsable",
+                f"--account={slurm_account}",
                 "-D",
                 options.directory,
                 "-o",
@@ -386,6 +388,7 @@ def extract_provenance(seq_list):
     log.info("Extract provenance run wise")
 
     nightdir = date_to_dir(options.date)
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     for sequence in seq_list:
         if sequence.type == "DATA":
@@ -393,6 +396,7 @@ def extract_provenance(seq_list):
             pedcal_run_id = str(sequence.pedcal_run)
             cmd = [
                 "sbatch",
+                f"--account={slurm_account}",
                 "-D",
                 options.directory,
                 "-o",
@@ -433,6 +437,7 @@ def merge_files(sequence_list, data_level="DL2"):
 
     data_dir = destination_dir(data_level, create_dir=False)
     pattern, prefix = get_pattern(data_level)
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     for sequence in sequence_list:
         if sequence.type == "DATA":
@@ -440,6 +445,7 @@ def merge_files(sequence_list, data_level="DL2"):
 
             cmd = [
                 "sbatch",
+                f"--account={slurm_account}",
                 "-D",
                 options.directory,
                 "-o",
@@ -467,12 +473,14 @@ def merge_muon_files(sequence_list):
 
     data_dir = destination_dir("MUON", create_dir=False)
     pattern, prefix = get_pattern("MUON")
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     for sequence in sequence_list:
         merged_file = Path(data_dir) / f"muons_LST-1.Run{sequence.run:05d}.fits"
 
         cmd = [
             "sbatch",
+            f"--account={slurm_account}",
             "-D",
             options.directory,
             "-o",
@@ -499,10 +507,12 @@ def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     return [
         "sbatch",
         "--parsable",
+        f"--account={slurm_account}",
         "-D",
         options.directory,
         "-o",
@@ -541,10 +551,12 @@ def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
     datacheck_dir = destination_dir("DATACHECK", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     return [
         "sbatch",
         "--parsable",
+        f"--account={slurm_account}",
         "-D",
         options.directory,
         "-o",

{lstosa-0.10.15 → lstosa-0.10.17}/src/osa/scripts/gain_selection.py

@@ -114,6 +114,7 @@ def get_sbatch_script(
 ):
     """Build the sbatch job pilot script for running the gain selection."""
     mem_per_job = cfg.get("SLURM", "MEMSIZE_GAINSEL")
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
     sbatch_script = dedent(
             f"""\
         #!/bin/bash
@@ -123,6 +124,7 @@ def get_sbatch_script(
         #SBATCH --job-name "gain_selection_{run_id:05d}"
         #SBATCH --partition=short,long
         #SBATCH --mem={mem_per_job}
+        #SBATCH --account={slurm_account}
         """
         )
     
@@ -380,13 +382,20 @@ def GainSel_finished(date: datetime) -> bool:
     """Check if gain selection finished successfully."""
     flagfile = GainSel_flag_file(date)
     return flagfile.exists()
-
+   
 
 def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool:
     """Search for failed jobs in the log directory."""
     base_dir = Path(cfg.get("LST1", "BASE"))
     log_dir = base_dir / f"R0G/log/{date_to_dir(date)}"
-    history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history")
+    history_files = list(log_dir.glob(f"gain_selection_{run_id:05d}.????.history"))
+    summary_table = run_summary_table(date)
+    n_subruns = summary_table[summary_table["run_id"] == run_id]["n_subruns"]
+    
+    if len(history_files) != n_subruns:
+        log.debug(f"All history files of run {run_id} were not created yet")
+        return False
+
     failed_subruns = []
     log.info(f"Checking all history files of run {run_id}")
     

{lstosa-0.10.15 → lstosa-0.10.17}/src/osa/scripts/sequencer.py

@@ -119,6 +119,11 @@ def single_process(telescope):
             log.info(f"Sequencer already finished for date {date_to_iso(options.date)}. Exiting")
             sys.exit(0)
 
+        elif timeout_in_sequencer(options.date) and not options.force_submit:
+            log.info(f"Some jobs of sequencer finished in TIMEOUT for date {date_to_iso(options.date)}."
+                " Please relaunch the affected sequences manually.")
+            sys.exit(0)
+
     # Build the sequences
     sequence_list = build_sequences(options.date)
 
@@ -332,7 +337,7 @@ def is_sequencer_running(date: datetime.datetime) -> bool:
 
 
 def is_sequencer_completed(date: datetime.datetime) -> bool:
-    """Check if the jobs launched by sequencer are running or pending for the given date."""
+    """Check if the jobs launched by sequencer are already completed."""
     summary_table = run_summary_table(date)
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
     sacct_output = run_sacct()
@@ -350,5 +355,24 @@ def is_sequencer_completed(date: datetime.datetime) -> bool:
     return True
 
 
+def timeout_in_sequencer(date: datetime.datetime) -> bool:
+    """Check if any of the jobs launched by sequencer finished in timeout."""
+    summary_table = run_summary_table(date)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    sacct_output = run_sacct()
+    sacct_info = get_sacct_output(sacct_output)
+
+    for run in data_runs["run_id"]:
+        jobs_run = sacct_info[sacct_info["JobName"]==f"LST1_{run:05d}"]
+        if len(jobs_run["JobID"].unique())>1:
+            last_job_id = sorted(jobs_run["JobID"].unique())[-1]
+            jobs_run = sacct_info[sacct_info["JobID"]==last_job_id]
+        timeout_jobs = jobs_run[(jobs_run["State"] == "TIMEOUT")]
+        if len(timeout_jobs) != 0:
+            return True
+
+    return False
+
+
 if __name__ == "__main__":
     main()

{lstosa-0.10.15 → lstosa-0.10.17}/src/osa/scripts/tests/test_osa_scripts.py

@@ -8,6 +8,7 @@ import pytest
 import yaml
 
 from osa.configs import options
+from osa.configs.config import cfg
 from osa.scripts.closer import is_sequencer_successful, is_finished_check
 
 ALL_SCRIPTS = [
@@ -326,10 +327,12 @@ def test_daily_longterm_cmd():
 
     job_ids = ["12345", "54321"]
     cmd = daily_longterm_cmd(parent_job_ids=job_ids)
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     expected_cmd = [
         "sbatch",
         "--parsable",
+        f"--account={slurm_account}",
         "-D",
         options.directory,
         "-o",

{lstosa-0.10.15 → lstosa-0.10.17}/src/osa/workflow/stages.py

@@ -75,15 +75,15 @@ class AnalysisStage:
         reproduced in subsequent trials.
         """
 
-        if self.command == "lstchain_data_r0_to_dl1":
+        if self.command == cfg.get("lstchain", "r0_to_dl1"):
             self._remove_dl1a_output()
-        elif self.command == "lstchain_dl1ab":
+        elif self.command == cfg.get("lstchain", "dl1ab"):
             self._remove_dl1b_output('dl1_LST-1.Run')
-        elif self.command == "lstchain_check_dl1":
+        elif self.command == cfg.get("lstchain", "check_dl1"):
             self._remove_dl1b_output('datacheck_dl1_LST-1.Run')
-        elif self.command == "onsite_create_calibration_file":
+        elif self.command == cfg.get("lstchain", "charge_calibration"):
             self._remove_calibration()
-        elif self.command == "onsite_create_drs4_pedestal_file":
+        elif self.command == cfg.get("lstchain", "drs4_baseline"):
             self._remove_drs4_baseline()
 
     def _remove_drs4_baseline(self):
@@ -121,9 +121,16 @@ class AnalysisStage:
 
     def _write_checkpoint(self):
         """Write the checkpoint in the history file."""
+        command_to_prod_id = {
+            cfg.get("lstchain", "r0_to_dl1"): options.prod_id,
+            cfg.get("lstchain", "dl1ab"): options.dl1_prod_id,
+            cfg.get("lstchain", "check_dl1"): options.dl1_prod_id,
+            cfg.get("lstchain", "dl1_to_dl2"): options.dl2_prod_id
+        }
+        prod_id = command_to_prod_id.get(self.command)
         history(
             run=self.run,
-            prod_id=options.prod_id,
+            prod_id=prod_id,
             stage=self.command,
             return_code=self.rc,
             history_file=self.history_file,