lstosa 0.11.3__tar.gz → 0.11.5__tar.gz

{lstosa-0.11.3 → lstosa-0.11.5}/.mailmap

@@ -1,6 +1,7 @@
 Andres Baquero <obaquero@ucm.es> andres-baquero <obaquero@ucm.es>
 
-Daniel Morcuende <dmorcuen@ucm.es> morcuended <dmorcuen@ucm.es>
+Daniel Morcuende <daniel.morcuende@cta-observatory.org> morcuended <dmorcuen@ucm.es>
+Daniel Morcuende <daniel.morcuende@cta-observatory.org> morcuended <dmorcuende@iaa.es>
 
 Jose Luis Contreras <jlcontreras@fis.ucm.es> Jose Contreras <jose.contreras@cp02.ctan.cta-observatory.org>
 Jose Luis Contreras <jlcontreras@fis.ucm.es> Jose Luis Contreras <pda.jl.contreras@gmail.com>

{lstosa-0.11.3 → lstosa-0.11.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.3
+Version: 0.11.5
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.3 → lstosa-0.11.5}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.3
+Version: 0.11.5
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.11.3'
-__version_tuple__ = version_tuple = (0, 11, 3)
+__version__ = version = '0.11.5'
+__version_tuple__ = version_tuple = (0, 11, 5)
+
+__commit_id__ = commit_id = 'gdc0cee0a4'

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/configs/sequencer.cfg

@@ -44,6 +44,7 @@ PROD_ID: v0.1.0
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
 DL1_PROD_ID: tailcut84
+DL1_PROD_ID_PREFIX: tailcut
 DL2_PROD_ID: model2
 
 [lstchain]

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/job.py

@@ -722,7 +722,7 @@ def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
 
     # Keep only the jobs corresponding to OSA sequences
     sacct_output = sacct_output[
-        (sacct_output["JobName"].str.contains("batch"))
+        (~sacct_output["JobID"].str.contains(r"\."))
         | (sacct_output["JobName"].str.contains("LST1"))
     ]
 

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/paths.py

@@ -433,11 +433,12 @@ def get_dl1_prod_id(config_filename):
         
     picture_thresh = data["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
     boundary_thresh = data["tailcuts_clean_with_pedestal_threshold"]["boundary_thresh"]
+    dl1_prod_id_prefix = cfg.get("LST1", "DL1_PROD_ID_PREFIX")
 
     if boundary_thresh == 4:
-        return f"tailcut{picture_thresh}{boundary_thresh}"
+        return f"{dl1_prod_id_prefix}{picture_thresh}{boundary_thresh}"
     else:
-        return f"tailcut{picture_thresh}{boundary_thresh:02d}"
+        return f"{dl1_prod_id_prefix}{picture_thresh}{boundary_thresh:02d}"
 
 
 def get_dl2_nsb_prod_id(rf_model: Path) -> str:

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/scripts/gain_selection.py

@@ -224,10 +224,10 @@ def launch_gainsel_for_data_run(
                             else:
                                 log.warning(f"Gain selection failed for run {run_id:05d}.{subrun:04d}")
                         elif gainsel_rc == "0":
-                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d},"
+                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d}, "
                                         "no additional jobs will be submitted for this subrun.") 
                 else:
-                    log.debug("Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
+                    log.debug(f"Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
                     if not simulate:
                         log_file = log_dir / f"r0_to_r0g_{run_id:05d}.{subrun:04d}.log"
                         job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/scripts/sequencer.py

@@ -11,7 +11,7 @@ import os
 import sys
 from decimal import Decimal
 import datetime
-
+import re
 from osa import osadb
 from osa.configs import options
 from osa.configs.config import cfg
@@ -50,6 +50,7 @@ __all__ = [
     "update_sequence_status",
     "get_status_for_sequence",
     "output_matrix",
+    "check_catB_status",
     "report_sequences",
     "update_job_info",
 ]
@@ -205,6 +206,34 @@ def update_sequence_status(seq_list):
             )
             seq.muonstatus = int(Decimal(get_status_for_sequence(seq, "MUON") * 100) / seq.subruns)
             seq.dl2status = int(Decimal(get_status_for_sequence(seq, "DL2") * 100))
+            seq.catbstatus = check_catB_status(seq)
+
+
+def check_catB_status(seq):
+    catbstatus = "None"
+
+    if seq.type == "DATA":
+        directory = options.directory
+
+        closed_files = list(directory.glob(f"catB*{seq.run}*.closed"))
+        if closed_files:
+            catbstatus = "CLOSED"
+        else:
+            log_files = list(options.log_directory.glob(f"catB_calibration_{seq.run}_*.err"))
+            if log_files:
+                filename = sorted(log_files)[-1].name
+                match = re.search(f"catB_calibration_{seq.run}_(\d+).err", filename)
+                if match:
+                    job_id = match.group(1)
+
+                    sacct_output = run_sacct(job_id)
+                    sacct_info = get_sacct_output(sacct_output)
+
+                    if not sacct_info.empty:
+                        catbstatus = sacct_info.iloc[0]["State"]
+
+    return catbstatus
+
 
 
 def get_status_for_sequence(sequence, data_level) -> int:
@@ -238,7 +267,10 @@ def get_status_for_sequence(sequence, data_level) -> int:
     elif data_level == "DATACHECK":
         try:
             directory = options.directory / sequence.dl1_prod_id
+            alternative_directory = destination_dir(concept="DATACHECK", create_dir=False, dl1_prod_id=sequence.dl1_prod_id)
             files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+            files += list(alternative_directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+            
         except AttributeError:
             return 0
         
@@ -275,7 +307,7 @@ def report_sequences(sequence_list):
         "Exit",
     ]
     if options.tel_id in ["LST1", "LST2"]:
-        header.extend(("DL1%", "MUONS%", "DL1AB%", "DATACHECK%", "DL2%"))
+        header.extend(("DL1%", "MUONS%", "CAT-B","DL1AB%", "DATACHECK%", "DL2%"))
     matrix = [header]
     for sequence in sequence_list:
         row_list = [
@@ -294,12 +326,13 @@ def report_sequences(sequence_list):
             sequence.exit,
         ]
         if sequence.type in ["DRS4", "PEDCALIB"]:
-            row_list.extend((None, None, None, None, None))
+            row_list.extend((None, None, None, None, None, None))
         elif sequence.type == "DATA":
             row_list.extend(
                 (
                     sequence.dl1status,
                     sequence.muonstatus,
+                    sequence.catbstatus,
                     sequence.dl1abstatus,
                     sequence.datacheckstatus,
                     sequence.dl2status,

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/scripts/sequencer_webmaker.py

@@ -107,7 +107,7 @@ def lines_to_matrix(lines: Iterable) -> list:
     matrix = []
     for line in lines:
         l_fields = line.split()
-        if len(l_fields) == 18:
+        if len(l_fields) == 19:
             matrix.append(l_fields)
     return matrix
 

{lstosa-0.11.3 → lstosa-0.11.5}/src/osa/scripts/tests/test_osa_scripts.py

@@ -163,10 +163,10 @@ def test_simulated_sequencer(
     assert rc.stdout == dedent(
         f"""\
         =================================== Starting sequencer.py at {now} UTC for LST, Telescope: LST1, Date: 2020-01-17 ===================================
-        Tel   Seq  Parent  Type      Run   Subruns  Source        Action  Tries  JobID  State  CPU_time  Exit  DL1%  MUONS%  DL1AB%  DATACHECK%  DL2%  
-        LST1    1  None    PEDCALIB  1809  5        None          None    None   None   None   None      None  None  None    None    None        None  
-        LST1    2       1  DATA      1807  11       Crab          None    None   None   None   None      None     0       0       0           0   100  
-        LST1    3       1  DATA      1808  9        MadeUpSource  None    None   None   None   None      None     0       0       0           0   100  
+        Tel   Seq  Parent  Type      Run   Subruns  Source        Action  Tries  JobID  State  CPU_time  Exit  DL1%  MUONS%  CAT-B  DL1AB%  DATACHECK%  DL2%  
+        LST1    1  None    PEDCALIB  1809  5        None          None    None   None   None   None      None  None  None    None   None    None        None  
+        LST1    2       1  DATA      1807  11       Crab          None    None   None   None   None      None     0       0  None        0           0   100  
+        LST1    3       1  DATA      1808  9        MadeUpSource  None    None   None   None   None      None     0       0  None        0           0   100  
         """  # noqa: E501
     )