lstosa 0.11.3__py3-none-any.whl → 0.11.5__py3-none-any.whl

{lstosa-0.11.3.dist-info → lstosa-0.11.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.3
+Version: 0.11.5
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.3.dist-info → lstosa-0.11.5.dist-info}/RECORD

@@ -1,9 +1,9 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=3HI3wVakaLT6OqNDadrX8IIIRO-nmIN4eu3bFgy5iM8,513
+osa/_version.py,sha256=ZyFxcHP5D9gYt6tCf5RRnkEsC8isbt9wg2nFTag5BOE,706
 osa/conftest.py,sha256=AdSHdWqjeMTgIeZEF8fdw6eSliySouivMWCPIqUV2os,24667
-osa/job.py,sha256=-W3t4_aP5dKRVy2BtyPE1eL4RRBfxIWYT_Ue88GrKXk,27866
+osa/job.py,sha256=CunwW7xA4mWEocS6KkDZ1K6h_LYh_ePZMyGHyCa6CKg,27863
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
-osa/paths.py,sha256=_3-avNABr7I2koiOo4MMDymi98l7JdIj1zUVbTjHdYg,18011
+osa/paths.py,sha256=B8-HA39YmThl2Dd9p3OFuYSYJf9MWg4Qdc0zmAhWjOE,18100
 osa/raw.py,sha256=ZNIsuqfx5ljoz_hwhSuafdKf-wr8-cxRJmel-A2endg,1337
 osa/report.py,sha256=sL2V7n8Y_UUaSDbWJY2o4UxDb4FU5AaFIRR8R25DB8o,4634
 osa/version.py,sha256=9T2TtuGBQeOy5PJDxMCeGlqx5baxLaq47VmFTDc09z8,796
@@ -12,7 +12,7 @@ osa/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/configs/config.py,sha256=cX0Vr4sorBVQ2KRqPIRSEGENKH_uC5lrgVHx1hp6YTk,1148
 osa/configs/datamodel.py,sha256=L_WRM91PBlMrtuE30akh7YR-56P0g9D994qzKSfhNJc,1950
 osa/configs/options.py,sha256=WbsyKhOs1Ud4Yt21O8KcISUvJYmhipa8vl_pxD_TZf4,558
-osa/configs/sequencer.cfg,sha256=VALe3KDii6UPOkEjI2ZsjV4nhswR8h_t4UUu85jWU4s,5624
+osa/configs/sequencer.cfg,sha256=49m-umbLo9w4gCv1oEdvFqukCh1py2nJDduOu28b_Qc,5652
 osa/high_level/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/high_level/selection_cuts.toml,sha256=ReSmcKtOPZY5JsZ9ExnxYdz7OrJEB8gghCbzHmeOyFg,128
 osa/high_level/significance.py,sha256=mMeG_y2wDnt0O2lSosYkSjaGZQl0V4GnuFrqKwSKSbE,9066
@@ -39,19 +39,19 @@ osa/scripts/calibration_pipeline.py,sha256=oWX2L9WIofVHp5qYsX3YMfwoslfiUgnXtWA2E
 osa/scripts/closer.py,sha256=vhoe9wkvPvMc5O_kXqAiBuWeqm_KrBpYzgb9zs9ZORw,21718
 osa/scripts/copy_datacheck.py,sha256=rAJu5-5oJxspynjYqVWLDKnXTF7Ib9xtzSf3_DiIHsk,3212
 osa/scripts/datasequence.py,sha256=0EZYlkX7ouaD66Ia2a5SWKbL0Fhlk8p4knKh0TG-2Gw,8592
-osa/scripts/gain_selection.py,sha256=PUG1J2V2SIboVKXJ-u4-h_XJyOGzfurazPu37M7lmhA,23238
+osa/scripts/gain_selection.py,sha256=1-wUCKuwI0W7yjja951sTXTKsGBl5YunLYcT1TwhCMQ,23240
 osa/scripts/gainsel_webmaker.py,sha256=40_DX7RUmImX-31iqRTFPEUvdVXXPSKp0xMEJWuAu80,5006
 osa/scripts/provprocess.py,sha256=Zv8sHTOgGNuEzu1QPUF142VQbHyfOsrxO073-xA6KG8,19107
 osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
 osa/scripts/reprocessing.py,sha256=IL30e2McFA0EGbzvkTXdzmKJGBTbafYlCA-PsafyzyY,4571
-osa/scripts/sequencer.py,sha256=2t-fcD1pTvYV54vFSxxKID-L7pkRg7geXo-E43cnL6M,11927
+osa/scripts/sequencer.py,sha256=hqkTXuYKvHHD6-s_wbzBulb2qpMF-pTLif0UOcIXu70,13150
 osa/scripts/sequencer_catB_tailcuts.py,sha256=zVBuCBiiIoO3HKHbJ0zqaDtuwxkxcziR4T5i-2WyDX8,10476
-osa/scripts/sequencer_webmaker.py,sha256=a9okC-l9UMJHELTLLM2Q933-JELFOhXEbbH1Kbp1E8g,4936
+osa/scripts/sequencer_webmaker.py,sha256=NpH5MrX2pgbVdi1ntp2xX5Yka7Gwp5PuoykkH5-UQq4,4936
 osa/scripts/show_run_summary.py,sha256=ofznLdoFHKQB-TELD6g5zMqD_TE9M9BuJR12zHvNdXE,2518
 osa/scripts/simulate_processing.py,sha256=D3b0kjyUS6Lm7pkQNJ-tJzQMveTwoPabfBBW80RMqN0,6828
 osa/scripts/update_source_catalog.py,sha256=Po4KSBOQCAT2Do1DUu5wnKV48Dq_pONfvtD47hh8fYI,8277
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=gyijYBzLQ19J1Az1mOdLm9bMxMwCkZryd78SfxCNtwg,15088
+osa/scripts/tests/test_osa_scripts.py,sha256=HaKVW-E2Gs6vvJoozIeCAg8-sxyu9k5zwNLFDPpCVrs,15116
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/tests/test_jobs.py,sha256=ozrMBjHujDhEMzEvJE7WynChqjbBTcORHsHmFXozeWk,16112
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
@@ -76,9 +76,9 @@ osa/workflow/dl3.py,sha256=ZMXPrdJA0WOxDiHEW9sdM2vmYix8R3eSMTGc6o42yxg,9860
 osa/workflow/stages.py,sha256=ILg91VyNXcy--si7CpDa1UFRiugqIH6nKl10Ru2zZVc,7422
 osa/workflow/tests/test_dl3.py,sha256=81Vt4lNxNjdKi_ehzymqfFWFjncw7GuQcix9S0Yigaw,384
 osa/workflow/tests/test_stages.py,sha256=aslfOIjf-rvf3r9N7NtimaOKkVB6K1M3bidgHOzRkMs,3985
-lstosa-0.11.3.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.11.3.dist-info/METADATA,sha256=8E9rqpM04-dAuH_l9y14_F_n3FojBSeIwQ8oJLk3xTA,7302
-lstosa-0.11.3.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-lstosa-0.11.3.dist-info/entry_points.txt,sha256=CzDKpLjZZQm8jJBxOVpMR8Czpgg_Yh-k6IPETp30VZE,1048
-lstosa-0.11.3.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.11.3.dist-info/RECORD,,
+lstosa-0.11.5.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.11.5.dist-info/METADATA,sha256=w662afh4R8FFG9Ln0iYtMWvvE8pCYXYcVj1rDezy_dA,7302
+lstosa-0.11.5.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+lstosa-0.11.5.dist-info/entry_points.txt,sha256=CzDKpLjZZQm8jJBxOVpMR8Czpgg_Yh-k6IPETp30VZE,1048
+lstosa-0.11.5.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.11.5.dist-info/RECORD,,

osa/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.11.3'
-__version_tuple__ = version_tuple = (0, 11, 3)
+__version__ = version = '0.11.5'
+__version_tuple__ = version_tuple = (0, 11, 5)
+
+__commit_id__ = commit_id = None

osa/configs/sequencer.cfg

@@ -44,6 +44,7 @@ PROD_ID: v0.1.0
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
 DL1_PROD_ID: tailcut84
+DL1_PROD_ID_PREFIX: tailcut
 DL2_PROD_ID: model2
 
 [lstchain]

osa/job.py

@@ -722,7 +722,7 @@ def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
 
     # Keep only the jobs corresponding to OSA sequences
     sacct_output = sacct_output[
-        (sacct_output["JobName"].str.contains("batch"))
+        (~sacct_output["JobID"].str.contains(r"\."))
         | (sacct_output["JobName"].str.contains("LST1"))
     ]
 

osa/paths.py

@@ -433,11 +433,12 @@ def get_dl1_prod_id(config_filename):
         
     picture_thresh = data["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
     boundary_thresh = data["tailcuts_clean_with_pedestal_threshold"]["boundary_thresh"]
+    dl1_prod_id_prefix = cfg.get("LST1", "DL1_PROD_ID_PREFIX")
 
     if boundary_thresh == 4:
-        return f"tailcut{picture_thresh}{boundary_thresh}"
+        return f"{dl1_prod_id_prefix}{picture_thresh}{boundary_thresh}"
     else:
-        return f"tailcut{picture_thresh}{boundary_thresh:02d}"
+        return f"{dl1_prod_id_prefix}{picture_thresh}{boundary_thresh:02d}"
 
 
 def get_dl2_nsb_prod_id(rf_model: Path) -> str:

osa/scripts/gain_selection.py

@@ -224,10 +224,10 @@ def launch_gainsel_for_data_run(
                             else:
                                 log.warning(f"Gain selection failed for run {run_id:05d}.{subrun:04d}")
                         elif gainsel_rc == "0":
-                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d},"
+                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d}, "
                                         "no additional jobs will be submitted for this subrun.") 
                 else:
-                    log.debug("Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
+                    log.debug(f"Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
                     if not simulate:
                         log_file = log_dir / f"r0_to_r0g_{run_id:05d}.{subrun:04d}.log"
                         job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"

osa/scripts/sequencer.py

@@ -11,7 +11,7 @@ import os
 import sys
 from decimal import Decimal
 import datetime
-
+import re
 from osa import osadb
 from osa.configs import options
 from osa.configs.config import cfg
@@ -50,6 +50,7 @@ __all__ = [
     "update_sequence_status",
     "get_status_for_sequence",
     "output_matrix",
+    "check_catB_status",
     "report_sequences",
     "update_job_info",
 ]
@@ -205,6 +206,34 @@ def update_sequence_status(seq_list):
             )
             seq.muonstatus = int(Decimal(get_status_for_sequence(seq, "MUON") * 100) / seq.subruns)
             seq.dl2status = int(Decimal(get_status_for_sequence(seq, "DL2") * 100))
+            seq.catbstatus = check_catB_status(seq)
+
+
+def check_catB_status(seq):
+    catbstatus = "None"
+
+    if seq.type == "DATA":
+        directory = options.directory
+
+        closed_files = list(directory.glob(f"catB*{seq.run}*.closed"))
+        if closed_files:
+            catbstatus = "CLOSED"
+        else:
+            log_files = list(options.log_directory.glob(f"catB_calibration_{seq.run}_*.err"))
+            if log_files:
+                filename = sorted(log_files)[-1].name
+                match = re.search(f"catB_calibration_{seq.run}_(\d+).err", filename)
+                if match:
+                    job_id = match.group(1)
+
+                    sacct_output = run_sacct(job_id)
+                    sacct_info = get_sacct_output(sacct_output)
+
+                    if not sacct_info.empty:
+                        catbstatus = sacct_info.iloc[0]["State"]
+
+    return catbstatus
+
 
 
 def get_status_for_sequence(sequence, data_level) -> int:
@@ -238,7 +267,10 @@ def get_status_for_sequence(sequence, data_level) -> int:
     elif data_level == "DATACHECK":
         try:
             directory = options.directory / sequence.dl1_prod_id
+            alternative_directory = destination_dir(concept="DATACHECK", create_dir=False, dl1_prod_id=sequence.dl1_prod_id)
             files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+            files += list(alternative_directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+            
         except AttributeError:
             return 0
         
@@ -275,7 +307,7 @@ def report_sequences(sequence_list):
         "Exit",
     ]
     if options.tel_id in ["LST1", "LST2"]:
-        header.extend(("DL1%", "MUONS%", "DL1AB%", "DATACHECK%", "DL2%"))
+        header.extend(("DL1%", "MUONS%", "CAT-B","DL1AB%", "DATACHECK%", "DL2%"))
     matrix = [header]
     for sequence in sequence_list:
         row_list = [
@@ -294,12 +326,13 @@ def report_sequences(sequence_list):
             sequence.exit,
         ]
         if sequence.type in ["DRS4", "PEDCALIB"]:
-            row_list.extend((None, None, None, None, None))
+            row_list.extend((None, None, None, None, None, None))
         elif sequence.type == "DATA":
             row_list.extend(
                 (
                     sequence.dl1status,
                     sequence.muonstatus,
+                    sequence.catbstatus,
                     sequence.dl1abstatus,
                     sequence.datacheckstatus,
                     sequence.dl2status,

osa/scripts/sequencer_webmaker.py

@@ -107,7 +107,7 @@ def lines_to_matrix(lines: Iterable) -> list:
     matrix = []
     for line in lines:
         l_fields = line.split()
-        if len(l_fields) == 18:
+        if len(l_fields) == 19:
             matrix.append(l_fields)
     return matrix
 

osa/scripts/tests/test_osa_scripts.py

@@ -163,10 +163,10 @@ def test_simulated_sequencer(
     assert rc.stdout == dedent(
         f"""\
         =================================== Starting sequencer.py at {now} UTC for LST, Telescope: LST1, Date: 2020-01-17 ===================================
-        Tel   Seq  Parent  Type      Run   Subruns  Source        Action  Tries  JobID  State  CPU_time  Exit  DL1%  MUONS%  DL1AB%  DATACHECK%  DL2%  
-        LST1    1  None    PEDCALIB  1809  5        None          None    None   None   None   None      None  None  None    None    None        None  
-        LST1    2       1  DATA      1807  11       Crab          None    None   None   None   None      None     0       0       0           0   100  
-        LST1    3       1  DATA      1808  9        MadeUpSource  None    None   None   None   None      None     0       0       0           0   100  
+        Tel   Seq  Parent  Type      Run   Subruns  Source        Action  Tries  JobID  State  CPU_time  Exit  DL1%  MUONS%  CAT-B  DL1AB%  DATACHECK%  DL2%  
+        LST1    1  None    PEDCALIB  1809  5        None          None    None   None   None   None      None  None  None    None   None    None        None  
+        LST1    2       1  DATA      1807  11       Crab          None    None   None   None   None      None     0       0  None        0           0   100  
+        LST1    3       1  DATA      1808  9        MadeUpSource  None    None   None   None   None      None     0       0  None        0           0   100  
         """  # noqa: E501
     )