lstosa 0.10.7__py3-none-any.whl → 0.10.9__py3-none-any.whl

{lstosa-0.10.7.dist-info → lstosa-0.10.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.7
+Version: 0.10.9
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,7 +20,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy ~=5.0
-Requires-Dist: lstchain >=0.10.5
+Requires-Dist: lstchain >=0.10.7
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas

{lstosa-0.10.7.dist-info → lstosa-0.10.9.dist-info}/RECORD

@@ -1,9 +1,9 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=Tqz9jfXJ9oRFN5_F1Y4Bx09mLTPY4KYQNRrHQQNprBE,413
-osa/conftest.py,sha256=b_26FciV8NZAdHj2S10E8cE4bettinLJY83Roe1a5L4,19358
-osa/job.py,sha256=acdiaRlHsc2GLyz2Oz_RBiawJo9QKK-tsajYdM7X0_Q,25393
+osa/_version.py,sha256=8jtDtmBITB3xhjNoKEFLlUOHmnfoHfF78GzDAEdlgvA,413
+osa/conftest.py,sha256=_NERtB9t-Oi7ykW4QDYy9ZbgsrYxXMmQ3QkDScH5JyE,19601
+osa/job.py,sha256=OnjF88kTVdURcrIR9iPenATNx2HteDFlAKtOX4fD144,26603
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
-osa/paths.py,sha256=nkXPUdQAzIlvB71lOERiPjgET58fj5DEbWazWqKbIJw,11869
+osa/paths.py,sha256=28FJuyr9yDcxL6oA10TINr0CRVDBleQZU8rnqwlHNq4,12905
 osa/raw.py,sha256=ZNIsuqfx5ljoz_hwhSuafdKf-wr8-cxRJmel-A2endg,1337
 osa/report.py,sha256=sL2V7n8Y_UUaSDbWJY2o4UxDb4FU5AaFIRR8R25DB8o,4634
 osa/version.py,sha256=9T2TtuGBQeOy5PJDxMCeGlqx5baxLaq47VmFTDc09z8,796
@@ -36,7 +36,7 @@ osa/provenance/config/logger.yaml,sha256=hy_lH3DfbRFh2VM_iawI-c-3wE0cjTRHy465C2e
 osa/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/scripts/autocloser.py,sha256=j7nRvglIv_Ol_GJzbcfqPHquBIWwQU515GkDRfc6_ho,14540
 osa/scripts/calibration_pipeline.py,sha256=g9o1chqCRRSo7GNuQZRABjGnJYjZyfhkPgRLPmuV994,5703
-osa/scripts/closer.py,sha256=eZLUlqot4EXwL9IlU21tKRZ2GvZ4i8ill0c2QdEkoDY,15219
+osa/scripts/closer.py,sha256=FPqUHaZcKyI20u9PhBCSqO-IXa1El3w_0DRXTKBTVTg,17778
 osa/scripts/copy_datacheck.py,sha256=tfDs6oTdPbii4BOXp6bTHuED0xNJeqaPFrv6Ed7ZnWc,3104
 osa/scripts/datasequence.py,sha256=gXAp8arbLPEK-sca9VnME6-2XfUzBFIoEFchlUZYrXI,9260
 osa/scripts/gain_selection.py,sha256=yaz2fKicuCDgLbD3fhb3l9LBlXLEEO3yuZO-oDc2IPo,8084
@@ -50,7 +50,7 @@ osa/scripts/show_run_summary_tcu.py,sha256=SoDLVKdQHOJkfenFguBOfXf10Gyv7heXSQAFn
 osa/scripts/simulate_processing.py,sha256=NiRVYiwZENt_mnKncytgJT23_-tJMb1B5PswM12nnX4,6941
 osa/scripts/update_source_catalog.py,sha256=GHwWFc-y6S4KkUJxUVM5drdAnVDD0-n3D-Tv3CCmh4E,7218
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=xgBic0sW-eao2MSgYgnP-MB6QbmDK6IIY69m0SGdzaA,12588
+osa/scripts/tests/test_osa_scripts.py,sha256=1B2UFc3idLGSyqeuzf4IEvb6sEbtfaV8ogj1LoEoEyQ,12851
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/tests/test_jobs.py,sha256=F0jsHZ9BYB_cCHHxlXyO9v1E5_-mBJhuFtshtsAtnXo,15260
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
@@ -75,9 +75,9 @@ osa/workflow/dl3.py,sha256=kz7L5jcKHFJ--UdQ8HQKLzWO6nxc2LLOTz42ExcqzTk,9921
 osa/workflow/stages.py,sha256=WYgUM2XDIaUjCc4_Zs_VSGW6gk73EaKcHk6ZMnPds74,6692
 osa/workflow/tests/test_dl3.py,sha256=aY5bb-8OcZGAXG3JPCZihChzkA_GsWjRIa31BHZn3Dg,299
 osa/workflow/tests/test_stages.py,sha256=TmC00XFACWZp740TQeFaokWi3C50ovj_XGiySWrrdZk,3944
-lstosa-0.10.7.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.10.7.dist-info/METADATA,sha256=v28ofFiZTcCliCrNe6aGtQ-C9GxvHlJm__FfAdKTKfQ,7348
-lstosa-0.10.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-lstosa-0.10.7.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
-lstosa-0.10.7.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.10.7.dist-info/RECORD,,
+lstosa-0.10.9.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.10.9.dist-info/METADATA,sha256=iUVQ_IpWWQ4G57LjU50aR0miy3TjqRS11aKsTxM6BD4,7348
+lstosa-0.10.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+lstosa-0.10.9.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
+lstosa-0.10.9.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.10.9.dist-info/RECORD,,

osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.7'
-__version_tuple__ = version_tuple = (0, 10, 7)
+__version__ = version = '0.10.9'
+__version_tuple__ = version_tuple = (0, 10, 9)

osa/conftest.py

@@ -482,7 +482,14 @@ def datacheck_dl1_files(base_test_dir):
 
 @pytest.fixture(scope="session")
 def longterm_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "v0.1.0" / "20200117"
+    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date)
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def longterm_link_latest_dir(base_test_dir):
+    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

osa/job.py

@@ -702,6 +702,37 @@ def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
     return sacct_output
 
 
+def get_closer_sacct_output(sacct_output) -> pd.DataFrame:
+    """
+    Fetch the information of jobs in the queue launched by AUTOCLOSER using the sacct 
+    SLURM output and store it in a pandas dataframe.
+
+    Returns
+    -------
+    queue_list: pd.DataFrame
+    """
+    sacct_output = pd.read_csv(sacct_output, names=FORMAT_SLURM)
+
+    # Keep only the jobs corresponding to AUTOCLOSER sequences 
+    # Until the merging of muon files is fixed, check all jobs except "lstchain_merge_muon_files"
+    sacct_output = sacct_output[
+        (sacct_output["JobName"].str.contains("lstchain_merge_hdf5_files"))
+        | (sacct_output["JobName"].str.contains("lstchain_check_dl1"))
+        | (sacct_output["JobName"].str.contains("lstchain_longterm_dl1_check"))
+        | (sacct_output["JobName"].str.contains("lstchain_cherenkov_transparency"))
+        | (sacct_output["JobName"].str.contains("provproces"))
+    ]
+
+    try:
+        sacct_output["JobID"] = sacct_output["JobID"].apply(lambda x: x.split("_")[0])
+        sacct_output["JobID"] = sacct_output["JobID"].str.strip(".batch").astype(int)
+
+    except AttributeError:
+        log.debug("No job info could be obtained from sacct")
+
+    return sacct_output
+
+
 def filter_jobs(job_info: pd.DataFrame, sequence_list: Iterable):
     """Filter the job info list to get the values of the jobs in the current queue."""
     sequences_info = pd.DataFrame([vars(seq) for seq in sequence_list])

osa/paths.py

@@ -18,6 +18,7 @@ from osa.configs.datamodel import Sequence
 from osa.utils import utils
 from osa.utils.logging import myLogger
 
+
 log = myLogger(logging.getLogger(__name__))
 
 __all__ = [
@@ -346,3 +347,26 @@ def create_source_directories(source_list: list, cuts_dir: Path):
         if source is not None:
             source_dir = cuts_dir / source
             source_dir.mkdir(parents=True, exist_ok=True)
+
+
+def get_latest_version_file(longterm_files: List[str]) -> Path:
+    """Get the latest version path of the produced longterm DL1 datacheck files for a given date."""
+    return max(
+        longterm_files,
+        key=lambda path: int(path.parents[1].name.split(".")[1])
+        if path.parents[1].name.startswith("v")
+        else "",
+    )
+
+
+def create_longterm_symlink():
+    """If the created longterm DL1 datacheck file corresponds to the latest 
+    version available, make symlink to it in the "all" common directory."""
+    nightdir = utils.date_to_dir(options.date)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+    linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
+    all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+    latest_version_file = get_latest_version_file(all_longterm_files)
+
+    log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
+    linked_longterm_file.symlink_to(latest_version_file)

osa/scripts/closer.py

@@ -8,6 +8,7 @@ import re
 import shutil
 import subprocess
 import sys
+import time
 from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Tuple, Iterable, List
@@ -15,15 +16,21 @@ from typing import Tuple, Iterable, List
 from osa import osadb
 from osa.configs import options
 from osa.configs.config import cfg
-from osa.job import are_all_jobs_correctly_finished, save_job_information
+from osa.job import (
+    are_all_jobs_correctly_finished, 
+    save_job_information, 
+    run_sacct, 
+    get_closer_sacct_output
+)
 from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import destination_dir
+from osa.paths import destination_dir, create_longterm_symlink
 from osa.raw import is_raw_data_available
 from osa.report import start
 from osa.utils.cliopts import closercliparsing
 from osa.utils.logging import myLogger
 from osa.utils.register import register_found_pattern
+from osa.utils.mail import send_warning_mail
 from osa.utils.utils import (
     night_finished_flag,
     is_day_closed,
@@ -152,10 +159,13 @@ def post_process(seq_tuple):
     post_process_files(seq_list)
 
     # Merge DL1 datacheck files and produce PDFs. It also produces
-    # the daily datacheck report using the longterm script.
+    # the daily datacheck report using the longterm script, and updates
+    # the longterm DL1 datacheck file with the cherenkov_transparency script.
     if cfg.getboolean("lstchain", "merge_dl1_datacheck"):
         list_job_id = merge_dl1_datacheck(seq_list)
-        daily_datacheck(daily_longterm_cmd(list_job_id))
+        longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
+        cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
+        create_longterm_symlink()
 
     # Extract the provenance info
     extract_provenance(seq_list)
@@ -169,6 +179,24 @@ def post_process(seq_tuple):
     if not options.no_dl2:
         merge_files(seq_list, data_level="DL2")
 
+    time.sleep(600)
+
+    # Check if all jobs launched by autocloser finished correctly 
+    # before creating the NightFinished.txt file
+    n_max = 6
+    n = 0
+    while not all_closer_jobs_finished_correctly() & n <= n_max:
+        log.info(
+            "All jobs launched by autocloser did not finished correctly yet. "
+            "Checking again in 10 minutes..."
+        )
+        time.sleep(600)
+        n += 1
+
+    if n > n_max:
+        send_warning_mail(date=options.date)
+        return False
+
     if options.seqtoclose is None:
         database = cfg.get("database", "path")
         if database:
@@ -466,6 +494,7 @@ def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
 
     return [
         "sbatch",
+        "--parsable",
         "-D",
         options.directory,
         "-o",
@@ -484,11 +513,60 @@ def daily_datacheck(cmd: List[str]):
     log.info("Daily dl1 checks using longterm script.")
     log.debug(f"Executing {stringify(cmd)}")
 
+    if not options.simulate and not options.test and shutil.which("sbatch") is not None:
+        job = subprocess.run(
+            cmd,
+            encoding="utf-8",
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        job_id = job.stdout.strip()
+        return job_id
+    else:
+        log.debug("Simulate launching scripts")
+
+
+def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
+    """Build the cherenkov transparency command."""
+    nightdir = date_to_dir(options.date)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
+    longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
+
+    return [
+        "sbatch",
+        "-D",
+        options.directory,
+        "-o",
+        "log/cherenkov_transparency_%j.log",
+        f"--dependency=afterok:{longterm_job_id}",
+        "lstchain_cherenkov_transparency",
+        f"--update-datacheck-file={longterm_datacheck_file}",
+        f"--input-dir={datacheck_dir}",
+    ]
+
+
+def cherenkov_transparency(cmd: List[str]):
+    """Update longterm dl1 check file with cherenkov transparency information."""
+    log.info("Update longterm dl1 check file with cherenkov_transparency script.")
+    log.debug(f"Executing {stringify(cmd)}")
+
     if not options.simulate and not options.test and shutil.which("sbatch") is not None:
         subprocess.run(cmd, check=True)
     else:
         log.debug("Simulate launching scripts")
 
 
+def all_closer_jobs_finished_correctly():
+    """Check if all the jobs launched by autocloser finished correctly."""
+    sacct_output = run_sacct()
+    jobs_closer = get_closer_sacct_output(sacct_output)
+    if len(jobs_closer[jobs_closer["State"]!="COMPLETED"])==0:
+        return True
+    else:
+        return False
+
+
 if __name__ == "__main__":
     main()

osa/scripts/tests/test_osa_scripts.py

@@ -179,6 +179,9 @@ def test_closer(
     drs4_time_calibration_files,
     systematic_correction_files,
     merged_run_summary,
+    longterm_dir,
+    longterm_link_latest_dir,
+    daily_datacheck_dl1_files,
 ):
     # First assure that the end of night flag is not set and remove it otherwise
     night_finished_flag = Path(
@@ -198,6 +201,10 @@ def test_closer(
     for obs_file in test_observed_data:
         assert obs_file.exists()
     assert merged_run_summary.exists()
+    assert longterm_dir.exists()
+    assert longterm_link_latest_dir.exists()
+    for check_file in daily_datacheck_dl1_files:
+        assert check_file.exists()
 
     run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1")
     closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed"
@@ -319,6 +326,7 @@ def test_daily_longterm_cmd():
 
     expected_cmd = [
         "sbatch",
+        "--parsable",
         "-D",
         options.directory,
         "-o",