lstosa 0.10.7__tar.gz → 0.10.9__tar.gz

{lstosa-0.10.7 → lstosa-0.10.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.7
+Version: 0.10.9
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,7 +20,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain>=0.10.5
+Requires-Dist: lstchain>=0.10.7
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas

{lstosa-0.10.7 → lstosa-0.10.9}/environment.yml

@@ -25,7 +25,7 @@ dependencies:
   - ctapipe_io_lst=0.22
   - ctaplot~=0.6.4
   - pyirf~=0.10
-  - lstchain>=0.10.5
+  - lstchain>=0.10.7
   - tenacity
   # dev dependencies
   - pytest

{lstosa-0.10.7 → lstosa-0.10.9}/pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.9"
 dependencies = [
     "astropy~=5.0",
-    "lstchain>=0.10.5",
+    "lstchain>=0.10.7",
     "matplotlib",
     "numpy",
     "pandas",

{lstosa-0.10.7 → lstosa-0.10.9}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.7
+Version: 0.10.9
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,7 +20,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain>=0.10.5
+Requires-Dist: lstchain>=0.10.7
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas

{lstosa-0.10.7 → lstosa-0.10.9}/src/lstosa.egg-info/requires.txt

@@ -1,5 +1,5 @@
 astropy~=5.0
-lstchain>=0.10.5
+lstchain>=0.10.7
 matplotlib
 numpy
 pandas

{lstosa-0.10.7 → lstosa-0.10.9}/src/osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.7'
-__version_tuple__ = version_tuple = (0, 10, 7)
+__version__ = version = '0.10.9'
+__version_tuple__ = version_tuple = (0, 10, 9)

{lstosa-0.10.7 → lstosa-0.10.9}/src/osa/conftest.py

@@ -482,7 +482,14 @@ def datacheck_dl1_files(base_test_dir):
 
 @pytest.fixture(scope="session")
 def longterm_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "v0.1.0" / "20200117"
+    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date)
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def longterm_link_latest_dir(base_test_dir):
+    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

{lstosa-0.10.7 → lstosa-0.10.9}/src/osa/job.py

@@ -702,6 +702,37 @@ def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
     return sacct_output
 
 
+def get_closer_sacct_output(sacct_output) -> pd.DataFrame:
+    """
+    Fetch the information of jobs in the queue launched by AUTOCLOSER using the sacct 
+    SLURM output and store it in a pandas dataframe.
+
+    Returns
+    -------
+    queue_list: pd.DataFrame
+    """
+    sacct_output = pd.read_csv(sacct_output, names=FORMAT_SLURM)
+
+    # Keep only the jobs corresponding to AUTOCLOSER sequences 
+    # Until the merging of muon files is fixed, check all jobs except "lstchain_merge_muon_files"
+    sacct_output = sacct_output[
+        (sacct_output["JobName"].str.contains("lstchain_merge_hdf5_files"))
+        | (sacct_output["JobName"].str.contains("lstchain_check_dl1"))
+        | (sacct_output["JobName"].str.contains("lstchain_longterm_dl1_check"))
+        | (sacct_output["JobName"].str.contains("lstchain_cherenkov_transparency"))
+        | (sacct_output["JobName"].str.contains("provproces"))
+    ]
+
+    try:
+        sacct_output["JobID"] = sacct_output["JobID"].apply(lambda x: x.split("_")[0])
+        sacct_output["JobID"] = sacct_output["JobID"].str.strip(".batch").astype(int)
+
+    except AttributeError:
+        log.debug("No job info could be obtained from sacct")
+
+    return sacct_output
+
+
 def filter_jobs(job_info: pd.DataFrame, sequence_list: Iterable):
     """Filter the job info list to get the values of the jobs in the current queue."""
     sequences_info = pd.DataFrame([vars(seq) for seq in sequence_list])

{lstosa-0.10.7 → lstosa-0.10.9}/src/osa/paths.py

@@ -18,6 +18,7 @@ from osa.configs.datamodel import Sequence
 from osa.utils import utils
 from osa.utils.logging import myLogger
 
+
 log = myLogger(logging.getLogger(__name__))
 
 __all__ = [
@@ -346,3 +347,26 @@ def create_source_directories(source_list: list, cuts_dir: Path):
         if source is not None:
             source_dir = cuts_dir / source
             source_dir.mkdir(parents=True, exist_ok=True)
+
+
+def get_latest_version_file(longterm_files: List[str]) -> Path:
+    """Get the latest version path of the produced longterm DL1 datacheck files for a given date."""
+    return max(
+        longterm_files,
+        key=lambda path: int(path.parents[1].name.split(".")[1])
+        if path.parents[1].name.startswith("v")
+        else "",
+    )
+
+
+def create_longterm_symlink():
+    """If the created longterm DL1 datacheck file corresponds to the latest 
+    version available, make symlink to it in the "all" common directory."""
+    nightdir = utils.date_to_dir(options.date)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+    linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
+    all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+    latest_version_file = get_latest_version_file(all_longterm_files)
+
+    log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
+    linked_longterm_file.symlink_to(latest_version_file)

{lstosa-0.10.7 → lstosa-0.10.9}/src/osa/scripts/closer.py

@@ -8,6 +8,7 @@ import re
 import shutil
 import subprocess
 import sys
+import time
 from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Tuple, Iterable, List
@@ -15,15 +16,21 @@ from typing import Tuple, Iterable, List
 from osa import osadb
 from osa.configs import options
 from osa.configs.config import cfg
-from osa.job import are_all_jobs_correctly_finished, save_job_information
+from osa.job import (
+    are_all_jobs_correctly_finished, 
+    save_job_information, 
+    run_sacct, 
+    get_closer_sacct_output
+)
 from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import destination_dir
+from osa.paths import destination_dir, create_longterm_symlink
 from osa.raw import is_raw_data_available
 from osa.report import start
 from osa.utils.cliopts import closercliparsing
 from osa.utils.logging import myLogger
 from osa.utils.register import register_found_pattern
+from osa.utils.mail import send_warning_mail
 from osa.utils.utils import (
     night_finished_flag,
     is_day_closed,
@@ -152,10 +159,13 @@ def post_process(seq_tuple):
     post_process_files(seq_list)
 
     # Merge DL1 datacheck files and produce PDFs. It also produces
-    # the daily datacheck report using the longterm script.
+    # the daily datacheck report using the longterm script, and updates
+    # the longterm DL1 datacheck file with the cherenkov_transparency script.
     if cfg.getboolean("lstchain", "merge_dl1_datacheck"):
         list_job_id = merge_dl1_datacheck(seq_list)
-        daily_datacheck(daily_longterm_cmd(list_job_id))
+        longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
+        cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
+        create_longterm_symlink()
 
     # Extract the provenance info
     extract_provenance(seq_list)
@@ -169,6 +179,24 @@ def post_process(seq_tuple):
     if not options.no_dl2:
         merge_files(seq_list, data_level="DL2")
 
+    time.sleep(600)
+
+    # Check if all jobs launched by autocloser finished correctly 
+    # before creating the NightFinished.txt file
+    n_max = 6
+    n = 0
+    while not all_closer_jobs_finished_correctly() & n <= n_max:
+        log.info(
+            "All jobs launched by autocloser did not finished correctly yet. "
+            "Checking again in 10 minutes..."
+        )
+        time.sleep(600)
+        n += 1
+
+    if n > n_max:
+        send_warning_mail(date=options.date)
+        return False
+
     if options.seqtoclose is None:
         database = cfg.get("database", "path")
         if database:
@@ -466,6 +494,7 @@ def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
 
     return [
         "sbatch",
+        "--parsable",
         "-D",
         options.directory,
         "-o",
@@ -484,11 +513,60 @@ def daily_datacheck(cmd: List[str]):
     log.info("Daily dl1 checks using longterm script.")
     log.debug(f"Executing {stringify(cmd)}")
 
+    if not options.simulate and not options.test and shutil.which("sbatch") is not None:
+        job = subprocess.run(
+            cmd,
+            encoding="utf-8",
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        job_id = job.stdout.strip()
+        return job_id
+    else:
+        log.debug("Simulate launching scripts")
+
+
+def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
+    """Build the cherenkov transparency command."""
+    nightdir = date_to_dir(options.date)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
+    longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
+
+    return [
+        "sbatch",
+        "-D",
+        options.directory,
+        "-o",
+        "log/cherenkov_transparency_%j.log",
+        f"--dependency=afterok:{longterm_job_id}",
+        "lstchain_cherenkov_transparency",
+        f"--update-datacheck-file={longterm_datacheck_file}",
+        f"--input-dir={datacheck_dir}",
+    ]
+
+
+def cherenkov_transparency(cmd: List[str]):
+    """Update longterm dl1 check file with cherenkov transparency information."""
+    log.info("Update longterm dl1 check file with cherenkov_transparency script.")
+    log.debug(f"Executing {stringify(cmd)}")
+
     if not options.simulate and not options.test and shutil.which("sbatch") is not None:
         subprocess.run(cmd, check=True)
     else:
         log.debug("Simulate launching scripts")
 
 
+def all_closer_jobs_finished_correctly():
+    """Check if all the jobs launched by autocloser finished correctly."""
+    sacct_output = run_sacct()
+    jobs_closer = get_closer_sacct_output(sacct_output)
+    if len(jobs_closer[jobs_closer["State"]!="COMPLETED"])==0:
+        return True
+    else:
+        return False
+
+
 if __name__ == "__main__":
     main()

{lstosa-0.10.7 → lstosa-0.10.9}/src/osa/scripts/tests/test_osa_scripts.py

@@ -179,6 +179,9 @@ def test_closer(
     drs4_time_calibration_files,
     systematic_correction_files,
     merged_run_summary,
+    longterm_dir,
+    longterm_link_latest_dir,
+    daily_datacheck_dl1_files,
 ):
     # First assure that the end of night flag is not set and remove it otherwise
     night_finished_flag = Path(
@@ -198,6 +201,10 @@ def test_closer(
     for obs_file in test_observed_data:
         assert obs_file.exists()
     assert merged_run_summary.exists()
+    assert longterm_dir.exists()
+    assert longterm_link_latest_dir.exists()
+    for check_file in daily_datacheck_dl1_files:
+        assert check_file.exists()
 
     run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1")
     closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed"
@@ -319,6 +326,7 @@ def test_daily_longterm_cmd():
 
     expected_cmd = [
         "sbatch",
+        "--parsable",
         "-D",
         options.directory,
         "-o",