lstosa 0.11.6__tar.gz → 0.11.7__tar.gz

{lstosa-0.11.6 → lstosa-0.11.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.6
+Version: 0.11.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

lstosa-0.11.7/crontab/crontab.txt

@@ -0,0 +1,97 @@
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_ENV="/local/home/lstanalyzer/osa-env.sh"
+CRON_TZ=UTC
+# Update astropy auxiliary data (e.g. IERS) every week
+0 0 * * SUN source /fefs/aswg/lstosa/utils/osa-conda; python /fefs/aswg/data/aux/astropy/get_iers_data.py  > /dev/null 2>&1
+#
+# lstosa jobs
+# -------------------------------------------------------------------------------------------------------------------------
+# [COPY OBS OVERVIEW] Copy observations overview and daily camera checks to lst1 datacheck webserver
+# -------------------------------------------------------------------------------------------------------------------------
+00 09 * * * /usr/bin/bash /fefs/aswg/lstosa/misc/copy_camera_daily_check_to_www.sh >/dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [RUN SUMMARY] Produce the run summary file at the end of the observations to make sure it contains all the runs taken
+# -------------------------------------------------------------------------------------------------------------------------
+55 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_create_run_summary -d $obsdate -o /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunSummary --r0-path=/fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events --overwrite > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [MERGED RUN SUMMARY] Produce the merged run summary file containing all the runs taken by the LST.
+#                      This only appends the runs of the given date.
+# -------------------------------------------------------------------------------------------------------------------------
+00 07 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_merge_run_summaries -d $obsdate /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Catalog/merged_RunSummary.ecsv --monitoring-path=/fefs/onsite/data/lst-pipe/LSTN-01/monitoring > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [GAIN SELECTION] Launch Gain Selection process 
+# -------------------------------------------------------------------------------------------------------------------------
+32 07-18 * * * . "$CRON_ENV"; /fefs/aswg/workspace/maria.rivero/crontabs/launchers/launch_GainSel.sh
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [GAIN SELECTION CHECK & GAIN SEL WEB MAKER] Make gain selection check and xhtml table and copy it to the lst1 webserver
+# -------------------------------------------------------------------------------------------------------------------------
+00,15,30,45 08-19 * * * . "$CRON_ENV"; /fefs/aswg/workspace/maria.rivero/crontabs/launchers/launch_GainSelCheck.sh
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SIMULATED SEQUENCER] Launch sequencer once in the morning simulated. It also produces the run summary file if not found.
+# -------------------------------------------------------------------------------------------------------------------------
+05 08 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -s -d $obsdate LST1 > /dev/null 2>&1
+#
+#  -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER 1] Run only up to DL1A
+# -------------------------------------------------------------------------------------------------------------------------
+20,50 07-19 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg --no-dl1ab -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_1_LST1.log 2>&1
+#
+#  -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER_CATB_TAILCUTS] Runs from DL1A, produces Cat-B calibration and tailcut files
+# -------------------------------------------------------------------------------------------------------------------------
+10,50 07-20 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_catB_tailcuts -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 -v >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_tailcuts_LST1.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER 2] Finishes the processing producing DL1AB and DL2
+# -------------------------------------------------------------------------------------------------------------------------
+59,30 08-20 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_2_LST1.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER WEB] Make sequencer xhtml table and copy it to the lst1 webserver
+# -------------------------------------------------------------------------------------------------------------------------
+*/20 07-22 * * * . "$CRON_ENV"; /fefs/aswg/workspace/maria.rivero/crontabs/launchers/launch_SequencerWeb.sh
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [AUTOCLOSER] Launch the closer without forcing it (no -f option).
+# -------------------------------------------------------------------------------------------------------------------------
+25,55 9-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; autocloser -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Autocloser_log/${obsdate}_LST1.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [COPY DATACHECK] Copy the available calibration and DL1 datacheck to the LST1 webserver.
+# -------------------------------------------------------------------------------------------------------------------------
+05,25,45 07-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; copy_datacheck -c /fefs/aswg/lstosa/cfg/sexquencer_v0.11.cfg -d $obsdate LST1 > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SOURCE CATALOG]
+# -------------------------------------------------------------------------------------------------------------------------
+52 10-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; update_source_catalog -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate > /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunCatalog/log/Source-catalog-$obsdate 2>&1
+#
+#  ------------------------------------------------------------------------------------------------------------------------
+# [COPY CUMULATIVE LST-1 OBSERVATION TIME]
+# (added by Daniel Morcuende on 2024-04-05)
+# -------------------------------------------------------------------------------------------------------------------------
+0 18 *  * FRI /usr/bin/bash /fefs/aswg/lstosa/misc/update_cumulative_obs_time.sh >/dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+#TROUBLESHOOTING
+# -------------------------------------------------------------------------------------------------------------------------
+#*/20 7-22 * * * obsdate=`date +\%Y-\%m-\%d`; cd /fefs/aswg/lstosa/troubleshooting/ && /fefs/aswg/software/conda/bin/python troubleshooting.py >> /fefs/aswg/lstosa/troubleshooting/log/cron_log_${obsdate}.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# Daniel Morcuende (2023-11-08) periodically remove leftover jobs kept in the SLURM as pending after job dependency not 
+# satisfied because of failure (Reason in squeue: 'DependencyNeverSatisfied').
+# It does only look for lstosa jobs launched by sequencer with the prefix "LST1_" and also the merging jobs launched by 
+# autocloser like "longterm_dl1_datacheck".
+# -------------------------------------------------------------------------------------------------------------------------
+*/30 * * * * /local/home/lstanalyzer/remove_dependency_never_satisfied_jobs.sh > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# Restart camera processing of calibration runs in realtime in case it was killed for whatever reason
+# -------------------------------------------------------------------------------------------------------------------------
+04 * * * * cd /local/home/lstanalyzer/camera/lst-camera-watchers;./lst1_watcher_restarter

lstosa-0.11.7/crontab/launchers/launch_GainSel.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------
+# Launch Gain Selection process given an observation date (OBS_DATE), 
+# configuration file (CFG), and after activating the conda environment
+# (CONDA_ENV); these three parameters are exported from osa-env.sh 
+# Please note that Gain Selection will start up to 6 AM (when the night has ended),
+# and Gain Selection register is saved in {OBS_DATE}_LST1.log
+# --------------------------------------------------------------------
+
+obsdate=$(date +\%Y\%m\%d -d yesterday)
+
+LOGDIR="${LSTN1}/OSA/GainSel_log"
+WORKDIR="${LSTN1}/OSA/GainSel/${obsdate}"
+
+LOGFILE="${LOGDIR}/${OBS_DATE}_LST1.log"
+FLAG_FILE="${WORKDIR}/GainSelFinished.txt"
+
+# -------------------------
+# Check GainSelFinished file exists
+# -------------------------
+if [ -e "$FLAG_FILE" ]; then
+    echo "GainSelFinished.txt exists for ${OBS_DATE}" >> "$LOGFILE"
+    exit
+fi
+# -------------------------
+# Environment
+# -------------------------
+source "$CONDA_ENV"
+
+# -------------------------
+# Run GAIN SELECTION
+# -------------------------
+{
+    gain_selection \
+        --no-queue-check \
+        -c "$CFG" \
+        -d "$OBS_DATE" \
+        -s 6 
+
+} >> "$LOGFILE" 2>&1

lstosa-0.11.7/crontab/launchers/launch_GainSelCheck.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------
+# Check Gain Selection process given an observation date (OBS_DATE),
+# configuration file (CFG), and after activating the conda environment
+# (CONDA_ENV); these three parameters are exported from osa-env.sh; 
+# When Gain Selection has finished, it 
+# creates a flag file for the given date: GainSelFinished.txt
+# ...
+# Make gain selection xhtml table and copy it to the lst1 webserver.
+# --------------------------------------------------------------------
+
+obsdate=$(date +\%Y\%m\%d -d yesterday)
+
+WORKDIR="${LSTN1}/OSA/GainSel/${obsdate}"
+FLAG_FILE="${WORKDIR}/GainSelFinished.txt"
+
+LOGDIR="${LSTN1}/OSA/GainSel_log"
+LOGFILE="${LOGDIR}/${OBS_DATE}_check_LST1.log"
+LOGFILE_WEB="${LOGDIR}/GainSelWeb_${OBS_DATE}_LST1.log"
+
+GS_HTML="${LSTN1}/OSA/GainSelWeb/osa_gainsel_status_${OBS_DATE}.html"
+GS_LSTOSA="/home/www/html/datacheck/lstosa/gainsel.xhtml"
+
+# ---------------------------------
+# Check GainSelFinished file exists
+# ---------------------------------
+if [ -e "$FLAG_FILE" ]; then
+    echo "GainSelFinished.txt exists for ${OBS_DATE}" >> "$LOGFILE"
+    exit
+fi
+
+# -------------------------
+# Environment
+# -------------------------
+source "$CONDA_ENV"
+
+# -------------------------
+# Run GAIN SELECTION CHECK
+# -------------------------
+{
+    gain_selection --check \
+        -c "$CFG" \
+	-d "$OBS_DATE" 
+
+} >> "$LOGFILE" 2>&1
+
+# -------------------------
+# Update GAIN SELECTION WEB
+# -------------------------
+{
+    gainsel_webmaker \
+        -c "$CFG"
+} >> "$LOGFILE_WEB" 2>&1
+
+{
+    if [ $? = 0 ]; then
+        scp "$GS_HTML" datacheck:"$GS_LSTOSA"
+    fi
+} >> "$LOGFILE_WEB" 2>&1

lstosa-0.11.7/crontab/launchers/launch_SequencerWeb.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------
+# Make sequencer xhtml table and copy it to the lst1 webserver
+# --------------------------------------------------------------------
+
+obsdate=$(date +\%Y\%m\%d -d yesterday)
+LOGDIR="${LSTN1}/OSA/Minor_logs/"
+LOGFILE="${LOGDIR}/${obsdate}_sequencer-web.log"
+LOGFILE2="${LOGDIR}/${obsdate}_sequencer-web_2.log"
+
+HTMLDIR="${LSTN1}/OSA/SequencerWeb/"
+HTMLFILE="${HTMLDIR}/osa_status_${obsdate}.html"
+HTMLFILE2="${HTMLDIR}/osa_status_2_${obsdate}.html"
+
+SEQUENCER_WEB="/home/www/html/datacheck/lstosa/sequencer.xhtml"
+SEQUENCER2_WEB="/home/www/html/datacheck/lstosa/sequencer_2.xhtml"
+
+# -------------------------
+# Environment
+# -------------------------
+source "$CONDA_ENV"
+
+# -------------------------
+# Run SEQUENCER WEBMAKER
+# -------------------------
+{
+    sequencer_webmaker \
+        -c "$CFG"
+} >> "$LOGFILE" 2>&1
+
+{
+    if [ $? = 0 ]; then
+        scp "$HTMLFILE" datacheck:"$SEQUENCER_WEB"
+    fi
+} >> "$LOGFILE" 2>&1
+
+# -------------------------
+# Run SEQUENCER WEBMAKER 2
+# -------------------------
+
+{
+    sequencer_webmaker_2 \
+        -c "$CFG"
+} >> "$LOGFILE2" 2>&1
+
+{
+    if [ $? = 0 ]; then
+        scp "$HTMLFILE2" datacheck:"$SEQUENCER2_WEB"
+    fi
+} >> "$LOGFILE2" 2>&1

lstosa-0.11.7/crontab/osa-env.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+OBS_DATE=$(date +%Y-%m-%d -d yesterday)
+CFG="/fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg"
+CONDA_ENV="/fefs/aswg/lstosa/utils/osa-conda"
+
+LSTN1="/fefs/onsite/data/lst-pipe/LSTN-01"
+
+export OBS_DATE CFG CONDA_ENV
+export LSTN1

{lstosa-0.11.6 → lstosa-0.11.7}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.6
+Version: 0.11.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.6 → lstosa-0.11.7}/src/lstosa.egg-info/SOURCES.txt

@@ -12,6 +12,10 @@ environment.yml
 pyproject.toml
 crontab/crontab-orig.txt
 crontab/crontab.txt
+crontab/osa-env.sh
+crontab/launchers/launch_GainSel.sh
+crontab/launchers/launch_GainSelCheck.sh
+crontab/launchers/launch_SequencerWeb.sh
 dev/mysql.py
 docs/Makefile
 docs/authors.rst
@@ -94,6 +98,7 @@ src/osa/scripts/copy_datacheck.py
 src/osa/scripts/datasequence.py
 src/osa/scripts/gain_selection.py
 src/osa/scripts/gainsel_webmaker.py
+src/osa/scripts/interleaved_date.py
 src/osa/scripts/provprocess.py
 src/osa/scripts/reprocess_longterm.py
 src/osa/scripts/reprocessing.py

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.11.6'
-__version_tuple__ = version_tuple = (0, 11, 6)
+__version__ = version = '0.11.7'
+__version_tuple__ = version_tuple = (0, 11, 7)
 
-__commit_id__ = commit_id = 'g934050aad'
+__commit_id__ = commit_id = 'g38091dd33'

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/configs/sequencer.cfg

@@ -26,11 +26,12 @@ DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
+DATACHECK_DIR: %(DL1_DIR)s/datacheck_files
 RF_MODELS: %(BASE)s/models/AllSky
 OSA_DIR: %(BASE)s/OSA
 CLOSER_DIR: %(OSA_DIR)s/Closer
 HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
-LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
+LONGTERM_DIR: %(DATACHECK_DIR)s/night_wise
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
 SOURCE_CATALOG: %(OSA_DIR)s/Catalog
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/conftest.py

@@ -511,15 +511,15 @@ def datacheck_dl1_files(base_test_dir):
 
 
 @pytest.fixture(scope="session")
-def longterm_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date)
+def longterm_dir(datacheck_dir):
+    directory = datacheck_dir / prod_id / date_to_dir(date)
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
 
 @pytest.fixture(scope="session")
-def longterm_link_latest_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all"
+def datacheck_dir(base_test_dir):
+    directory = base_test_dir / "DL1" / "datacheck_files" / "night_wise"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/paths.py

@@ -388,18 +388,66 @@ def is_job_completed(job_id: str):
     return False
 
 
-def create_longterm_symlink(cherenkov_job_id: str = None):
+def create_longterm_symlink():
     """If the created longterm DL1 datacheck file corresponds to the latest 
     version available, make symlink to it in the "all" common directory."""
-    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
-        nightdir = utils.date_to_dir(options.date)
-        longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
-        linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
-        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+    
+    nightdir = utils.date_to_dir(options.date)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR"))
+    
+    extensions = ["h5", "log", "html"]
+    
+    for ext in extensions:
+        linked_longterm_file = output_dir / f"night_wise/DL1_datacheck_{nightdir}.{ext}"
+        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.{ext}")
         latest_version_file = get_latest_version_file(all_longterm_files)
-        log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
         linked_longterm_file.unlink(missing_ok=True)
         linked_longterm_file.symlink_to(latest_version_file)
+
+
+def create_runwise_datacheck_symlinks():
+    """Create symlinks of the run-wise datacheck files in the "datacheck" directory."""
+    nightdir = utils.date_to_dir(options.date)
+    dl1_dir = Path(cfg.get("LST1", "DL1_DIR")) / nightdir / options.prod_id
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR")) / nightdir
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    patterns = [
+        "tailcut*/datacheck/datacheck_dl1_LST-1.Run?????.pdf",
+        "tailcut*/datacheck/datacheck_dl1_LST-1.Run?????.h5",
+    ]
+
+    for pattern in patterns:
+        for input_file in dl1_dir.rglob(pattern):
+            output_file = output_dir / input_file.name
+            if not output_file.is_symlink():
+                output_file.symlink_to(input_file.resolve())
+
+
+def create_muons_symlinks():
+    """Create symlinks of the muon files in the "datacheck" directory."""
+    nightdir = utils.date_to_dir(options.date)
+    muons_dir = destination_dir("MUON", create_dir=False)
+    muons_file_list = muons_dir.rglob("muons_LST-1*.fits")
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR")) / nightdir / "muons"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    for input_file in muons_file_list:
+        output_file = output_dir / input_file.name
+        if not output_file.is_symlink():
+            print(f"input file exists: {input_file.exists()}")
+            output_file.symlink_to(input_file.resolve())
+
+
+def create_datacheck_symlinks(cherenkov_job_id: str=None):
+    """Once all steps of autocloser have finished successfully, create symlinks of the run-wise
+    and night-wise datacheck files, and of the muon files in the "datacheck" directory."""
+    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
+        log.info("Creating symlinks of the datacheck and muon files in the common directory.")
+        create_longterm_symlink()
+        create_runwise_datacheck_symlinks()
+        create_muons_symlinks()
     else:
         log.warning(f"Job {cherenkov_job_id} (lstchain_cherenkov_transparency) did not finish successfully.")
 

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/scripts/closer.py

@@ -26,7 +26,7 @@ from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
 from osa.paths import (
     destination_dir,
-    create_longterm_symlink,
+    create_datacheck_symlinks,
     dl1_datacheck_longterm_file_exits
 )
 from osa.raw import is_raw_data_available
@@ -161,7 +161,7 @@ def post_process(seq_tuple):
     
     if dl1_datacheck_longterm_file_exits() and not options.test:
         if cfg.getboolean("lstchain", "create_longterm_symlink"):
-            create_longterm_symlink()
+            create_datacheck_symlinks()
 
     else:
         # Close the sequences
@@ -187,7 +187,7 @@ def post_process(seq_tuple):
             longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
             cherenkov_job_id = cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
             if cfg.getboolean("lstchain", "create_longterm_symlink"):
-                create_longterm_symlink(cherenkov_job_id)
+                create_datacheck_symlinks(cherenkov_job_id)
 
         time.sleep(600)
 
@@ -584,7 +584,7 @@ def merge_muon_files(sequence_list):
 def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     """Build the daily longterm command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut*")
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
@@ -629,7 +629,7 @@ def daily_datacheck(cmd: List[str]):
 def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
     """Build the cherenkov transparency command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut*")
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
     slurm_account = cfg.get("SLURM", "ACCOUNT")

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/scripts/gain_selection.py

@@ -334,21 +334,23 @@ def update_history_file(run_id: str, subrun: str, log_dir: Path, history_file: P
         log.debug(f"Cannot find a job_id for the run {run_id:05d}.{subrun:04d}")
     else:
         job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
         if job_status.item() in ["RUNNING", "PENDING"]:
             log.info(f"Job {job_id} is still running.")
             return
-            
+
         elif job_status.item() == "COMPLETED":
             log.debug(f"Job {job_id} finished successfully, updating history file.")
             string_to_write = (
-                f"{run_id:05d}.{subrun:04d} gain_selection 0\n"
+                f"{now} | {run_id:05d}.{subrun:04d} gain_selection 0\n"
             )
             append_to_file(history_file, string_to_write)
-        
+
         else:
             log.info(f"Job {job_id} failed, updating history file.")
             string_to_write = (
-                f"{run_id:05d}.{subrun:04d} gain_selection 1\n"
+                f"{now} | {run_id:05d}.{subrun:04d} gain_selection 1\n"
             )
             append_to_file(history_file, string_to_write)
 

{lstosa-0.11.6 → lstosa-0.11.7}/src/osa/scripts/gainsel_webmaker.py

@@ -144,6 +144,10 @@ def main():
         content = "<p>No data found</p>"
         log.warning(f"No data found for date {date}, creating an empty HTML file.")
 
+    elif len(Table.read(run_summary_file)[Table.read(run_summary_file)["run_type"] == "DATA"]) == 0:
+        content = "<p>Only calibration events were taken</p>"
+        log.warning(f"No DATA runs for date {date}, creating an empty HTML file.")
+
     else:
         # Get the table with the gain selection check report in HTML format:
         table_gain_selection_jobs = check_failed_jobs(options.date)