lstosa 0.11.5__tar.gz → 0.11.7__tar.gz

{lstosa-0.11.5 → lstosa-0.11.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.5
+Version: 0.11.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

lstosa-0.11.7/crontab/crontab.txt

@@ -0,0 +1,97 @@
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_ENV="/local/home/lstanalyzer/osa-env.sh"
+CRON_TZ=UTC
+# Update astropy auxiliary data (e.g. IERS) every week
+0 0 * * SUN source /fefs/aswg/lstosa/utils/osa-conda; python /fefs/aswg/data/aux/astropy/get_iers_data.py  > /dev/null 2>&1
+#
+# lstosa jobs
+# -------------------------------------------------------------------------------------------------------------------------
+# [COPY OBS OVERVIEW] Copy observations overview and daily camera checks to lst1 datacheck webserver
+# -------------------------------------------------------------------------------------------------------------------------
+00 09 * * * /usr/bin/bash /fefs/aswg/lstosa/misc/copy_camera_daily_check_to_www.sh >/dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [RUN SUMMARY] Produce the run summary file at the end of the observations to make sure it contains all the runs taken
+# -------------------------------------------------------------------------------------------------------------------------
+55 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_create_run_summary -d $obsdate -o /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunSummary --r0-path=/fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events --overwrite > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [MERGED RUN SUMMARY] Produce the merged run summary file containing all the runs taken by the LST.
+#                      This only appends the runs of the given date.
+# -------------------------------------------------------------------------------------------------------------------------
+00 07 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_merge_run_summaries -d $obsdate /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Catalog/merged_RunSummary.ecsv --monitoring-path=/fefs/onsite/data/lst-pipe/LSTN-01/monitoring > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [GAIN SELECTION] Launch Gain Selection process 
+# -------------------------------------------------------------------------------------------------------------------------
+32 07-18 * * * . "$CRON_ENV"; /fefs/aswg/workspace/maria.rivero/crontabs/launchers/launch_GainSel.sh
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [GAIN SELECTION CHECK & GAIN SEL WEB MAKER] Make gain selection check and xhtml table and copy it to the lst1 webserver
+# -------------------------------------------------------------------------------------------------------------------------
+00,15,30,45 08-19 * * * . "$CRON_ENV"; /fefs/aswg/workspace/maria.rivero/crontabs/launchers/launch_GainSelCheck.sh
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SIMULATED SEQUENCER] Launch sequencer once in the morning simulated. It also produces the run summary file if not found.
+# -------------------------------------------------------------------------------------------------------------------------
+05 08 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -s -d $obsdate LST1 > /dev/null 2>&1
+#
+#  -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER 1] Run only up to DL1A
+# -------------------------------------------------------------------------------------------------------------------------
+20,50 07-19 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg --no-dl1ab -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_1_LST1.log 2>&1
+#
+#  -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER_CATB_TAILCUTS] Runs from DL1A, produces Cat-B calibration and tailcut files
+# -------------------------------------------------------------------------------------------------------------------------
+10,50 07-20 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_catB_tailcuts -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 -v >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_tailcuts_LST1.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER 2] Finishes the processing producing DL1AB and DL2
+# -------------------------------------------------------------------------------------------------------------------------
+59,30 08-20 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_2_LST1.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SEQUENCER WEB] Make sequencer xhtml table and copy it to the lst1 webserver
+# -------------------------------------------------------------------------------------------------------------------------
+*/20 07-22 * * * . "$CRON_ENV"; /fefs/aswg/workspace/maria.rivero/crontabs/launchers/launch_SequencerWeb.sh
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [AUTOCLOSER] Launch the closer without forcing it (no -f option).
+# -------------------------------------------------------------------------------------------------------------------------
+25,55 9-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; autocloser -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Autocloser_log/${obsdate}_LST1.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [COPY DATACHECK] Copy the available calibration and DL1 datacheck to the LST1 webserver.
+# -------------------------------------------------------------------------------------------------------------------------
+05,25,45 07-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; copy_datacheck -c /fefs/aswg/lstosa/cfg/sexquencer_v0.11.cfg -d $obsdate LST1 > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# [SOURCE CATALOG]
+# -------------------------------------------------------------------------------------------------------------------------
+52 10-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; update_source_catalog -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate > /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunCatalog/log/Source-catalog-$obsdate 2>&1
+#
+#  ------------------------------------------------------------------------------------------------------------------------
+# [COPY CUMULATIVE LST-1 OBSERVATION TIME]
+# (added by Daniel Morcuende on 2024-04-05)
+# -------------------------------------------------------------------------------------------------------------------------
+0 18 *  * FRI /usr/bin/bash /fefs/aswg/lstosa/misc/update_cumulative_obs_time.sh >/dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+#TROUBLESHOOTING
+# -------------------------------------------------------------------------------------------------------------------------
+#*/20 7-22 * * * obsdate=`date +\%Y-\%m-\%d`; cd /fefs/aswg/lstosa/troubleshooting/ && /fefs/aswg/software/conda/bin/python troubleshooting.py >> /fefs/aswg/lstosa/troubleshooting/log/cron_log_${obsdate}.log 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# Daniel Morcuende (2023-11-08) periodically remove leftover jobs kept in the SLURM as pending after job dependency not 
+# satisfied because of failure (Reason in squeue: 'DependencyNeverSatisfied').
+# It does only look for lstosa jobs launched by sequencer with the prefix "LST1_" and also the merging jobs launched by 
+# autocloser like "longterm_dl1_datacheck".
+# -------------------------------------------------------------------------------------------------------------------------
+*/30 * * * * /local/home/lstanalyzer/remove_dependency_never_satisfied_jobs.sh > /dev/null 2>&1
+#
+# -------------------------------------------------------------------------------------------------------------------------
+# Restart camera processing of calibration runs in realtime in case it was killed for whatever reason
+# -------------------------------------------------------------------------------------------------------------------------
+04 * * * * cd /local/home/lstanalyzer/camera/lst-camera-watchers;./lst1_watcher_restarter

lstosa-0.11.7/crontab/launchers/launch_GainSel.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------
+# Launch Gain Selection process given an observation date (OBS_DATE), 
+# configuration file (CFG), and after activating the conda environment
+# (CONDA_ENV); these three parameters are exported from osa-env.sh 
+# Please note that Gain Selection will start up to 6 AM (when the night has ended),
+# and Gain Selection register is saved in {OBS_DATE}_LST1.log
+# --------------------------------------------------------------------
+
+obsdate=$(date +\%Y\%m\%d -d yesterday)
+
+LOGDIR="${LSTN1}/OSA/GainSel_log"
+WORKDIR="${LSTN1}/OSA/GainSel/${obsdate}"
+
+LOGFILE="${LOGDIR}/${OBS_DATE}_LST1.log"
+FLAG_FILE="${WORKDIR}/GainSelFinished.txt"
+
+# -------------------------
+# Check GainSelFinished file exists
+# -------------------------
+if [ -e "$FLAG_FILE" ]; then
+    echo "GainSelFinished.txt exists for ${OBS_DATE}" >> "$LOGFILE"
+    exit
+fi
+# -------------------------
+# Environment
+# -------------------------
+source "$CONDA_ENV"
+
+# -------------------------
+# Run GAIN SELECTION
+# -------------------------
+{
+    gain_selection \
+        --no-queue-check \
+        -c "$CFG" \
+        -d "$OBS_DATE" \
+        -s 6 
+
+} >> "$LOGFILE" 2>&1

lstosa-0.11.7/crontab/launchers/launch_GainSelCheck.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------
+# Check Gain Selection process given an observation date (OBS_DATE),
+# configuration file (CFG), and after activating the conda environment
+# (CONDA_ENV); these three parameters are exported from osa-env.sh; 
+# When Gain Selection has finished, it 
+# creates a flag file for the given date: GainSelFinished.txt
+# ...
+# Make gain selection xhtml table and copy it to the lst1 webserver.
+# --------------------------------------------------------------------
+
+obsdate=$(date +\%Y\%m\%d -d yesterday)
+
+WORKDIR="${LSTN1}/OSA/GainSel/${obsdate}"
+FLAG_FILE="${WORKDIR}/GainSelFinished.txt"
+
+LOGDIR="${LSTN1}/OSA/GainSel_log"
+LOGFILE="${LOGDIR}/${OBS_DATE}_check_LST1.log"
+LOGFILE_WEB="${LOGDIR}/GainSelWeb_${OBS_DATE}_LST1.log"
+
+GS_HTML="${LSTN1}/OSA/GainSelWeb/osa_gainsel_status_${OBS_DATE}.html"
+GS_LSTOSA="/home/www/html/datacheck/lstosa/gainsel.xhtml"
+
+# ---------------------------------
+# Check GainSelFinished file exists
+# ---------------------------------
+if [ -e "$FLAG_FILE" ]; then
+    echo "GainSelFinished.txt exists for ${OBS_DATE}" >> "$LOGFILE"
+    exit
+fi
+
+# -------------------------
+# Environment
+# -------------------------
+source "$CONDA_ENV"
+
+# -------------------------
+# Run GAIN SELECTION CHECK
+# -------------------------
+{
+    gain_selection --check \
+        -c "$CFG" \
+	-d "$OBS_DATE" 
+
+} >> "$LOGFILE" 2>&1
+
+# -------------------------
+# Update GAIN SELECTION WEB
+# -------------------------
+{
+    gainsel_webmaker \
+        -c "$CFG"
+} >> "$LOGFILE_WEB" 2>&1
+
+{
+    if [ $? = 0 ]; then
+        scp "$GS_HTML" datacheck:"$GS_LSTOSA"
+    fi
+} >> "$LOGFILE_WEB" 2>&1

lstosa-0.11.7/crontab/launchers/launch_SequencerWeb.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# --------------------------------------------------------------------
+# Make sequencer xhtml table and copy it to the lst1 webserver
+# --------------------------------------------------------------------
+
+obsdate=$(date +\%Y\%m\%d -d yesterday)
+LOGDIR="${LSTN1}/OSA/Minor_logs/"
+LOGFILE="${LOGDIR}/${obsdate}_sequencer-web.log"
+LOGFILE2="${LOGDIR}/${obsdate}_sequencer-web_2.log"
+
+HTMLDIR="${LSTN1}/OSA/SequencerWeb/"
+HTMLFILE="${HTMLDIR}/osa_status_${obsdate}.html"
+HTMLFILE2="${HTMLDIR}/osa_status_2_${obsdate}.html"
+
+SEQUENCER_WEB="/home/www/html/datacheck/lstosa/sequencer.xhtml"
+SEQUENCER2_WEB="/home/www/html/datacheck/lstosa/sequencer_2.xhtml"
+
+# -------------------------
+# Environment
+# -------------------------
+source "$CONDA_ENV"
+
+# -------------------------
+# Run SEQUENCER WEBMAKER
+# -------------------------
+{
+    sequencer_webmaker \
+        -c "$CFG"
+} >> "$LOGFILE" 2>&1
+
+{
+    if [ $? = 0 ]; then
+        scp "$HTMLFILE" datacheck:"$SEQUENCER_WEB"
+    fi
+} >> "$LOGFILE" 2>&1
+
+# -------------------------
+# Run SEQUENCER WEBMAKER 2
+# -------------------------
+
+{
+    sequencer_webmaker_2 \
+        -c "$CFG"
+} >> "$LOGFILE2" 2>&1
+
+{
+    if [ $? = 0 ]; then
+        scp "$HTMLFILE2" datacheck:"$SEQUENCER2_WEB"
+    fi
+} >> "$LOGFILE2" 2>&1

lstosa-0.11.7/crontab/osa-env.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+OBS_DATE=$(date +%Y-%m-%d -d yesterday)
+CFG="/fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg"
+CONDA_ENV="/fefs/aswg/lstosa/utils/osa-conda"
+
+LSTN1="/fefs/onsite/data/lst-pipe/LSTN-01"
+
+export OBS_DATE CFG CONDA_ENV
+export LSTN1

{lstosa-0.11.5 → lstosa-0.11.7}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.5
+Version: 0.11.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.5 → lstosa-0.11.7}/src/lstosa.egg-info/SOURCES.txt

@@ -12,6 +12,10 @@ environment.yml
 pyproject.toml
 crontab/crontab-orig.txt
 crontab/crontab.txt
+crontab/osa-env.sh
+crontab/launchers/launch_GainSel.sh
+crontab/launchers/launch_GainSelCheck.sh
+crontab/launchers/launch_SequencerWeb.sh
 dev/mysql.py
 docs/Makefile
 docs/authors.rst
@@ -94,6 +98,7 @@ src/osa/scripts/copy_datacheck.py
 src/osa/scripts/datasequence.py
 src/osa/scripts/gain_selection.py
 src/osa/scripts/gainsel_webmaker.py
+src/osa/scripts/interleaved_date.py
 src/osa/scripts/provprocess.py
 src/osa/scripts/reprocess_longterm.py
 src/osa/scripts/reprocessing.py

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.11.5'
-__version_tuple__ = version_tuple = (0, 11, 5)
+__version__ = version = '0.11.7'
+__version_tuple__ = version_tuple = (0, 11, 7)
 
-__commit_id__ = commit_id = 'gdc0cee0a4'
+__commit_id__ = commit_id = 'g38091dd33'

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/configs/sequencer.cfg

@@ -26,11 +26,12 @@ DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
+DATACHECK_DIR: %(DL1_DIR)s/datacheck_files
 RF_MODELS: %(BASE)s/models/AllSky
 OSA_DIR: %(BASE)s/OSA
 CLOSER_DIR: %(OSA_DIR)s/Closer
 HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
-LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
+LONGTERM_DIR: %(DATACHECK_DIR)s/night_wise
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
 SOURCE_CATALOG: %(OSA_DIR)s/Catalog
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/conftest.py

@@ -511,15 +511,15 @@ def datacheck_dl1_files(base_test_dir):
 
 
 @pytest.fixture(scope="session")
-def longterm_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date)
+def longterm_dir(datacheck_dir):
+    directory = datacheck_dir / prod_id / date_to_dir(date)
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
 
 @pytest.fixture(scope="session")
-def longterm_link_latest_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all"
+def datacheck_dir(base_test_dir):
+    directory = base_test_dir / "DL1" / "datacheck_files" / "night_wise"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/nightsummary/database.py

@@ -14,27 +14,21 @@ __all__ = ["query", "db_available", "get_run_info_from_TCU"]
 
 log = myLogger(logging.getLogger(__name__))
 
-
-CACO_DB = cfg.get("database", "caco_db")
 TCU_DB = cfg.get("database", "tcu_db")
 
-
 def db_available():
     """Check the connection to the TCU database."""
-    caco_client = MongoClient(CACO_DB, serverSelectionTimeoutMS=3000)
     tcu_client = MongoClient(TCU_DB, serverSelectionTimeoutMS=3000)
     try:
-        caco_client.server_info()
         tcu_client.server_info()
     except ConnectionFailure:
-        log.warning("TCU or CaCo database not available. No source info will be added.")
+        log.warning("TCU database not available. No source info will be added.")
         return False
     else:
-        log.debug("TCU and CaCo database are available. Source info will be added.")
+        log.debug("TCU database is available. Source info will be added.")
         return True
 
-
-def query(obs_id: int, property_name: str):
+def query(obs_id: int):
     """
     Query the source name and coordinates from TCU database.
 
@@ -42,13 +36,11 @@ def query(obs_id: int, property_name: str):
     ----------
     obs_id : int
         Run number
-    property_name : str
-        Properties from drive information e.g. `DriveControl_SourceName`,
-        `DriveControl_RA_Target`, `DriveControl_Dec_Target`
+
 
     Returns
     -------
-    query_result : str or None
+    query_result : Dict
         Query result from database. It can be either the source name or its coordinates.
 
     Raises
@@ -60,47 +52,47 @@ def query(obs_id: int, property_name: str):
     if not isinstance(obs_id, int):
         obs_id = int(obs_id)
 
-    caco_client = MongoClient(CACO_DB)
-    tcu_client = MongoClient(TCU_DB)
-
-    with caco_client, tcu_client:
-        run_info = caco_client["CACO"]["RUN_INFORMATION"]
-        run = run_info.find_one({"run_number": obs_id})
+    try:
+        tcu_client = MongoClient(TCU_DB, serverSelectionTimeoutMS=3000)
+        db = tcu_client["lst1_obs_summary"]
+        camera_col = db["camera"]
 
-        try:
-            start = datetime.fromisoformat(run["start_time"].replace("Z", ""))
-            end = datetime.fromisoformat(run["stop_time"].replace("Z", ""))
-        except TypeError:
-            return None
+        run_info = camera_col.find_one({"run_number": obs_id})
 
-        bridges_monitoring = tcu_client["bridgesmonitoring"]
-        property_collection = bridges_monitoring["properties"]
-        chunk_collection = bridges_monitoring["chunks"]
-        descriptors = property_collection.find(
-            {"property_name": property_name},
-        )
+        if not run_info:
+            log.info(f"Run {obs_id} not found 'lst1_obs_summary.camera'")
+        else:
 
-        entries = {"name": property_name, "time": [], "value": []}
+            tstart = run_info.get("tstart")
+            tstop = run_info.get("tstop")
+            run_type = run_info.get("kind")
 
-        for descriptor in descriptors:
-            query_property = {"pid": descriptor["_id"]}
+            tstart_iso = datetime.fromtimestamp(tstart).isoformat(sep=" ", timespec="seconds")
 
-            if start is not None:
-                query_property["begin"] = {"$gte": start}
+            log.info(f"Run {obs_id} ({run_type}) found.")
+            log.info(f"Time: {tstart_iso} (Timestamp: {tstart})")
 
-            if end is not None:
-                query_property["end"] = {"$lte": end}
+            telescope_col = db["telescope"]
+            query = {
+                "tstart": {"$lte": tstop},
+                "tstop": {"$gte": tstart}
+            }
 
-            chunks = chunk_collection.find(query_property)
+            tel_doc = telescope_col.find_one(query, sort=[("tstart", -1)])
 
-            for chunk in chunks:
-                for value in chunk["values"]:
-                    entries["time"].append(value["t"])
-                    entries["value"].append(value["val"])
+            if tel_doc:
 
-                    source_name = entries["value"][0]
-                    return source_name if source_name != "" else None
+                config = tel_doc.get("data", {}).get("structure", [])[0]
+                target = config.get("target", {})
+                source_name = target.get("name", "Desconocido")
+                ra = target.get("source_ra", "N/A")
+                dec = target.get("source_dec", "N/A")
+                return {"source_name": source_name, "ra": ra, "dec": dec}
+            else:
+                log.info("\nNo information found for that time range in 'lst1_obs_summary.telescope'.")
 
+    except Exception as e:
+        log.info(f"ERROR: {e}")
 
 def get_run_info_from_TCU(run_id: int, tcu_server: str) -> Tuple:
     """

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/nightsummary/extract.py

@@ -173,15 +173,16 @@ def extract_runs(summary_table):
             # Make sure we are looking at actual data runs. Avoid test runs.
             if run.run > 0 and run.type == "DATA":
                 log.debug(f"Looking info in TCU DB for run {run.run}")
-                run.source_name = database.query(
-                    obs_id=run.run, property_name="DriveControl_SourceName"
+
+                tcu_result = database.query(
+                    obs_id=run.run
                 )
-                run.source_ra = database.query(
-                    obs_id=run.run, property_name="DriveControl_RA_Target"
-                    )
-                run.source_dec = database.query(
-                    obs_id=run.run, property_name="DriveControl_Dec_Target"
-                    )
+
+                if tcu_result is not None:
+                    run.source_name = tcu_result.get("source_name")
+                    run.source_ra = tcu_result.get("ra")
+                    run.source_dec = tcu_result.get("dec")
+
                 # Store this source information (run_id, source_name, source_ra, source_dec)
                 # into an astropy Table and save to disk in RunCatalog files. In this way, the
                 # information can be dumped anytime later more easily than accessing the

lstosa-0.11.7/src/osa/nightsummary/tests/test_database.py

@@ -0,0 +1,8 @@
+from osa.nightsummary import database
+
+def test_query():
+    result = database.query(obs_id=20038)
+    assert result is None
+
+
+

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/paths.py

@@ -388,18 +388,66 @@ def is_job_completed(job_id: str):
     return False
 
 
-def create_longterm_symlink(cherenkov_job_id: str = None):
+def create_longterm_symlink():
     """If the created longterm DL1 datacheck file corresponds to the latest 
     version available, make symlink to it in the "all" common directory."""
-    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
-        nightdir = utils.date_to_dir(options.date)
-        longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
-        linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
-        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+    
+    nightdir = utils.date_to_dir(options.date)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR"))
+    
+    extensions = ["h5", "log", "html"]
+    
+    for ext in extensions:
+        linked_longterm_file = output_dir / f"night_wise/DL1_datacheck_{nightdir}.{ext}"
+        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.{ext}")
         latest_version_file = get_latest_version_file(all_longterm_files)
-        log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
         linked_longterm_file.unlink(missing_ok=True)
         linked_longterm_file.symlink_to(latest_version_file)
+
+
+def create_runwise_datacheck_symlinks():
+    """Create symlinks of the run-wise datacheck files in the "datacheck" directory."""
+    nightdir = utils.date_to_dir(options.date)
+    dl1_dir = Path(cfg.get("LST1", "DL1_DIR")) / nightdir / options.prod_id
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR")) / nightdir
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    patterns = [
+        "tailcut*/datacheck/datacheck_dl1_LST-1.Run?????.pdf",
+        "tailcut*/datacheck/datacheck_dl1_LST-1.Run?????.h5",
+    ]
+
+    for pattern in patterns:
+        for input_file in dl1_dir.rglob(pattern):
+            output_file = output_dir / input_file.name
+            if not output_file.is_symlink():
+                output_file.symlink_to(input_file.resolve())
+
+
+def create_muons_symlinks():
+    """Create symlinks of the muon files in the "datacheck" directory."""
+    nightdir = utils.date_to_dir(options.date)
+    muons_dir = destination_dir("MUON", create_dir=False)
+    muons_file_list = muons_dir.rglob("muons_LST-1*.fits")
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR")) / nightdir / "muons"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    for input_file in muons_file_list:
+        output_file = output_dir / input_file.name
+        if not output_file.is_symlink():
+            print(f"input file exists: {input_file.exists()}")
+            output_file.symlink_to(input_file.resolve())
+
+
+def create_datacheck_symlinks(cherenkov_job_id: str=None):
+    """Once all steps of autocloser have finished successfully, create symlinks of the run-wise
+    and night-wise datacheck files, and of the muon files in the "datacheck" directory."""
+    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
+        log.info("Creating symlinks of the datacheck and muon files in the common directory.")
+        create_longterm_symlink()
+        create_runwise_datacheck_symlinks()
+        create_muons_symlinks()
     else:
         log.warning(f"Job {cherenkov_job_id} (lstchain_cherenkov_transparency) did not finish successfully.")
 

{lstosa-0.11.5 → lstosa-0.11.7}/src/osa/scripts/closer.py

@@ -26,7 +26,7 @@ from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
 from osa.paths import (
     destination_dir,
-    create_longterm_symlink,
+    create_datacheck_symlinks,
     dl1_datacheck_longterm_file_exits
 )
 from osa.raw import is_raw_data_available
@@ -161,7 +161,7 @@ def post_process(seq_tuple):
     
     if dl1_datacheck_longterm_file_exits() and not options.test:
         if cfg.getboolean("lstchain", "create_longterm_symlink"):
-            create_longterm_symlink()
+            create_datacheck_symlinks()
 
     else:
         # Close the sequences
@@ -187,7 +187,7 @@ def post_process(seq_tuple):
             longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
             cherenkov_job_id = cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
             if cfg.getboolean("lstchain", "create_longterm_symlink"):
-                create_longterm_symlink(cherenkov_job_id)
+                create_datacheck_symlinks(cherenkov_job_id)
 
         time.sleep(600)
 
@@ -584,7 +584,7 @@ def merge_muon_files(sequence_list):
 def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     """Build the daily longterm command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut*")
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
@@ -629,7 +629,7 @@ def daily_datacheck(cmd: List[str]):
 def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
     """Build the cherenkov transparency command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut*")
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
     slurm_account = cfg.get("SLURM", "ACCOUNT")