lstosa 0.11.2__tar.gz → 0.11.4__tar.gz

{lstosa-0.11.2 → lstosa-0.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.2
+Version: 0.11.4
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

lstosa-0.11.4/crontab/crontab.txt

@@ -0,0 +1,83 @@
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_TZ=UTC
+# Update astropy auxiliary data (e.g. IERS) every week
+0 0 * * SUN source /fefs/aswg/lstosa/utils/osa-conda; python /fefs/aswg/data/aux/astropy/get_iers_data.py  > /dev/null 2>&1
+#
+# lstosa jobs
+#
+# [COPY OBS OVERVIEW] Copy observations overview and daily camera checks to lst1 datacheck webserver
+#
+00 09 * * * /usr/bin/bash /fefs/aswg/lstosa/misc/copy_camera_daily_check_to_www.sh >/dev/null 2>&1
+#
+# [RUN SUMMARY] Produce the run summary file at the end of the observations to make sure it contains all the runs taken
+#
+50 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_create_run_summary -d $obsdate -o /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunSummary --r0-path=/fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events --overwrite > /dev/null 2>&1
+#
+# [MERGED RUN SUMMARY] Produce the merged run summary file containing all the runs taken by the LST.
+#                      This only appends the runs of the given date.
+#
+55 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_merge_run_summaries -d $obsdate /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Catalog/merged_RunSummary.ecsv --monitoring-path=/fefs/onsite/data/lst-pipe/LSTN-01/monitoring > /dev/null 2>&1
+#
+# [GAIN SELECTION]
+#
+00 07-15 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; gain_selection --no-queue-check -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate -s 6 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/GainSel_log/${obsdate}_LST1.log 2>&1
+#
+# [GAIN SELECTION CHECK]
+#
+20,50 08-16 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; gain_selection --check -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/GainSel_log/${obsdate}_check_LST1.log 2>&1
+#
+# [SIMULATED SEQUENCER] Launch sequencer once in the morning simulated. It also produces the run summary file if not found.
+#
+05 09 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -s -d $obsdate LST1 > /dev/null 2>&1
+#
+# [SEQUENCER 1] Run only up to DL1A
+#
+25,32 08-10 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg --no-dl1ab -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_1_LST1.log 2>&1
+#
+# [SEQUENCER_CATB_TAILCUTS] Runs from DL1A, produces Cat-B calibration and tailcut files
+#
+00,30 08-18 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_catB_tailcuts -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_tailcuts_LST1.log 2>&1
+#
+# [SEQUENCER 2] Finishes the processing producing DL1AB and DL2
+#
+04,40 09-19 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_2_LST1.log 2>&1
+#
+# [SEQUENCER WEB] Make sequencer xhtml table and copy it to the lst1 webserver
+#
+*/10 08-22 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_webmaker -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg > /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Minor_logs/${obsdate}_sequencer-web.log 2>&1; if [ $? = 0 ]; then scp /fefs/onsite/data/lst-pipe/LSTN-01/OSA/SequencerWeb/osa_status_$obsdate.html datacheck:/home/www/html/datacheck/lstosa/sequencer.xhtml; fi > /dev/null 2>&1
+#
+# [GAIN SEL WEB SEQUENCER] Make gain selection xhtml table and copy it to the lst1 webserver
+#
+*/10 07-18 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; gainsel_webmaker -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg > /dev/null 2>&1; if [ $? = 0 ]; then scp /fefs/onsite/data/lst-pipe/LSTN-01/OSA/GainSelWeb/osa_gainsel_status_$obsdate.html datacheck:/home/www/html/datacheck/lstosa/gainsel.xhtml; fi > /dev/null 2>&1
+#
+# [AUTOCLOSER] Launch the closer without forcing it (no -f option).
+#
+25 10-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; autocloser -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Autocloser_log/${obsdate}_LST1.log 2>&1
+#
+# [COPY DATACHECK] Copy the available calibration and DL1 datacheck to the LST1 webserver.
+#
+05,20,40 08-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; copy_datacheck -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 > /dev/null 2>&1
+#
+# [SOURCE CATALOG]
+#
+00 11-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; update_source_catalog -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate > /dev/null 2>&1
+#
+# [COPY CUMULATIVE LST-1 OBSERVATION TIME]
+# (added by Daniel Morcuende on 2024-04-05)
+#
+0 18 *  * FRI /usr/bin/bash /fefs/aswg/lstosa/misc/update_cumulative_obs_time.sh >/dev/null 2>&1
+
+
+# Daniel Morcuende (2023-11-08) periodically remove leftover jobs kept in the SLURM as pending after job dependency not satisfied because of failure (Reason in squeue: 'DependencyNeverSatisfied').
+# It does only look for lstosa jobs launched by sequencer with the prefix "LST1_" and also the merging jobs launched by autocloser like "longterm_dl1_datacheck".
+*/30 * * * * /local/home/lstanalyzer/remove_dependency_never_satisfied_jobs.sh > /dev/null 2>&1
+
+
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_TZ=UTC
+#
+# Restart camera processing of calibration runs in realtime in case it was killed for whatever reason
+#
+04 * * * * cd /local/home/lstanalyzer/camera/lst-camera-watchers;./lst1_watcher_restarter

{lstosa-0.11.2 → lstosa-0.11.4}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.2
+Version: 0.11.4
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.2 → lstosa-0.11.4}/src/lstosa.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ codemeta.json
 environment-lstcam.yml
 environment.yml
 pyproject.toml
+crontab/crontab-orig.txt
 crontab/crontab.txt
 dev/mysql.py
 docs/Makefile

{lstosa-0.11.2 → lstosa-0.11.4}/src/osa/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.11.2'
-__version_tuple__ = version_tuple = (0, 11, 2)
+__version__ = version = '0.11.4'
+__version_tuple__ = version_tuple = (0, 11, 4)
+
+__commit_id__ = commit_id = 'gd434a8783'

{lstosa-0.11.2 → lstosa-0.11.4}/src/osa/configs/sequencer.cfg

@@ -44,6 +44,7 @@ PROD_ID: v0.1.0
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
 DL1_PROD_ID: tailcut84
+DL1_PROD_ID_PREFIX: tailcut
 DL2_PROD_ID: model2
 
 [lstchain]

{lstosa-0.11.2 → lstosa-0.11.4}/src/osa/job.py

@@ -99,7 +99,10 @@ def are_all_jobs_correctly_finished(sequence_list):
     flag = True
     analysis_directory = Path(options.directory)
     for sequence in sequence_list:
-        history_files_list = analysis_directory.rglob(f"*{sequence.run}*.history")
+        if sequence.type != "DATA":
+            continue
+        else:
+            history_files_list = analysis_directory.rglob(f"*{sequence.run}*.history")
         
         if not options.test:
             try:
@@ -719,7 +722,7 @@ def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
 
     # Keep only the jobs corresponding to OSA sequences
     sacct_output = sacct_output[
-        (sacct_output["JobName"].str.contains("batch"))
+        (~sacct_output["JobID"].str.contains(r"\."))
         | (sacct_output["JobName"].str.contains("LST1"))
     ]
 

{lstosa-0.11.2 → lstosa-0.11.4}/src/osa/paths.py

@@ -433,11 +433,12 @@ def get_dl1_prod_id(config_filename):
         
     picture_thresh = data["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
     boundary_thresh = data["tailcuts_clean_with_pedestal_threshold"]["boundary_thresh"]
+    dl1_prod_id_prefix = cfg.get("LST1", "DL1_PROD_ID_PREFIX")
 
     if boundary_thresh == 4:
-        return f"tailcut{picture_thresh}{boundary_thresh}"
+        return f"{dl1_prod_id_prefix}{picture_thresh}{boundary_thresh}"
     else:
-        return f"tailcut{picture_thresh}{boundary_thresh:02d}"
+        return f"{dl1_prod_id_prefix}{picture_thresh}{boundary_thresh:02d}"
 
 
 def get_dl2_nsb_prod_id(rf_model: Path) -> str:

{lstosa-0.11.2 → lstosa-0.11.4}/src/osa/scripts/gain_selection.py

@@ -224,10 +224,10 @@ def launch_gainsel_for_data_run(
                             else:
                                 log.warning(f"Gain selection failed for run {run_id:05d}.{subrun:04d}")
                         elif gainsel_rc == "0":
-                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d},"
+                            log.debug(f"Gain selection finished successfully for run {run_id:05d}.{subrun:04d}, "
                                         "no additional jobs will be submitted for this subrun.") 
                 else:
-                    log.debug("Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
+                    log.debug(f"Creating and launching the gain selection sbatch script for subrun {run_id:05d}.{subrun:04d}")
                     if not simulate:
                         log_file = log_dir / f"r0_to_r0g_{run_id:05d}.{subrun:04d}.log"
                         job_file = log_dir / f"gain_selection_{run_id:05d}.{subrun:04d}.sh"

{lstosa-0.11.2 → lstosa-0.11.4}/src/osa/scripts/sequencer.py

@@ -238,7 +238,10 @@ def get_status_for_sequence(sequence, data_level) -> int:
     elif data_level == "DATACHECK":
         try:
             directory = options.directory / sequence.dl1_prod_id
+            alternative_directory = destination_dir(concept="DATACHECK", create_dir=False, dl1_prod_id=sequence.dl1_prod_id)
             files = list(directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+            files += list(alternative_directory.glob(f"datacheck_dl1_LST-1*{sequence.run}*.h5"))
+            
         except AttributeError:
             return 0