lstosa 0.11.1__tar.gz → 0.11.3__tar.gz

{lstosa-0.11.1 → lstosa-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.1
+Version: 0.11.3
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

lstosa-0.11.3/crontab/crontab.txt

@@ -0,0 +1,83 @@
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_TZ=UTC
+# Update astropy auxiliary data (e.g. IERS) every week
+0 0 * * SUN source /fefs/aswg/lstosa/utils/osa-conda; python /fefs/aswg/data/aux/astropy/get_iers_data.py  > /dev/null 2>&1
+#
+# lstosa jobs
+#
+# [COPY OBS OVERVIEW] Copy observations overview and daily camera checks to lst1 datacheck webserver
+#
+00 09 * * * /usr/bin/bash /fefs/aswg/lstosa/misc/copy_camera_daily_check_to_www.sh >/dev/null 2>&1
+#
+# [RUN SUMMARY] Produce the run summary file at the end of the observations to make sure it contains all the runs taken
+#
+50 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_create_run_summary -d $obsdate -o /fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunSummary --r0-path=/fefs/onsite/data/R0/LSTN-01/lst-arraydaq/events --overwrite > /dev/null 2>&1
+#
+# [MERGED RUN SUMMARY] Produce the merged run summary file containing all the runs taken by the LST.
+#                      This only appends the runs of the given date.
+#
+55 06 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_merge_run_summaries -d $obsdate /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Catalog/merged_RunSummary.ecsv --monitoring-path=/fefs/onsite/data/lst-pipe/LSTN-01/monitoring > /dev/null 2>&1
+#
+# [GAIN SELECTION]
+#
+00 07-15 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; gain_selection --no-queue-check -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate -s 6 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/GainSel_log/${obsdate}_LST1.log 2>&1
+#
+# [GAIN SELECTION CHECK]
+#
+20,50 08-16 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; gain_selection --check -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/GainSel_log/${obsdate}_check_LST1.log 2>&1
+#
+# [SIMULATED SEQUENCER] Launch sequencer once in the morning simulated. It also produces the run summary file if not found.
+#
+05 09 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -s -d $obsdate LST1 > /dev/null 2>&1
+#
+# [SEQUENCER 1] Run only up to DL1A
+#
+25,32 08-10 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg --no-dl1ab -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_1_LST1.log 2>&1
+#
+# [SEQUENCER_CATB_TAILCUTS] Runs from DL1A, produces Cat-B calibration and tailcut files
+#
+00,30 08-18 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_catB_tailcuts -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_tailcuts_LST1.log 2>&1
+#
+# [SEQUENCER 2] Finishes the processing producing DL1AB and DL2
+#
+04,40 09-19 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Sequencer_log/${obsdate}_2_LST1.log 2>&1
+#
+# [SEQUENCER WEB] Make sequencer xhtml table and copy it to the lst1 webserver
+#
+*/10 08-22 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_webmaker -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg > /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Minor_logs/${obsdate}_sequencer-web.log 2>&1; if [ $? = 0 ]; then scp /fefs/onsite/data/lst-pipe/LSTN-01/OSA/SequencerWeb/osa_status_$obsdate.html datacheck:/home/www/html/datacheck/lstosa/sequencer.xhtml; fi > /dev/null 2>&1
+#
+# [GAIN SEL WEB SEQUENCER] Make gain selection xhtml table and copy it to the lst1 webserver
+#
+*/10 07-18 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; gainsel_webmaker -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg > /dev/null 2>&1; if [ $? = 0 ]; then scp /fefs/onsite/data/lst-pipe/LSTN-01/OSA/GainSelWeb/osa_gainsel_status_$obsdate.html datacheck:/home/www/html/datacheck/lstosa/gainsel.xhtml; fi > /dev/null 2>&1
+#
+# [AUTOCLOSER] Launch the closer without forcing it (no -f option).
+#
+25 10-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; autocloser -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 >> /fefs/onsite/data/lst-pipe/LSTN-01/OSA/Autocloser_log/${obsdate}_LST1.log 2>&1
+#
+# [COPY DATACHECK] Copy the available calibration and DL1 datacheck to the LST1 webserver.
+#
+05,20,40 08-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; copy_datacheck -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate LST1 > /dev/null 2>&1
+#
+# [SOURCE CATALOG]
+#
+00 11-23 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; update_source_catalog -c /fefs/aswg/lstosa/cfg/sequencer_v0.11.cfg -d $obsdate > /dev/null 2>&1
+#
+# [COPY CUMULATIVE LST-1 OBSERVATION TIME]
+# (added by Daniel Morcuende on 2024-04-05)
+#
+0 18 *  * FRI /usr/bin/bash /fefs/aswg/lstosa/misc/update_cumulative_obs_time.sh >/dev/null 2>&1
+
+
+# Daniel Morcuende (2023-11-08) periodically remove leftover jobs kept in the SLURM as pending after job dependency not satisfied because of failure (Reason in squeue: 'DependencyNeverSatisfied').
+# It does only look for lstosa jobs launched by sequencer with the prefix "LST1_" and also the merging jobs launched by autocloser like "longterm_dl1_datacheck".
+*/30 * * * * /local/home/lstanalyzer/remove_dependency_never_satisfied_jobs.sh > /dev/null 2>&1
+
+
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_TZ=UTC
+#
+# Restart camera processing of calibration runs in realtime in case it was killed for whatever reason
+#
+04 * * * * cd /local/home/lstanalyzer/camera/lst-camera-watchers;./lst1_watcher_restarter

{lstosa-0.11.1 → lstosa-0.11.3}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.1
+Version: 0.11.3
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.1 → lstosa-0.11.3}/src/lstosa.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ codemeta.json
 environment-lstcam.yml
 environment.yml
 pyproject.toml
+crontab/crontab-orig.txt
 crontab/crontab.txt
 dev/mysql.py
 docs/Makefile

{lstosa-0.11.1 → lstosa-0.11.3}/src/osa/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.11.1'
-__version_tuple__ = version_tuple = (0, 11, 1)
+__version__ = version = '0.11.3'
+__version_tuple__ = version_tuple = (0, 11, 3)

{lstosa-0.11.1 → lstosa-0.11.3}/src/osa/job.py

@@ -99,7 +99,10 @@ def are_all_jobs_correctly_finished(sequence_list):
     flag = True
     analysis_directory = Path(options.directory)
     for sequence in sequence_list:
-        history_files_list = analysis_directory.rglob(f"*{sequence.run}*.history")
+        if sequence.type != "DATA":
+            continue
+        else:
+            history_files_list = analysis_directory.rglob(f"*{sequence.run}*.history")
         
         if not options.test:
             try:

{lstosa-0.11.1 → lstosa-0.11.3}/src/osa/scripts/autocloser.py

@@ -73,16 +73,16 @@ class Telescope:
 
         if self.is_closed():
             log.info(f"{self.telescope} is already closed! Ignoring {self.telescope}")
-            return
+            sys.exit(0)
         if not analysis_path(self.telescope).exists():
             log.warning(
                 f"Analysis directory does not exist for {self.telescope}! "
                 f"Ignoring {self.telescope}"
             )
-            return
+            sys.exit(0)
         if not self.lock_automatic_sequencer() and not ignore_cronlock:
             log.warning(f"{self.telescope} already locked! Ignoring {self.telescope}")
-            return
+            sys.exit(0)
         if not self.simulate_sequencer(date, config_file, test, no_gainsel):
             log.warning(
                 f"Simulation of the sequencer failed "

{lstosa-0.11.1 → lstosa-0.11.3}/src/osa/scripts/datasequence.py

@@ -207,7 +207,7 @@ def dl1ab(run_str: str, dl1b_config: Path, dl1_prod_id: str) -> int:
                 f"Cat-B calibration did not finish yet for run {run_str[:5]}. "
                 "Please try again later."
             )
-            sys.exit(1)
+            sys.exit(0)
 
     if options.simulate:
         return 0

{lstosa-0.11.1 → lstosa-0.11.3}/src/osa/scripts/reprocessing.py

@@ -22,7 +22,17 @@ def number_of_pending_jobs():
 
 
 def run_script(
-    script: str, date, config: Path, no_dl2: bool, no_gainsel: bool, no_calib: bool, simulate: bool, force: bool
+    script: str, 
+    date, 
+    config: Path, 
+    no_dl2: bool, 
+    no_gainsel: bool, 
+    no_calib: bool, 
+    no_dl1ab: bool, 
+    simulate: bool, 
+    force: bool,
+    overwrite_tailcuts: bool,
+    overwrite_catb: bool,
 ):
     """Run the sequencer for a given date."""
     osa_config = Path(config).resolve()
@@ -38,12 +48,21 @@ def run_script(
     if no_calib:
         cmd.append("--no-calib")
 
+    if no_dl1ab:
+        cmd.append("--no-dl1ab")
+
     if simulate:
         cmd.append("--simulate")
 
     if force:
         cmd.append("--force")
 
+    if overwrite_tailcuts:
+        cmd.append("--overwrite-tailcuts")
+    
+    if overwrite_catb:
+        cmd.append("--overwrite-catB")
+        
     # Append the telescope to the command in the last place
     cmd.append("LST1")
 
@@ -69,8 +88,11 @@ def get_list_of_dates(dates_file):
 @click.option("--no-dl2", is_flag=True, help="Do not run the DL2 step.")
 @click.option("--no-gainsel", is_flag=True, help="Do not require gain selection to be finished.")
 @click.option("--no-calib", is_flag=True, help="Do not run the calibration step.")
+@click.option("--no-dl1ab", is_flag=True, help="Do not run the DL1AB step.")
 @click.option("-s", "--simulate", is_flag=True, help="Activate simulation mode.")
 @click.option("-f", "--force", is_flag=True, help="Force the autocloser to close the day.")
+@click.option("--overwrite-tailcuts", is_flag=True, help="Overwrite the tailcuts config file if it already exists.")
+@click.option("--overwrite-catB", is_flag=True, help="Overwrite the Cat-B calibration files if they already exist.")
 @click.option(
     "-c",
     "--config",
@@ -79,7 +101,7 @@ def get_list_of_dates(dates_file):
     help="Path to the OSA config file.",
 )
 @click.argument(
-    "script", type=click.Choice(["sequencer", "closer", "copy_datacheck", "autocloser"])
+    "script", type=click.Choice(["sequencer", "closer", "copy_datacheck", "autocloser", "sequencer_catB_tailcuts"])
 )
 @click.argument("dates-file", type=click.Path(exists=True))
 def main(
@@ -89,9 +111,12 @@ def main(
     no_dl2: bool = False,
     no_gainsel: bool = False,
     no_calib: bool = False,
+    no_dl1ab: bool = False,
     simulate: bool = False,
     force: bool = False,
-):
+    overwrite_tailcuts: bool = False,
+    overwrite_catb: bool = False,
+    ):
     """
     Loop over the dates listed in the input file and launch the script for each of them.
     The input file should list the dates in the format YYYY-MM-DD one date per line.
@@ -107,7 +132,19 @@ def main(
         # Avoid running jobs while it is still night time
         wait_for_daytime()
 
-        run_script(script, date, config, no_dl2, no_gainsel, no_calib, simulate, force)
+        run_script(
+            script,
+            date,
+            config,
+            no_dl2,
+            no_gainsel,
+            no_calib,
+            no_dl1ab,
+            simulate,
+            force,
+            overwrite_tailcuts,
+            overwrite_catb,
+        )
         log.info("Waiting 1 minute to launch the process for the next date...\n")
         time.sleep(60)
 

{lstosa-0.11.1 → lstosa-0.11.3}/src/osa/scripts/sequencer_catB_tailcuts.py

@@ -56,6 +56,12 @@ parser.add_argument(
     default=False,
     help="Overwrite the tailcuts config file if it already exists.",
 )
+parser.add_argument(
+    "--overwrite-catB",
+    action="store_true",
+    default=False,
+    help="Overwrite the Cat-B calibration files if they already exist.",
+)
 parser.add_argument(
     "tel_id",
     choices=["ST", "LST1", "LST2", "all"],
@@ -111,7 +117,7 @@ def launch_catB_calibration(run_id: int):
     before and it finished successfully, it creates a catB_{run}.closed file.
     """
     job_id = get_catB_last_job_id(run_id)
-    if job_id:
+    if job_id and not options.overwrite_catB:
         job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
         if job_status.item() in ["RUNNING", "PENDING"]:
             log.debug(f"Job {job_id} (corresponding to run {run_id:05d}) is still running.")
@@ -129,8 +135,11 @@ def launch_catB_calibration(run_id: int):
 
     else:
         if catB_calibration_file_exists(run_id):
-            log.info(f"Cat-B calibration file already produced for run {run_id:05d}.")
-            return 
+            if not options.overwrite_catB:
+                log.info(f"Cat-B calibration file already produced for run {run_id:05d}.")
+                return 
+            else:
+                log.info(f"Cat-B calibration file already produced for run {run_id:05d}. Overwriting it.")
 
         command = cfg.get("lstchain", "catB_calibration")
         if cfg.getboolean("lstchain", "use_lstcam_env_for_CatB_calib"):
@@ -162,6 +171,9 @@ def launch_catB_calibration(run_id: int):
             cmd.append(f"--dl1-dir={analysis_dir}")
             cmd.append(f"--lstchain-version={lstchain_version[1:]}")
 
+        if options.overwrite_catB:
+            cmd.append("--yes")
+
         if not options.simulate:
             job = sp.run(cmd, encoding="utf-8", capture_output=True, text=True, check=True)
             job_id = job.stdout.strip()
@@ -217,6 +229,7 @@ def main():
     options.tel_id = opts.tel_id
     options.simulate = opts.simulate
     options.overwrite_tailcuts = opts.overwrite_tailcuts
+    options.overwrite_catB = opts.overwrite_catB
     options.date = opts.date
     options.date = set_default_date_if_needed()
     options.configfile = opts.config.resolve()