lstosa 0.10.12__tar.gz → 0.10.14__tar.gz

{lstosa-0.10.12 → lstosa-0.10.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.12
+Version: 0.10.14
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

lstosa-0.10.14/crontab/crontab.txt

@@ -0,0 +1,101 @@
+#SHELL=/usr/bin/bash
+#BASH_ENV="/local/home/lstanalyzer/.bashrc"
+#
+#-Datacheck jobs
+#
+#30 5 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Data_Check_daily_script_v2.sh
+#05 00-07 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_v7.sh
+#05 00-07 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v7.sh
+#05 00-07 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_v9.sh
+# EDIT Daniel Morcuende 2023-01-30: Running the data check during the night might be causing busy states of the camera during
+# data taking. I modified the cron above to launch only during the day.
+05 08-13 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_v9.sh
+#00 08 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v7.sh
+#00 09 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v7.sh
+00 08 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v9.sh
+#05 12 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v7.sh
+#05 12 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_no_nightsummary_v7.sh
+05 12 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_no_nightsummary_v9.sh
+#05 12 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v9.sh
+#00 14,16,18,20,21,22,23 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_v8.sh
+#00 14,16,18,20,21,22,23 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_close_night_v8.sh
+#00 14,16,18,20,21,22,23 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_v10.sh
+# EDIT Daniel Morcuende 2023-01-30: Running the data check during the night might be causing busy states of the camera during
+# data taking. I modified the cron above to launch only during the day up to 18:00 UTC.
+00 14,16,18 * * * /usr/bin/bash /fefs/onsite/pipeline/data-check/Process_Run_incremental_script_v10.sh
+#40 * * * *  /usr/bin/bash /fefs/onsite/pipeline/data-check/test_cron.sh
+
+
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_TZ=UTC
+# Update astropy auxiliary data (e.g. IERS) every week
+0 0 * * SUN source /fefs/aswg/lstosa/utils/osa-conda; python /fefs/aswg/data/aux/astropy/get_iers_data.py  > /dev/null 2>&1
+#
+# lstosa jobs
+#
+# [COPY OBS OVERVIEW] Copy observations overview and daily camera checks to lst1 datacheck webserver
+#
+00 09 * * * /usr/bin/bash /fefs/aswg/lstosa/misc/copy_camera_daily_check_to_www.sh >/dev/null 2>&1
+#
+# [RUN SUMMARY] Produce the run summary file at the end of the observations to make sure it contains all the runs taken
+#
+05 07 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_create_run_summary -d $obsdate --overwrite > /dev/null 2>&1
+#
+# [MERGED RUN SUMMARY] Produce the merged run summary file containing all the runs taken by the LST.
+#                      This only appends the runs of the given date.
+#
+15 07 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; source /fefs/aswg/lstosa/utils/osa-conda; lstchain_merge_run_summaries -d $obsdate /fefs/aswg/data/real/OSA/Catalog/merged_RunSummary.ecsv > /dev/null 2>&1
+#
+# [GAIN SELECTION]
+#
+#50 07 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; conda activate /fefs/aswg/software/conda/envs/osa-dev; gain_selection --no-queue-check -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg -d $obsdate -o /fefs/aswg/data/real/R0G -s 7 >> /fefs/aswg/data/real/OSA/GainSel_log/${obsdate}_LST1.log 2>&1
+#
+# [GAIN SELECTION CHECK]
+#
+#20 08 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; conda activate /fefs/aswg/software/conda/envs/osa-dev; gain_selection --check -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg -d $obsdate -o /fefs/aswg/data/real/R0G -s 7 >> /fefs/aswg/data/real/OSA/GainSel_log/${obsdate}_check_LST1.log 2>&1
+#
+# [SIMULATED SEQUENCER] Launch sequencer once in the morning simulated. It also produces the run summary file if not found.
+#
+10 08 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg -s -d $obsdate LST1 >> /fefs/aswg/data/real/OSA/Sequencer_log/${obsdate}_LST1.log 2>&1
+#
+# [SEQUENCER] Launch sequencer once in the morning
+#
+50 07 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg -d $obsdate --no-gainsel --no-dl2 LST1 > /dev/null 2>&1
+#
+# [WEB SEQUENCER] Make sequencer xhtml table and copy it to the lst1 webserver
+#
+*/15 08-18 * * * obsdate=`date +\%Y\%m\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; sequencer_webmaker -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg; if [ $? = 0 ]; then scp /fefs/aswg/data/real/OSA/SequencerWeb/osa_status_$obsdate.html datacheck:/home/www/html/datacheck/lstosa/sequencer.xhtml; fi > /dev/null 2>&1
+#
+# [AUTOCLOSER] Launch the closer without forcing it (no -f option).
+#
+0 10-18/2 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; autocloser -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg -d $obsdate --no-gainsel --no-dl2 LST1 >> /fefs/aswg/data/real/OSA/Autocloser_log/${obsdate}_LST1.log 2>&1
+#
+# [COPY DATACHECK] Copy the available calibration and DL1 datacheck to the LST1 webserver.
+#
+05,20,40 08-18 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; copy_datacheck -c /fefs/aswg/lstosa/cfg/sequencer_v0.10.cfg -d $obsdate LST1 > /dev/null 2>&1
+#
+# [SOURCE CATALOG]
+#
+00 11-19 * * * obsdate=`date +\%Y-\%m-\%d -d yesterday`; export obsdate; source /fefs/aswg/lstosa/utils/osa-conda; update_source_catalog $obsdate > /dev/null 2>&1
+#
+# [COPY CUMULATIVE LST-1 OBSERVATION TIME]
+# (added by Daniel Morcuende on 2024-04-05)
+#
+0 11 *  * FRI /usr/bin/bash /fefs/aswg/lstosa/misc/update_cumulative_obs_time.sh >/dev/null 2>&1
+
+
+# Daniel Morcuende (2023-11-08) periodically remove leftover jobs kept in the SLURM as pending after job dependency not satisfied because of failure (Reason in squeue: 'DependencyNeverSatisfied').
+# It does only look for lstosa jobs launched by sequencer with the prefix "LST1_" and also the merging jobs launched by autocloser like "longterm_dl1_datacheck".
+*/30 * * * * /local/home/lstanalyzer/remove_dependency_never_satisfied_jobs.sh > /dev/null 2>&1
+
+
+
+
+SHELL=/usr/bin/bash
+BASH_ENV="/local/home/lstanalyzer/.bashrc"
+CRON_TZ=UTC
+#
+# Restart camera processing of calibration runs in realtime in case it was killed for whatever reason
+#
+04 * * * * cd /local/home/lstanalyzer/camera/lst-camera-watchers;./lst1_watcher_restarter

{lstosa-0.10.12 → lstosa-0.10.14}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.12
+Version: 0.10.14
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.10.12 → lstosa-0.10.14}/src/lstosa.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ README.md
 codemeta.json
 environment.yml
 pyproject.toml
+crontab/crontab.txt
 dev/mysql.py
 docs/Makefile
 docs/authors.rst

{lstosa-0.10.12 → lstosa-0.10.14}/src/osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.12'
-__version_tuple__ = version_tuple = (0, 10, 12)
+__version__ = version = '0.10.14'
+__version_tuple__ = version_tuple = (0, 10, 14)

{lstosa-0.10.12 → lstosa-0.10.14}/src/osa/scripts/gain_selection.py

@@ -24,7 +24,7 @@ log = myLogger(logging.getLogger(__name__))
 
 PATH = "PATH=/fefs/aswg/software/offline_dvr/bin:$PATH"
 
-parser = argparse.ArgumentParser(add_help=False)
+parser = argparse.ArgumentParser()
 parser.add_argument(
         "--check",                                                                                       
         action="store_true",
@@ -50,13 +50,14 @@ parser.add_argument(
         "--date",                                                                                        
         default=None,
         type=str,
-        help="Night to apply the gain selection",
+        help="Night to apply the gain selection in YYYYMMDD format",
 )                                                                                                        
 parser.add_argument(                                                                                     
         "-l",                                                                                            
         "--dates-file",
         default=None,
-        help="List of dates to apply the gain selection",
+        help="List of dates to apply the gain selection. The input file should list"
+        "the dates in the format YYYYMMDD, one date per line.",
 )
 parser.add_argument(                                                                                     
         "-o",                                                                                            
@@ -79,12 +80,20 @@ parser.add_argument(
         default=18,
         help="Time to stop gain selection in HH format. Default is 18.",
 )
+parser.add_argument(                                                                                     
+        "-t",                                                                                            
+        "--tool",
+        type=str,
+        default=None,
+        help="Choose tool to apply the gain selection regardless the date. Possible options are: lst_dvr (by default used for dates "
+        "previous to 20231205) and lstchain_r0_to_r0g (by default used for dates later than 20231205).",
+)
 
 def get_sbatch_script(
-    run_id, subrun, input_file, output_dir, log_dir, log_file, ref_time, ref_counter, module, ref_source, script
+    run_id, subrun, input_file, output_dir, log_dir, log_file, ref_time, ref_counter, module, ref_source, tool
 ):
     """Build the sbatch job pilot script for running the gain selection."""
-    if script=="old":
+    if tool == "lst_dvr":
         return dedent(
             f"""\
         #!/bin/bash
@@ -98,7 +107,7 @@ def get_sbatch_script(
         lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
         """
         )
-    elif script=="new":
+    elif tool == "lstchain_r0_to_r0g":
         return dedent(
             f"""\
         #!/bin/bash
@@ -113,16 +122,17 @@ def get_sbatch_script(
         """
         )
 
-def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None, no_queue_check: bool = False):
+def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None, tool: str = None, no_queue_check: bool = False):
     """
     Submit the jobs to apply the gain selection to the data for a given date
     on a subrun-by-subrun basis.
     """
 
-    if date < "20231205":
-        script = "old"
-    else:
-        script = "new"
+    if not tool:
+        if date < "20231205":
+            tool = "lst_dvr"
+        else:
+            tool = "lstchain_r0_to_r0g"
 
     run_summary_dir = Path("/fefs/aswg/data/real/monitoring/RunSummary")
     run_summary_file = run_summary_dir / f"RunSummary_{date}.ecsv"
@@ -156,7 +166,7 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
         subrun_numbers = [int(file[-12:-8]) for file in files]
         input_files = []
 
-        if date < "20231205" and ref_source not in ["UCTS", "TIB"]:
+        if tool == "lst_dvr" and ref_source not in ["UCTS", "TIB"]:
             input_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
             log.info(
                 f"Run {run_id} does not have UCTS or TIB info, so gain selection cannot"
@@ -199,7 +209,7 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
                             ref_counter,
                             module,
                             ref_source,
-                            script,
+                            tool,
                         )
                     )
                 sp.run(["sbatch", job_file], check=True)
@@ -208,7 +218,8 @@ def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path =
     log.info(f"Found {len(calib_runs)} NO-DATA runs")
 
     for run in calib_runs:
-        log.info(f"Copying R0 files corresponding to run {run} directly to {output_dir}")
+        run_id = run["run_id"]
+        log.info(f"Copying R0 files corresponding to run {run_id} directly to {output_dir}")
         # Avoid copying files while it is still night time
         wait_for_daytime(start, end)
 
@@ -327,7 +338,14 @@ def main():
             check_failed_jobs(args.date, args.output_basedir)
         else:
             log.info(f"Applying gain selection to date {args.date}")
-            apply_gain_selection(args.date, args.start_time, args.end_time, args.output_basedir, no_queue_check=args.no_queue_check)
+            apply_gain_selection(
+                args.date, 
+                args.start_time, 
+                args.end_time, 
+                args.output_basedir,
+                args.tool,
+                no_queue_check=args.no_queue_check, 
+            )
 
 
     elif args.dates_file:
@@ -341,7 +359,14 @@ def main():
         else:
             for date in list_of_dates:
                 log.info(f"Applying gain selection to date {date}")
-                apply_gain_selection(date, args.start_time, args.end_time, args.output_basedir, no_queue_check=args.no_queue_check)
+                apply_gain_selection(
+                    date, 
+                    args.start_time, 
+                    args.end_time,
+                    args.output_basedir,
+                    args.tool,
+                    no_queue_check=args.no_queue_check,
+                )
             log.info("Done! No more dates to process.")
 
 

{lstosa-0.10.12 → lstosa-0.10.14}/src/osa/scripts/reprocessing.py

@@ -22,7 +22,7 @@ def number_of_pending_jobs():
 
 
 def run_script(
-    script: str, date, config: Path, no_dl2: bool, no_calib: bool, simulate: bool, force: bool
+    script: str, date, config: Path, no_dl2: bool, no_gainsel: bool, no_calib: bool, simulate: bool, force: bool
 ):
     """Run the sequencer for a given date."""
     osa_config = Path(config).resolve()
@@ -32,6 +32,9 @@ def run_script(
     if no_dl2:
         cmd.append("--no-dl2")
 
+    if no_gainsel:
+        cmd.append("--no-gainsel")
+
     if no_calib:
         cmd.append("--no-calib")
 
@@ -64,6 +67,7 @@ def get_list_of_dates(dates_file):
 
 @click.command()
 @click.option("--no-dl2", is_flag=True, help="Do not run the DL2 step.")
+@click.option("--no-gainsel", is_flag=True, help="Do not require gain selection to be finished.")
 @click.option("--no-calib", is_flag=True, help="Do not run the calibration step.")
 @click.option("-s", "--simulate", is_flag=True, help="Activate simulation mode.")
 @click.option("-f", "--force", is_flag=True, help="Force the autocloser to close the day.")
@@ -83,6 +87,7 @@ def main(
     dates_file: Path = None,
     config: Path = DEFAULT_CFG,
     no_dl2: bool = False,
+    no_gainsel: bool = False,
     no_calib: bool = False,
     simulate: bool = False,
     force: bool = False,
@@ -102,7 +107,7 @@ def main(
         # Avoid running jobs while it is still night time
         wait_for_daytime()
 
-        run_script(script, date, config, no_dl2, no_calib, simulate, force)
+        run_script(script, date, config, no_dl2, no_gainsel, no_calib, simulate, force)
         log.info("Waiting 1 minute to launch the process for the next date...\n")
         time.sleep(60)
 

{lstosa-0.10.12 → lstosa-0.10.14}/src/osa/scripts/sequencer_webmaker.py

@@ -55,7 +55,7 @@ def html_content(body: str, date: str) -> str:
     )
 
 
-def get_sequencer_output(date: str, config: str, test=False) -> list:
+def get_sequencer_output(date: str, config: str, test=False, no_gainsel=False) -> list:
     """Call sequencer to get table with the sequencer status report.
 
     Parameters
@@ -82,6 +82,9 @@ def get_sequencer_output(date: str, config: str, test=False) -> list:
         options.tel_id,
     ]
 
+    if no_gainsel:
+        commandargs.insert(1, "--no-gainsel")
+        
     if test:
         commandargs.insert(-1, "-t")
 
@@ -144,7 +147,7 @@ def main():
         sys.exit(1)
 
     # Get the table with the sequencer status report:
-    lines = get_sequencer_output(date, args.config, args.test)
+    lines = get_sequencer_output(date, args.config, test=args.test, no_gainsel=args.no_gainsel)
 
     # Build the html sequencer table that will be place in the body of the HTML file
     matrix = lines_to_matrix(lines)

{lstosa-0.10.12 → lstosa-0.10.14}/src/osa/utils/cliopts.py

@@ -447,6 +447,12 @@ def sequencer_webmaker_argparser():
     parser = ArgumentParser(
         description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser]
     )
+    parser.add_argument(
+        "--no-gainsel",
+        action="store_true",
+        default=False,
+        help="Do not check if the gain selection finished correctly (default False)",
+    )
     options.tel_id = "LST1"
     options.prod_id = get_prod_id()