lstosa 0.10.6__py3-none-any.whl → 0.10.7__py3-none-any.whl

{lstosa-0.10.6.dist-info → lstosa-0.10.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.6
+Version: 0.10.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,8 +20,8 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy ~=5.0
-Requires-Dist: lstchain ~=0.10.0
-Requires-Dist: matplotlib ~=3.7
+Requires-Dist: lstchain >=0.10.5
+Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pyyaml

{lstosa-0.10.6.dist-info → lstosa-0.10.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=0ek8_w_jF04UB6FpW1K7d1RgH0zQMd77H5TbKzdthjI,413
+osa/_version.py,sha256=Tqz9jfXJ9oRFN5_F1Y4Bx09mLTPY4KYQNRrHQQNprBE,413
 osa/conftest.py,sha256=b_26FciV8NZAdHj2S10E8cE4bettinLJY83Roe1a5L4,19358
 osa/job.py,sha256=acdiaRlHsc2GLyz2Oz_RBiawJo9QKK-tsajYdM7X0_Q,25393
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
@@ -39,8 +39,8 @@ osa/scripts/calibration_pipeline.py,sha256=g9o1chqCRRSo7GNuQZRABjGnJYjZyfhkPgRLP
 osa/scripts/closer.py,sha256=eZLUlqot4EXwL9IlU21tKRZ2GvZ4i8ill0c2QdEkoDY,15219
 osa/scripts/copy_datacheck.py,sha256=tfDs6oTdPbii4BOXp6bTHuED0xNJeqaPFrv6Ed7ZnWc,3104
 osa/scripts/datasequence.py,sha256=gXAp8arbLPEK-sca9VnME6-2XfUzBFIoEFchlUZYrXI,9260
-osa/scripts/gain_selection.py,sha256=BmNjtfeCFw5OGymaLG1PFPDcn8-yDOG6fWaifOCOHLM,8095
-osa/scripts/provprocess.py,sha256=pKR3wf3krAGmeHen_au2RmqJJtNBcX29J3XPQYKrHRE,18536
+osa/scripts/gain_selection.py,sha256=yaz2fKicuCDgLbD3fhb3l9LBlXLEEO3yuZO-oDc2IPo,8084
+osa/scripts/provprocess.py,sha256=mufkZe6_qwH3DGqTFxINIc01hciF5RMpw3n_Mp7vwXU,18629
 osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
 osa/scripts/reprocessing.py,sha256=D-J8Rl3GrkWpxYkk6ci79oJOMewgGdxLkQgaHCAZuqs,3417
 osa/scripts/sequencer.py,sha256=6Cg-eIExk0eN8-HXkO6DBeFEWOI-FznDueqJEpbEzos,8163
@@ -48,9 +48,9 @@ osa/scripts/sequencer_webmaker.py,sha256=99P1rJiBlvNidDeppvtw4kOoAfsOxSMJ2sm4xlg
 osa/scripts/show_run_summary.py,sha256=UpdTDRfncdUBKhPCy3reCKSk_saOsTLeMdyZHUV300Q,5197
 osa/scripts/show_run_summary_tcu.py,sha256=SoDLVKdQHOJkfenFguBOfXf10Gyv7heXSQAFnDVZqMs,2468
 osa/scripts/simulate_processing.py,sha256=NiRVYiwZENt_mnKncytgJT23_-tJMb1B5PswM12nnX4,6941
-osa/scripts/update_source_catalog.py,sha256=lJXYWI92YB0SrafsND4do0kjPOV_mP2MzuyMfBJbY-0,6265
+osa/scripts/update_source_catalog.py,sha256=GHwWFc-y6S4KkUJxUVM5drdAnVDD0-n3D-Tv3CCmh4E,7218
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=UiUTXw5uK2iMZt7thRS1UfqoAY5b9s8k8TB143BnXcw,12587
+osa/scripts/tests/test_osa_scripts.py,sha256=xgBic0sW-eao2MSgYgnP-MB6QbmDK6IIY69m0SGdzaA,12588
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/tests/test_jobs.py,sha256=F0jsHZ9BYB_cCHHxlXyO9v1E5_-mBJhuFtshtsAtnXo,15260
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
@@ -75,9 +75,9 @@ osa/workflow/dl3.py,sha256=kz7L5jcKHFJ--UdQ8HQKLzWO6nxc2LLOTz42ExcqzTk,9921
 osa/workflow/stages.py,sha256=WYgUM2XDIaUjCc4_Zs_VSGW6gk73EaKcHk6ZMnPds74,6692
 osa/workflow/tests/test_dl3.py,sha256=aY5bb-8OcZGAXG3JPCZihChzkA_GsWjRIa31BHZn3Dg,299
 osa/workflow/tests/test_stages.py,sha256=TmC00XFACWZp740TQeFaokWi3C50ovj_XGiySWrrdZk,3944
-lstosa-0.10.6.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.10.6.dist-info/METADATA,sha256=Z1GxQGKizr5McPZEq-mhPLuYRmNjzLV8teV87W62kdA,7354
-lstosa-0.10.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-lstosa-0.10.6.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
-lstosa-0.10.6.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.10.6.dist-info/RECORD,,
+lstosa-0.10.7.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.10.7.dist-info/METADATA,sha256=v28ofFiZTcCliCrNe6aGtQ-C9GxvHlJm__FfAdKTKfQ,7348
+lstosa-0.10.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+lstosa-0.10.7.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
+lstosa-0.10.7.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.10.7.dist-info/RECORD,,

osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.6'
-__version_tuple__ = version_tuple = (0, 10, 6)
+__version__ = version = '0.10.7'
+__version_tuple__ = version_tuple = (0, 10, 7)

osa/scripts/gain_selection.py

@@ -20,7 +20,7 @@ from osa.job import get_sacct_output, FORMAT_SLURM
 
 log = myLogger(logging.getLogger(__name__))
 
-PATH = "PATH=/fefs/aswg/software/gain_selection/bin:$PATH"
+PATH = "PATH=/fefs/aswg/software/offline_dvr/bin:$PATH"
 
 
 def get_sbatch_script(
@@ -36,7 +36,7 @@ def get_sbatch_script(
     #SBATCH --job-name "gain_selection_{run_id:05d}"
     #SBATCH --export {PATH}
 
-    lst_select_gain {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
+    lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
     """
     )
 

osa/scripts/provprocess.py

@@ -81,8 +81,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number):
         filter_cut = "all"
     cuts = {
         "calibration": ["drs4_pedestal", "calibrate_charge"],
-        "r0_to_dl1": ["r0_to_dl1", "dl1ab"],
-        "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"],
+        "r0_to_dl1": ["r0_to_dl1", "dl1ab", "dl1_datacheck"],
+        "dl1_to_dl2": ["dl1_to_dl2"],
     }
     cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"]
 
@@ -413,29 +413,31 @@ def produce_provenance(session_log_filename, base_filename):
             read_prov(filename=session_log_filename),
             str(paths_r0_dl1["out_path"]),
         )
-        dl1_lines = plines_r0 + plines_ab[1:]
+        plines_check = parse_lines_run(
+            "dl1_datacheck",
+            read_prov(filename=session_log_filename),
+            str(paths_r0_dl1["out_path"]),
+        )
+        dl1_lines = plines_r0 + plines_ab[1:] + plines_check[1:]
 
     # create r0_to_dl1 prov files only if filtering
     if options.filter == "r0_to_dl1":
-        produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1)
+        produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1)
 
     if options.filter == "dl1_to_dl2" or not options.filter:
-        paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
-        plines_check = parse_lines_run(
-            "dl1_datacheck",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        plines_dl2 = parse_lines_run(
-            "dl1_to_dl2",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        dl1_dl2_lines = plines_check + plines_dl2[1:]
+        if not options.no_dl2:
+            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
+            plines_dl2 = parse_lines_run(
+                "dl1_to_dl2",
+                read_prov(filename=session_log_filename),
+                str(paths_dl1_dl2["out_path"]),
+            )
+            dl1_dl2_lines = plines_dl2
 
     # create dl1_to_dl2 prov files only if filtering
     if options.filter == "dl1_to_dl2":
-        produce_provenance_files(plines_check + plines_dl2[1:], paths_dl1_dl2)
+        if not options.no_dl2:
+            produce_provenance_files(plines_dl2, paths_dl1_dl2)
 
     # create calibration_to_dl1 and calibration_to_dl2 prov files
     if not options.filter:

osa/scripts/tests/test_osa_scripts.py

@@ -99,10 +99,10 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 16
-    assert len(dl1["activity"]) == 4
-    assert len(dl1["used"]) == 13
-    assert len(dl1["wasGeneratedBy"]) == 7
+    assert len(dl1["entity"]) == 19
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)

osa/scripts/update_source_catalog.py

@@ -5,30 +5,30 @@ from pathlib import Path
 from textwrap import dedent
 
 import click
-import re
 import pandas as pd
 from astropy import units as u
-from astropy.table import Table, join, vstack, unique
+from astropy.table import Table, join, unique, vstack
 from astropy.time import Time
 from lstchain.io.io import dl1_params_lstcam_key
-from lstchain.reco.utils import get_effective_time, add_delta_t_key
+from lstchain.reco.utils import add_delta_t_key, get_effective_time
+
+from osa.paths import get_major_version
 from osa.utils.utils import get_lstchain_version
 
-pd.set_option('display.float_format', '{:.1f}'.format)
+pd.set_option("display.float_format", "{:.1f}".format)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s:%(levelname)s:%(message)s'
-)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(message)s")
 log = logging.getLogger(__name__)
 
 
 BASE_DL1 = Path("/fefs/aswg/data/real/DL1")
 BASE_MONITORING = Path("/fefs/aswg/data/real/monitoring")
+CATALOG_DIR = Path("/fefs/aswg/data/real/OSA/Catalog")
 
 
 def add_table_to_html(html_table):
-    return dedent(f"""\
+    return dedent(
+        f"""\
     <html>
       <head>
         <link href="osa.css" rel="stylesheet" type="text/css">
@@ -37,11 +37,13 @@ def add_table_to_html(html_table):
     {html_table}
     </body>
     </html>
-    """)
+    """
+    )
 
 
 def add_query_table_to_html(html_table):
-    return dedent(f"""\
+    return dedent(
+        f"""\
     <html>
      <head>
         <meta http-equiv="Content-type" content="text/html; charset=utf-8">
@@ -96,87 +98,111 @@ def add_query_table_to_html(html_table):
         {html_table}
         </body>
         </html>
-        """)
+        """
+    )
 
 
-def add_run_start_iso(table):
-    start_times = []
-    for timestamp in table["run_start"]:
-        start_time = Time(timestamp * u.ns, format="unix_tai")
-        start_times.append(start_time.utc.iso)
-    table.replace_column("run_start", start_times)
+def add_start_and_elapsed(table: Table, datedir: str, version: str) -> None:
+    """Add columns with the timestamp of first events and elapsed time of the runs.
 
+    This information is taken from the merged DL1 files. Two new columns are added
+    to the input table.
+
+    Parameters
+    ----------
+    table : astropy.table.Table
+        Astropy Table to which the two new columns are to be added.
+    datedir : str
+        Date directory in YYYYMMDD format.
+    version : str
+        Production version of the processing in the format 'vW.X.Y.Z'.
+    """
+    if "run_id" not in table.columns:
+        raise KeyError("Run ID not present in given table. Please check its content.")
 
-def add_elapsed(table, datedir, version):
+    start_times = []
     elapsed_times = []
+
     for run in table["run_id"]:
-        major_version = re.search(r'\D\d+\.\d+', version)[0]
+        major_version = get_major_version(version)
         file = BASE_DL1 / datedir / major_version / f"tailcut84/dl1_LST-1.Run{run:05d}.h5"
         df = pd.read_hdf(file, key=dl1_params_lstcam_key)
+
+        # Timestamp of the first event
+        first_time = Time(df["dragon_time"][0], format="unix", scale="utc")
+        start_times.append(first_time.utc.iso)
+
+        # Elapsed time of the run
         df_delta = add_delta_t_key(df)
         _, elapsed_t = get_effective_time(df_delta)
-
         elapsed_times.append(elapsed_t.to(u.min))
 
+    # Modify the input table by adding two new columns
     table.add_column(elapsed_times, name="Elapsed [min]")
+    table.add_column(start_times, name="Run start [UTC]")
 
 
 def copy_to_webserver(html_file, csv_file):
-    sp.run(["scp", str(html_file), "datacheck:/home/www/html/datacheck/lstosa/."])
-    sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."])
+    sp.run(["scp", str(html_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
+    sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
 
 
 @click.command()
-@click.argument(
-    'date',
-    type=click.DateTime(formats=["%Y-%m-%d"])
-)
+@click.argument("date", type=click.DateTime(formats=["%Y-%m-%d"]))
 @click.option("-v", "--version", type=str, default=get_lstchain_version())
 def main(date: datetime = None, version: str = get_lstchain_version()):
+    """Update source catalog with new run entries from a given date in format YYYY-MM-DD.
+
+    Notes
+    -----
+    It needs to be run as lstanalyzer user.
     """
-    Update source catalog with new run entries from a given date in
-    format YYYY-MM-DD. It needs to be run as lstanalyzer user.
-    """
-    csv_file = Path("/fefs/aswg/data/real/OSA/Catalog/LST_source_catalog.ecsv")
-    table = Table.read(csv_file)
+    catalog_path = CATALOG_DIR / "LST_source_catalog.ecsv"
+    catalog_table = Table.read(catalog_path)
 
-    # Open today's table and append its content to general table
+    # Open table for given date and append its content to the table with entire catalog
     datedir = date.strftime("%Y%m%d")
     today_catalog = Table.read(BASE_MONITORING / f"RunCatalog/RunCatalog_{datedir}.ecsv")
     today_runsummary = Table.read(BASE_MONITORING / f"RunSummary/RunSummary_{datedir}.ecsv")
+    # Keep only astronomical data runs
     today_runsummary = today_runsummary[today_runsummary["run_type"] == "DATA"]
-    todays_join = join(today_runsummary, today_catalog)
-    todays_join.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
-    todays_join.keep_columns(["run_id", "run_start", "source_name", "date_dir"])
+    todays_info = join(today_runsummary, today_catalog)
+    todays_info.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
+    todays_info.keep_columns(["run_id", "source_name", "date_dir"])
+
     # Add start of run in iso format and elapsed time for each run
     log.info("Getting run start and elapsed time")
-    add_run_start_iso(todays_join)
-    add_elapsed(todays_join, datedir, version)
-    # Change col names
-    todays_join.rename_column('run_id', 'Run ID')
-    todays_join.rename_column('run_start', 'Run start [UTC]')
-    todays_join.rename_column('source_name', 'Source name')
-    todays_join.rename_column('date_dir', 'Date directory')
-
-    # Add new rows
+    add_start_and_elapsed(todays_info, datedir, version)
+
+    # Change column names
+    todays_info.rename_column("run_id", "Run ID")
+    todays_info.rename_column("source_name", "Source name")
+    todays_info.rename_column("date_dir", "Date directory")
+
+    # Add new rows from given date to the whole catalog table
     log.info("Adding new rows to table")
-    new_table = vstack([table, todays_join])
-    table_unique = unique(new_table, keys="Run ID", keep='last')
+    new_table = vstack([catalog_table, todays_info])
+    table_unique = unique(new_table, keys="Run ID", keep="last")
 
-    # To pandas and HTML
+    # To pandas
     log.info("Converting to pandas and HTML")
     df = table_unique.to_pandas()
     df = df.sort_values(by="Run ID", ascending=False)
+
+    # To HTML
     html_table = df.to_html(index=False, justify="left")
     html_table = html_table.replace(
         '<table border="1" class="dataframe">',
-        '<table class="display compact" id="table139855676982704">')
+        '<table class="display compact" id="table139855676982704">',
+    )
     html_content = add_query_table_to_html(html_table)
-    html_file = Path("LST_source_catalog.html")
+
+    # Save the HTML and ECSV files and copy them to the LST-1 webserver
+    html_file = CATALOG_DIR / "LST_source_catalog.html"
     html_file.write_text(html_content)
-    table_unique.write(csv_file, delimiter=",", overwrite=True)
+    table_unique.write(catalog_path, delimiter=",", overwrite=True)
 
-    copy_to_webserver(html_file, csv_file)
+    copy_to_webserver(html_file, catalog_path)
 
 
 if __name__ == "__main__":