lstosa 0.10.6__tar.gz → 0.10.7__tar.gz

{lstosa-0.10.6 → lstosa-0.10.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.6
+Version: 0.10.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,8 +20,8 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain~=0.10.0
-Requires-Dist: matplotlib~=3.7
+Requires-Dist: lstchain>=0.10.5
+Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pyyaml

{lstosa-0.10.6 → lstosa-0.10.7}/environment.yml

@@ -8,7 +8,7 @@ dependencies:
   - pip
   - astropy~=5.0
   - ctapipe~=0.19.2
-  - matplotlib=3.7
+  - matplotlib
   - pyparsing
   - prov
   - pyyaml
@@ -24,8 +24,8 @@ dependencies:
   - protobuf=3.20
   - ctapipe_io_lst=0.22
   - ctaplot~=0.6.4
-  - pyirf=0.8
-  - lstchain~=0.10.0
+  - pyirf~=0.10
+  - lstchain>=0.10.5
   - tenacity
   # dev dependencies
   - pytest

{lstosa-0.10.6 → lstosa-0.10.7}/pyproject.toml

@@ -34,8 +34,8 @@ classifiers = [
 requires-python = ">=3.9"
 dependencies = [
     "astropy~=5.0",
-    "lstchain~=0.10.0",
-    "matplotlib~=3.7",
+    "lstchain>=0.10.5",
+    "matplotlib",
     "numpy",
     "pandas",
     "pyyaml",

{lstosa-0.10.6 → lstosa-0.10.7}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.6
+Version: 0.10.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,8 +20,8 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain~=0.10.0
-Requires-Dist: matplotlib~=3.7
+Requires-Dist: lstchain>=0.10.5
+Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pyyaml

{lstosa-0.10.6 → lstosa-0.10.7}/src/lstosa.egg-info/requires.txt

@@ -1,6 +1,6 @@
 astropy~=5.0
-lstchain~=0.10.0
-matplotlib~=3.7
+lstchain>=0.10.5
+matplotlib
 numpy
 pandas
 pyyaml

{lstosa-0.10.6 → lstosa-0.10.7}/src/osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.6'
-__version_tuple__ = version_tuple = (0, 10, 6)
+__version__ = version = '0.10.7'
+__version_tuple__ = version_tuple = (0, 10, 7)

{lstosa-0.10.6 → lstosa-0.10.7}/src/osa/scripts/gain_selection.py

@@ -20,7 +20,7 @@ from osa.job import get_sacct_output, FORMAT_SLURM
 
 log = myLogger(logging.getLogger(__name__))
 
-PATH = "PATH=/fefs/aswg/software/gain_selection/bin:$PATH"
+PATH = "PATH=/fefs/aswg/software/offline_dvr/bin:$PATH"
 
 
 def get_sbatch_script(
@@ -36,7 +36,7 @@ def get_sbatch_script(
     #SBATCH --job-name "gain_selection_{run_id:05d}"
     #SBATCH --export {PATH}
 
-    lst_select_gain {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
+    lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
     """
     )
 

{lstosa-0.10.6 → lstosa-0.10.7}/src/osa/scripts/provprocess.py

@@ -81,8 +81,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number):
         filter_cut = "all"
     cuts = {
         "calibration": ["drs4_pedestal", "calibrate_charge"],
-        "r0_to_dl1": ["r0_to_dl1", "dl1ab"],
-        "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"],
+        "r0_to_dl1": ["r0_to_dl1", "dl1ab", "dl1_datacheck"],
+        "dl1_to_dl2": ["dl1_to_dl2"],
     }
     cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"]
 
@@ -413,29 +413,31 @@ def produce_provenance(session_log_filename, base_filename):
             read_prov(filename=session_log_filename),
             str(paths_r0_dl1["out_path"]),
         )
-        dl1_lines = plines_r0 + plines_ab[1:]
+        plines_check = parse_lines_run(
+            "dl1_datacheck",
+            read_prov(filename=session_log_filename),
+            str(paths_r0_dl1["out_path"]),
+        )
+        dl1_lines = plines_r0 + plines_ab[1:] + plines_check[1:]
 
     # create r0_to_dl1 prov files only if filtering
     if options.filter == "r0_to_dl1":
-        produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1)
+        produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1)
 
     if options.filter == "dl1_to_dl2" or not options.filter:
-        paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
-        plines_check = parse_lines_run(
-            "dl1_datacheck",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        plines_dl2 = parse_lines_run(
-            "dl1_to_dl2",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        dl1_dl2_lines = plines_check + plines_dl2[1:]
+        if not options.no_dl2:
+            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
+            plines_dl2 = parse_lines_run(
+                "dl1_to_dl2",
+                read_prov(filename=session_log_filename),
+                str(paths_dl1_dl2["out_path"]),
+            )
+            dl1_dl2_lines = plines_dl2
 
     # create dl1_to_dl2 prov files only if filtering
     if options.filter == "dl1_to_dl2":
-        produce_provenance_files(plines_check + plines_dl2[1:], paths_dl1_dl2)
+        if not options.no_dl2:
+            produce_provenance_files(plines_dl2, paths_dl1_dl2)
 
     # create calibration_to_dl1 and calibration_to_dl2 prov files
     if not options.filter:

{lstosa-0.10.6 → lstosa-0.10.7}/src/osa/scripts/tests/test_osa_scripts.py

@@ -99,10 +99,10 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 16
-    assert len(dl1["activity"]) == 4
-    assert len(dl1["used"]) == 13
-    assert len(dl1["wasGeneratedBy"]) == 7
+    assert len(dl1["entity"]) == 19
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)

{lstosa-0.10.6 → lstosa-0.10.7}/src/osa/scripts/update_source_catalog.py

@@ -5,30 +5,30 @@ from pathlib import Path
 from textwrap import dedent
 
 import click
-import re
 import pandas as pd
 from astropy import units as u
-from astropy.table import Table, join, vstack, unique
+from astropy.table import Table, join, unique, vstack
 from astropy.time import Time
 from lstchain.io.io import dl1_params_lstcam_key
-from lstchain.reco.utils import get_effective_time, add_delta_t_key
+from lstchain.reco.utils import add_delta_t_key, get_effective_time
+
+from osa.paths import get_major_version
 from osa.utils.utils import get_lstchain_version
 
-pd.set_option('display.float_format', '{:.1f}'.format)
+pd.set_option("display.float_format", "{:.1f}".format)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s:%(levelname)s:%(message)s'
-)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(message)s")
 log = logging.getLogger(__name__)
 
 
 BASE_DL1 = Path("/fefs/aswg/data/real/DL1")
 BASE_MONITORING = Path("/fefs/aswg/data/real/monitoring")
+CATALOG_DIR = Path("/fefs/aswg/data/real/OSA/Catalog")
 
 
 def add_table_to_html(html_table):
-    return dedent(f"""\
+    return dedent(
+        f"""\
     <html>
       <head>
         <link href="osa.css" rel="stylesheet" type="text/css">
@@ -37,11 +37,13 @@ def add_table_to_html(html_table):
     {html_table}
     </body>
     </html>
-    """)
+    """
+    )
 
 
 def add_query_table_to_html(html_table):
-    return dedent(f"""\
+    return dedent(
+        f"""\
     <html>
      <head>
         <meta http-equiv="Content-type" content="text/html; charset=utf-8">
@@ -96,87 +98,111 @@ def add_query_table_to_html(html_table):
         {html_table}
         </body>
         </html>
-        """)
+        """
+    )
 
 
-def add_run_start_iso(table):
-    start_times = []
-    for timestamp in table["run_start"]:
-        start_time = Time(timestamp * u.ns, format="unix_tai")
-        start_times.append(start_time.utc.iso)
-    table.replace_column("run_start", start_times)
+def add_start_and_elapsed(table: Table, datedir: str, version: str) -> None:
+    """Add columns with the timestamp of first events and elapsed time of the runs.
 
+    This information is taken from the merged DL1 files. Two new columns are added
+    to the input table.
+
+    Parameters
+    ----------
+    table : astropy.table.Table
+        Astropy Table to which the two new columns are to be added.
+    datedir : str
+        Date directory in YYYYMMDD format.
+    version : str
+        Production version of the processing in the format 'vW.X.Y.Z'.
+    """
+    if "run_id" not in table.columns:
+        raise KeyError("Run ID not present in given table. Please check its content.")
 
-def add_elapsed(table, datedir, version):
+    start_times = []
     elapsed_times = []
+
     for run in table["run_id"]:
-        major_version = re.search(r'\D\d+\.\d+', version)[0]
+        major_version = get_major_version(version)
         file = BASE_DL1 / datedir / major_version / f"tailcut84/dl1_LST-1.Run{run:05d}.h5"
         df = pd.read_hdf(file, key=dl1_params_lstcam_key)
+
+        # Timestamp of the first event
+        first_time = Time(df["dragon_time"][0], format="unix", scale="utc")
+        start_times.append(first_time.utc.iso)
+
+        # Elapsed time of the run
         df_delta = add_delta_t_key(df)
         _, elapsed_t = get_effective_time(df_delta)
-
         elapsed_times.append(elapsed_t.to(u.min))
 
+    # Modify the input table by adding two new columns
     table.add_column(elapsed_times, name="Elapsed [min]")
+    table.add_column(start_times, name="Run start [UTC]")
 
 
 def copy_to_webserver(html_file, csv_file):
-    sp.run(["scp", str(html_file), "datacheck:/home/www/html/datacheck/lstosa/."])
-    sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."])
+    sp.run(["scp", str(html_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
+    sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
 
 
 @click.command()
-@click.argument(
-    'date',
-    type=click.DateTime(formats=["%Y-%m-%d"])
-)
+@click.argument("date", type=click.DateTime(formats=["%Y-%m-%d"]))
 @click.option("-v", "--version", type=str, default=get_lstchain_version())
 def main(date: datetime = None, version: str = get_lstchain_version()):
+    """Update source catalog with new run entries from a given date in format YYYY-MM-DD.
+
+    Notes
+    -----
+    It needs to be run as lstanalyzer user.
     """
-    Update source catalog with new run entries from a given date in
-    format YYYY-MM-DD. It needs to be run as lstanalyzer user.
-    """
-    csv_file = Path("/fefs/aswg/data/real/OSA/Catalog/LST_source_catalog.ecsv")
-    table = Table.read(csv_file)
+    catalog_path = CATALOG_DIR / "LST_source_catalog.ecsv"
+    catalog_table = Table.read(catalog_path)
 
-    # Open today's table and append its content to general table
+    # Open table for given date and append its content to the table with entire catalog
     datedir = date.strftime("%Y%m%d")
     today_catalog = Table.read(BASE_MONITORING / f"RunCatalog/RunCatalog_{datedir}.ecsv")
     today_runsummary = Table.read(BASE_MONITORING / f"RunSummary/RunSummary_{datedir}.ecsv")
+    # Keep only astronomical data runs
     today_runsummary = today_runsummary[today_runsummary["run_type"] == "DATA"]
-    todays_join = join(today_runsummary, today_catalog)
-    todays_join.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
-    todays_join.keep_columns(["run_id", "run_start", "source_name", "date_dir"])
+    todays_info = join(today_runsummary, today_catalog)
+    todays_info.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
+    todays_info.keep_columns(["run_id", "source_name", "date_dir"])
+
     # Add start of run in iso format and elapsed time for each run
     log.info("Getting run start and elapsed time")
-    add_run_start_iso(todays_join)
-    add_elapsed(todays_join, datedir, version)
-    # Change col names
-    todays_join.rename_column('run_id', 'Run ID')
-    todays_join.rename_column('run_start', 'Run start [UTC]')
-    todays_join.rename_column('source_name', 'Source name')
-    todays_join.rename_column('date_dir', 'Date directory')
-
-    # Add new rows
+    add_start_and_elapsed(todays_info, datedir, version)
+
+    # Change column names
+    todays_info.rename_column("run_id", "Run ID")
+    todays_info.rename_column("source_name", "Source name")
+    todays_info.rename_column("date_dir", "Date directory")
+
+    # Add new rows from given date to the whole catalog table
     log.info("Adding new rows to table")
-    new_table = vstack([table, todays_join])
-    table_unique = unique(new_table, keys="Run ID", keep='last')
+    new_table = vstack([catalog_table, todays_info])
+    table_unique = unique(new_table, keys="Run ID", keep="last")
 
-    # To pandas and HTML
+    # To pandas
     log.info("Converting to pandas and HTML")
     df = table_unique.to_pandas()
     df = df.sort_values(by="Run ID", ascending=False)
+
+    # To HTML
     html_table = df.to_html(index=False, justify="left")
     html_table = html_table.replace(
         '<table border="1" class="dataframe">',
-        '<table class="display compact" id="table139855676982704">')
+        '<table class="display compact" id="table139855676982704">',
+    )
     html_content = add_query_table_to_html(html_table)
-    html_file = Path("LST_source_catalog.html")
+
+    # Save the HTML and ECSV files and copy them to the LST-1 webserver
+    html_file = CATALOG_DIR / "LST_source_catalog.html"
     html_file.write_text(html_content)
-    table_unique.write(csv_file, delimiter=",", overwrite=True)
+    table_unique.write(catalog_path, delimiter=",", overwrite=True)
 
-    copy_to_webserver(html_file, csv_file)
+    copy_to_webserver(html_file, catalog_path)
 
 
 if __name__ == "__main__":