lstosa 0.10.5__tar.gz → 0.10.7__tar.gz

{lstosa-0.10.5 → lstosa-0.10.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.5
+Version: 0.10.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,8 +20,8 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain~=0.10.0
-Requires-Dist: matplotlib~=3.7
+Requires-Dist: lstchain>=0.10.5
+Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pyyaml

{lstosa-0.10.5 → lstosa-0.10.7}/environment.yml

@@ -8,7 +8,7 @@ dependencies:
   - pip
   - astropy~=5.0
   - ctapipe~=0.19.2
-  - matplotlib=3.7
+  - matplotlib
   - pyparsing
   - prov
   - pyyaml
@@ -24,8 +24,8 @@ dependencies:
   - protobuf=3.20
   - ctapipe_io_lst=0.22
   - ctaplot~=0.6.4
-  - pyirf=0.8
-  - lstchain~=0.10.0
+  - pyirf~=0.10
+  - lstchain>=0.10.5
   - tenacity
   # dev dependencies
   - pytest

{lstosa-0.10.5 → lstosa-0.10.7}/pyproject.toml

@@ -34,8 +34,8 @@ classifiers = [
 requires-python = ">=3.9"
 dependencies = [
     "astropy~=5.0",
-    "lstchain~=0.10.0",
-    "matplotlib~=3.7",
+    "lstchain>=0.10.5",
+    "matplotlib",
     "numpy",
     "pandas",
     "pyyaml",

{lstosa-0.10.5 → lstosa-0.10.7}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.5
+Version: 0.10.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -20,8 +20,8 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain~=0.10.0
-Requires-Dist: matplotlib~=3.7
+Requires-Dist: lstchain>=0.10.5
+Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: pyyaml

{lstosa-0.10.5 → lstosa-0.10.7}/src/lstosa.egg-info/requires.txt

@@ -1,6 +1,6 @@
 astropy~=5.0
-lstchain~=0.10.0
-matplotlib~=3.7
+lstchain>=0.10.5
+matplotlib
 numpy
 pandas
 pyyaml

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.5'
-__version_tuple__ = version_tuple = (0, 10, 5)
+__version__ = version = '0.10.7'
+__version_tuple__ = version_tuple = (0, 10, 7)

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/paths.py

@@ -9,9 +9,8 @@ from typing import List
 import lstchain
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
-                             find_time_calibration_file)
-from lstchain.scripts.onsite.onsite_create_calibration_file import \
-    search_filter
+                             find_time_calibration_file,
+                             find_filter_wheels)
 
 from osa.configs import options
 from osa.configs.config import DEFAULT_CFG, cfg
@@ -142,7 +141,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
         mongodb = cfg.get("database", "caco_db")
         try:
             # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = search_filter(int(run_id), mongodb)
+            options.filters = find_filter_wheels(int(run_id), mongodb)
         except IOError:
             log.warning("No filter information found in database. Assuming positions 52.")
             options.filters = 52

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/scripts/closer.py

@@ -193,7 +193,6 @@ def post_process_files(seq_list: list):
     output_files_set = set(Path(options.directory).rglob("*Run*"))
 
     DL1AB_RE = re.compile(rf"{options.dl1_prod_id}/dl1.*.(?:h5|hdf5|hdf)")
-    DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
     MUONS_RE = re.compile(r"muons.*.fits")
     DATACHECK_RE = re.compile(r"datacheck_dl1.*.(?:h5|hdf5|hdf)")
     INTERLEAVED_RE = re.compile(r"interleaved.*.(?:h5|hdf5|hdf)")
@@ -201,13 +200,16 @@ def post_process_files(seq_list: list):
     pattern_files = dict(
         [
             ("DL1AB", DL1AB_RE),
-            ("DL2", DL2_RE),
             ("MUON", MUONS_RE),
             ("DATACHECK", DATACHECK_RE),
             ("INTERLEAVED", INTERLEAVED_RE),
         ]
     )
 
+    if not options.no_dl2:
+        DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
+        pattern_files["DL2"] = DL2_RE
+
     for concept, pattern_re in pattern_files.items():
         log.info(f"Post processing {concept} files, {len(output_files_set)} files left")
 
@@ -368,6 +370,9 @@ def extract_provenance(seq_list):
                 nightdir,
                 options.prod_id,
             ]
+            if options.no_dl2:
+                cmd.append("--no-dl2")
+                
             if not options.simulate and not options.test and shutil.which("sbatch") is not None:
                 subprocess.run(cmd, check=True)
             else:

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/scripts/gain_selection.py

@@ -20,7 +20,7 @@ from osa.job import get_sacct_output, FORMAT_SLURM
 
 log = myLogger(logging.getLogger(__name__))
 
-PATH = "PATH=/fefs/aswg/software/gain_selection/bin:$PATH"
+PATH = "PATH=/fefs/aswg/software/offline_dvr/bin:$PATH"
 
 
 def get_sbatch_script(
@@ -36,7 +36,7 @@ def get_sbatch_script(
     #SBATCH --job-name "gain_selection_{run_id:05d}"
     #SBATCH --export {PATH}
 
-    lst_select_gain {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
+    lst_dvr {input_file} {output_dir} {ref_time} {ref_counter} {module} {ref_source}
     """
     )
 
@@ -206,8 +206,8 @@ def check_failed_jobs(date: str, output_basedir: Path = None):
 @click.option("--check", is_flag=True, default=False, help="Check for failed jobs.")
 @click.argument("dates-file", type=click.Path(exists=True, path_type=Path))
 @click.argument("output-basedir", type=click.Path(path_type=Path))
-@click.option("-s", "--start-time", type=int, default=10)
-@click.option("-e", "--end-time", type=int, default=18)
+@click.option("-s", "--start-time", type=int, default=10, help="Time to (re)start gain selection in HH format.")
+@click.option("-e", "--end-time", type=int, default=18, help="Time to stop gain selection in HH format.")
 def main(
     dates_file: Path = None, 
     output_basedir: Path = None, 

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/scripts/provprocess.py

@@ -81,8 +81,8 @@ def parse_lines_log(filter_cut, calib_runs, run_number):
         filter_cut = "all"
     cuts = {
         "calibration": ["drs4_pedestal", "calibrate_charge"],
-        "r0_to_dl1": ["r0_to_dl1", "dl1ab"],
-        "dl1_to_dl2": ["dl1_datacheck", "dl1_to_dl2"],
+        "r0_to_dl1": ["r0_to_dl1", "dl1ab", "dl1_datacheck"],
+        "dl1_to_dl2": ["dl1_to_dl2"],
     }
     cuts["all"] = cuts["calibration"] + cuts["r0_to_dl1"] + cuts["dl1_to_dl2"]
 
@@ -413,44 +413,48 @@ def produce_provenance(session_log_filename, base_filename):
             read_prov(filename=session_log_filename),
             str(paths_r0_dl1["out_path"]),
         )
-        dl1_lines = plines_r0 + plines_ab[1:]
+        plines_check = parse_lines_run(
+            "dl1_datacheck",
+            read_prov(filename=session_log_filename),
+            str(paths_r0_dl1["out_path"]),
+        )
+        dl1_lines = plines_r0 + plines_ab[1:] + plines_check[1:]
 
     # create r0_to_dl1 prov files only if filtering
     if options.filter == "r0_to_dl1":
-        produce_provenance_files(plines_r0 + plines_ab[1:], paths_r0_dl1)
+        produce_provenance_files(plines_r0 + plines_ab[1:] + plines_check[1:], paths_r0_dl1)
 
     if options.filter == "dl1_to_dl2" or not options.filter:
-        paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
-        plines_check = parse_lines_run(
-            "dl1_datacheck",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        plines_dl2 = parse_lines_run(
-            "dl1_to_dl2",
-            read_prov(filename=session_log_filename),
-            str(paths_dl1_dl2["out_path"]),
-        )
-        dl1_dl2_lines = plines_check + plines_dl2[1:]
+        if not options.no_dl2:
+            paths_dl1_dl2 = define_paths("dl1_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename)
+            plines_dl2 = parse_lines_run(
+                "dl1_to_dl2",
+                read_prov(filename=session_log_filename),
+                str(paths_dl1_dl2["out_path"]),
+            )
+            dl1_dl2_lines = plines_dl2
 
     # create dl1_to_dl2 prov files only if filtering
     if options.filter == "dl1_to_dl2":
-        produce_provenance_files(plines_check + plines_dl2[1:], paths_dl1_dl2)
+        if not options.no_dl2:
+            produce_provenance_files(plines_dl2, paths_dl1_dl2)
 
     # create calibration_to_dl1 and calibration_to_dl2 prov files
     if not options.filter:
         calibration_to_dl1 = define_paths(
             "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename
         )
-        calibration_to_dl2 = define_paths(
-            "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename
-        )
         calibration_to_dl1_lines = calibration_lines + dl1_lines[1:]
         lines_dl1 = copy.deepcopy(calibration_to_dl1_lines)
-        calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:]
-        lines_dl2 = copy.deepcopy(calibration_to_dl2_lines)
         produce_provenance_files(lines_dl1, calibration_to_dl1)
-        produce_provenance_files(lines_dl2, calibration_to_dl2)
+
+        if not options.no_dl2:
+            calibration_to_dl2 = define_paths(
+                "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename
+            )
+            calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:]
+            lines_dl2 = copy.deepcopy(calibration_to_dl2_lines)
+            produce_provenance_files(lines_dl2, calibration_to_dl2)
 
 
 def main():

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/scripts/tests/test_osa_scripts.py

@@ -99,10 +99,10 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 16
-    assert len(dl1["activity"]) == 4
-    assert len(dl1["used"]) == 13
-    assert len(dl1["wasGeneratedBy"]) == 7
+    assert len(dl1["entity"]) == 19
+    assert len(dl1["activity"]) == 5
+    assert len(dl1["used"]) == 15
+    assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/scripts/update_source_catalog.py

@@ -5,30 +5,30 @@ from pathlib import Path
 from textwrap import dedent
 
 import click
-import re
 import pandas as pd
 from astropy import units as u
-from astropy.table import Table, join, vstack, unique
+from astropy.table import Table, join, unique, vstack
 from astropy.time import Time
 from lstchain.io.io import dl1_params_lstcam_key
-from lstchain.reco.utils import get_effective_time, add_delta_t_key
+from lstchain.reco.utils import add_delta_t_key, get_effective_time
+
+from osa.paths import get_major_version
 from osa.utils.utils import get_lstchain_version
 
-pd.set_option('display.float_format', '{:.1f}'.format)
+pd.set_option("display.float_format", "{:.1f}".format)
 
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s:%(levelname)s:%(message)s'
-)
+logging.basicConfig(level=logging.INFO, format="%(asctime)s:%(levelname)s:%(message)s")
 log = logging.getLogger(__name__)
 
 
 BASE_DL1 = Path("/fefs/aswg/data/real/DL1")
 BASE_MONITORING = Path("/fefs/aswg/data/real/monitoring")
+CATALOG_DIR = Path("/fefs/aswg/data/real/OSA/Catalog")
 
 
 def add_table_to_html(html_table):
-    return dedent(f"""\
+    return dedent(
+        f"""\
     <html>
       <head>
         <link href="osa.css" rel="stylesheet" type="text/css">
@@ -37,11 +37,13 @@ def add_table_to_html(html_table):
     {html_table}
     </body>
     </html>
-    """)
+    """
+    )
 
 
 def add_query_table_to_html(html_table):
-    return dedent(f"""\
+    return dedent(
+        f"""\
     <html>
      <head>
         <meta http-equiv="Content-type" content="text/html; charset=utf-8">
@@ -96,87 +98,111 @@ def add_query_table_to_html(html_table):
         {html_table}
         </body>
         </html>
-        """)
+        """
+    )
 
 
-def add_run_start_iso(table):
-    start_times = []
-    for timestamp in table["run_start"]:
-        start_time = Time(timestamp * u.ns, format="unix_tai")
-        start_times.append(start_time.utc.iso)
-    table.replace_column("run_start", start_times)
+def add_start_and_elapsed(table: Table, datedir: str, version: str) -> None:
+    """Add columns with the timestamp of first events and elapsed time of the runs.
 
+    This information is taken from the merged DL1 files. Two new columns are added
+    to the input table.
+
+    Parameters
+    ----------
+    table : astropy.table.Table
+        Astropy Table to which the two new columns are to be added.
+    datedir : str
+        Date directory in YYYYMMDD format.
+    version : str
+        Production version of the processing in the format 'vW.X.Y.Z'.
+    """
+    if "run_id" not in table.columns:
+        raise KeyError("Run ID not present in given table. Please check its content.")
 
-def add_elapsed(table, datedir, version):
+    start_times = []
     elapsed_times = []
+
     for run in table["run_id"]:
-        major_version = re.search(r'\D\d+\.\d+', version)[0]
+        major_version = get_major_version(version)
         file = BASE_DL1 / datedir / major_version / f"tailcut84/dl1_LST-1.Run{run:05d}.h5"
         df = pd.read_hdf(file, key=dl1_params_lstcam_key)
+
+        # Timestamp of the first event
+        first_time = Time(df["dragon_time"][0], format="unix", scale="utc")
+        start_times.append(first_time.utc.iso)
+
+        # Elapsed time of the run
         df_delta = add_delta_t_key(df)
         _, elapsed_t = get_effective_time(df_delta)
-
         elapsed_times.append(elapsed_t.to(u.min))
 
+    # Modify the input table by adding two new columns
     table.add_column(elapsed_times, name="Elapsed [min]")
+    table.add_column(start_times, name="Run start [UTC]")
 
 
 def copy_to_webserver(html_file, csv_file):
-    sp.run(["scp", str(html_file), "datacheck:/home/www/html/datacheck/lstosa/."])
-    sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."])
+    sp.run(["scp", str(html_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
+    sp.run(["scp", str(csv_file), "datacheck:/home/www/html/datacheck/lstosa/."], check=True)
 
 
 @click.command()
-@click.argument(
-    'date',
-    type=click.DateTime(formats=["%Y-%m-%d"])
-)
+@click.argument("date", type=click.DateTime(formats=["%Y-%m-%d"]))
 @click.option("-v", "--version", type=str, default=get_lstchain_version())
 def main(date: datetime = None, version: str = get_lstchain_version()):
+    """Update source catalog with new run entries from a given date in format YYYY-MM-DD.
+
+    Notes
+    -----
+    It needs to be run as lstanalyzer user.
     """
-    Update source catalog with new run entries from a given date in
-    format YYYY-MM-DD. It needs to be run as lstanalyzer user.
-    """
-    csv_file = Path("/fefs/aswg/data/real/OSA/Catalog/LST_source_catalog.ecsv")
-    table = Table.read(csv_file)
+    catalog_path = CATALOG_DIR / "LST_source_catalog.ecsv"
+    catalog_table = Table.read(catalog_path)
 
-    # Open today's table and append its content to general table
+    # Open table for given date and append its content to the table with entire catalog
     datedir = date.strftime("%Y%m%d")
     today_catalog = Table.read(BASE_MONITORING / f"RunCatalog/RunCatalog_{datedir}.ecsv")
     today_runsummary = Table.read(BASE_MONITORING / f"RunSummary/RunSummary_{datedir}.ecsv")
+    # Keep only astronomical data runs
     today_runsummary = today_runsummary[today_runsummary["run_type"] == "DATA"]
-    todays_join = join(today_runsummary, today_catalog)
-    todays_join.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
-    todays_join.keep_columns(["run_id", "run_start", "source_name", "date_dir"])
+    todays_info = join(today_runsummary, today_catalog)
+    todays_info.add_column(date.strftime("%Y-%m-%d"), name="date_dir")
+    todays_info.keep_columns(["run_id", "source_name", "date_dir"])
+
     # Add start of run in iso format and elapsed time for each run
     log.info("Getting run start and elapsed time")
-    add_run_start_iso(todays_join)
-    add_elapsed(todays_join, datedir, version)
-    # Change col names
-    todays_join.rename_column('run_id', 'Run ID')
-    todays_join.rename_column('run_start', 'Run start [UTC]')
-    todays_join.rename_column('source_name', 'Source name')
-    todays_join.rename_column('date_dir', 'Date directory')
-
-    # Add new rows
+    add_start_and_elapsed(todays_info, datedir, version)
+
+    # Change column names
+    todays_info.rename_column("run_id", "Run ID")
+    todays_info.rename_column("source_name", "Source name")
+    todays_info.rename_column("date_dir", "Date directory")
+
+    # Add new rows from given date to the whole catalog table
     log.info("Adding new rows to table")
-    new_table = vstack([table, todays_join])
-    table_unique = unique(new_table, keys="Run ID", keep='last')
+    new_table = vstack([catalog_table, todays_info])
+    table_unique = unique(new_table, keys="Run ID", keep="last")
 
-    # To pandas and HTML
+    # To pandas
     log.info("Converting to pandas and HTML")
     df = table_unique.to_pandas()
     df = df.sort_values(by="Run ID", ascending=False)
+
+    # To HTML
     html_table = df.to_html(index=False, justify="left")
     html_table = html_table.replace(
         '<table border="1" class="dataframe">',
-        '<table class="display compact" id="table139855676982704">')
+        '<table class="display compact" id="table139855676982704">',
+    )
     html_content = add_query_table_to_html(html_table)
-    html_file = Path("LST_source_catalog.html")
+
+    # Save the HTML and ECSV files and copy them to the LST-1 webserver
+    html_file = CATALOG_DIR / "LST_source_catalog.html"
     html_file.write_text(html_content)
-    table_unique.write(csv_file, delimiter=",", overwrite=True)
+    table_unique.write(catalog_path, delimiter=",", overwrite=True)
 
-    copy_to_webserver(html_file, csv_file)
+    copy_to_webserver(html_file, catalog_path)
 
 
 if __name__ == "__main__":

{lstosa-0.10.5 → lstosa-0.10.7}/src/osa/utils/cliopts.py

@@ -326,6 +326,12 @@ def provprocess_argparser():
         dest="quit",
         help="use this flag to reset session and remove log file",
     )
+    parser.add_argument(
+        "--no-dl2",
+        action="store_true",
+        default=False,
+        help="Do not produce DL2 files (default False)",
+    )
     parser.add_argument(
         "drs4_pedestal_run_id", help="Number of the drs4_pedestal used in the calibration"
     )
@@ -354,6 +360,7 @@ def provprocessparsing():
     options.configfile = opts.config.resolve()
     options.filter = opts.filter
     options.quit = opts.quit
+    options.no_dl2 = opts.no_dl2
     set_prod_ids()