lstosa 0.10.18__py3-none-any.whl → 0.10.19__py3-none-any.whl

osa/utils/register.py

@@ -7,7 +7,11 @@ from pathlib import Path
 
 from osa.configs import options
 from osa.configs.config import cfg
-from osa.paths import destination_dir
+from osa.paths import (
+    destination_dir,
+    get_dl1_prod_id_and_config,
+    get_dl2_prod_id
+)
 from osa.utils.logging import myLogger
 from osa.veto import set_closed_sequence
 
@@ -70,11 +74,9 @@ def create_symlinks(input_file, output_file, prefix, suffix):
     """
 
     analysis_dir = Path(options.directory)
-    dl1ab_dir = analysis_dir / options.dl1_prod_id
 
     if prefix == "dl1_LST-1" and suffix == ".h5":
         dl1_filepath_analysis_dir = analysis_dir / input_file.name
-        dl1_filepath_dl1_dir = dl1ab_dir / input_file.name
         # Remove the original DL1 files pre DL1ab stage and keep only symlinks
         if dl1_filepath_analysis_dir.is_file() and not dl1_filepath_analysis_dir.is_symlink():
             dl1_filepath_analysis_dir.unlink()
@@ -83,8 +85,8 @@ def create_symlinks(input_file, output_file, prefix, suffix):
             dl1_filepath_analysis_dir.symlink_to(output_file.resolve())
 
         # Also set the symlink in the DL1ab subdirectory
-        if not dl1_filepath_dl1_dir.is_symlink():
-            dl1_filepath_dl1_dir.symlink_to(output_file.resolve())
+        if not input_file.is_symlink():
+            input_file.symlink_to(output_file.resolve())
 
     if prefix == "muons_LST-1" and suffix == ".fits":
         input_file.symlink_to(output_file.resolve())
@@ -93,7 +95,7 @@ def create_symlinks(input_file, output_file, prefix, suffix):
         input_file.symlink_to(output_file.resolve())
 
 
-def register_run_concept_files(run_string, concept):
+def register_run_concept_files(run_string, sequence_type, concept):
     """
     Prepare files to be moved to final destination directories
     from the running_analysis original directory.
@@ -107,17 +109,20 @@ def register_run_concept_files(run_string, concept):
     initial_dir = Path(options.directory)  # running_analysis
 
     # For MUON and INTERLEAVED data products, the initial directory is running_analysis
+    if sequence_type=="DATA":
+        dl1_prod_id = get_dl1_prod_id_and_config(int(run_string))[0]
+        dl2_prod_id = get_dl2_prod_id(int(run_string))
 
     if concept == "DL2":
-        initial_dir = initial_dir / options.dl2_prod_id
+        initial_dir = initial_dir / dl2_prod_id
 
     elif concept == "DL1AB":
-        initial_dir = initial_dir / options.dl1_prod_id
+        initial_dir = initial_dir / dl1_prod_id
 
     elif concept == "DATACHECK":
-        initial_dir = initial_dir / options.dl1_prod_id
+        initial_dir = initial_dir / dl1_prod_id
 
-    output_dir = destination_dir(concept, create_dir=False)
+    output_dir = destination_dir(concept, create_dir=False, dl1_prod_id=dl1_prod_id, dl2_prod_id=dl2_prod_id)
     data_level = cfg.get("PATTERN", f"{concept}TYPE")
     prefix = cfg.get("PATTERN", f"{concept}PREFIX")
     suffix = cfg.get("PATTERN", f"{concept}SUFFIX")
@@ -167,7 +172,7 @@ def register_non_existing_file(file_path, concept, seq_list):
 
             if run_str_found is not None:
                 log.debug(f"Registering file {run_str_found}")
-                register_run_concept_files(sequence.run_str, concept)
+                register_run_concept_files(sequence.run_str, sequence.type, concept)
                 if options.seqtoclose is None and not file_path.exists():
                     log.debug("File does not exists")
 
@@ -177,13 +182,13 @@ def register_non_existing_file(file_path, concept, seq_list):
 
             if calib_run_str_found is not None:
                 log.debug(f"Registering file {calib_run_str_found}")
-                register_run_concept_files(str(sequence.run), concept)
+                register_run_concept_files(str(sequence.run), sequence.type, concept)
                 if options.seqtoclose is None and not file_path.exists():
                     log.debug("File does not exists")
 
             if drs4_run_str_found is not None:
                 log.debug(f"Registering file {drs4_run_str_found}")
-                register_run_concept_files(str(sequence.previousrun), concept)
+                register_run_concept_files(str(sequence.previousrun), sequence.type, concept)
                 if options.seqtoclose is None and not file_path.exists():
                     log.debug("File does not exists")
 

osa/utils/tests/test_utils.py

@@ -75,3 +75,17 @@ def test_create_lock(base_test_dir):
     lock_path = base_test_dir / "test_lock.closed"
     is_closed = create_lock(lock_path)
     assert is_closed is False
+
+
+def test_get_RF_model(
+        run_catalog_dir,
+        run_catalog,
+        rf_models,
+        dl1b_config_files,
+        tailcuts_log_files,
+    ):
+    from osa.utils.utils import get_RF_model
+    from pathlib import Path
+
+    expected_model = Path("test_osa/test_files0/models/AllSky/20240918_v0.10.12_allsky_nsb_tuning_0.14/dec_2276")
+    assert get_RF_model(1807) == expected_model.resolve()

osa/utils/utils.py

@@ -4,10 +4,17 @@
 import inspect
 import logging
 import os
+import re
+import json
 import time
+import numpy as np
 from datetime import datetime, timedelta
 from pathlib import Path
 from socket import gethostname
+from gammapy.data import observatory_locations
+from astropy import units as u
+from astropy.table import Table
+from lstchain.onsite import find_filter_wheels
 
 import osa.paths
 from osa.configs import options
@@ -15,6 +22,7 @@ from osa.configs.config import cfg
 from osa.utils.iofile import write_to_file
 from osa.utils.logging import myLogger
 
+
 __all__ = [
     "get_lstchain_version",
     "date_to_dir",
@@ -26,12 +34,9 @@ __all__ = [
     "create_lock",
     "stringify",
     "gettag",
-    "get_dl1_prod_id",
-    "get_dl2_prod_id",
     "time_to_seconds",
     "DATACHECK_FILE_PATTERNS",
     "YESTERDAY",
-    "set_prod_ids",
     "is_night_time",
     "cron_lock",
     "example_seq",
@@ -85,44 +90,6 @@ def get_prod_id():
     return options.prod_id
 
 
-def get_dl1_prod_id():
-    """
-    Get the prod ID for the dl1 products provided
-    it is defined in the configuration file.
-
-    Returns
-    -------
-    dl1_prod_id: string
-    """
-    if not options.dl1_prod_id:
-        if cfg.get("LST1", "DL1_PROD_ID") is not None:
-            options.dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
-        else:
-            options.dl1_prod_id = get_lstchain_version()
-
-    log.debug(f"Getting prod ID for DL1 products: {options.dl1_prod_id}")
-
-    return options.dl1_prod_id
-
-
-def get_dl2_prod_id():
-    """
-
-    Returns
-    -------
-
-    """
-    if not options.dl2_prod_id:
-        if cfg.get("LST1", "DL2_PROD_ID") is not None:
-            options.dl2_prod_id = cfg.get("LST1", "DL2_PROD_ID")
-        else:
-            options.dl2_prod_id = get_lstchain_version()
-
-    log.debug(f"Getting prod ID for DL2 products: {options.dl2_prod_id}")
-
-    return options.dl2_prod_id
-
-
 def create_lock(lockfile) -> bool:
     """
     Create a lock file to prevent multiple instances of the same analysis.
@@ -244,21 +211,6 @@ def time_to_seconds(timestring):
     return int(hours) * 3600 + int(minutes) * 60 + int(seconds)
 
 
-def set_prod_ids():
-    """Set the product IDs."""
-    options.prod_id = get_prod_id()
-
-    if cfg.get("LST1", "DL1_PROD_ID") is not None:
-        options.dl1_prod_id = get_dl1_prod_id()
-    else:
-        options.dl1_prod_id = options.prod_id
-
-    if cfg.get("LST1", "DL2_PROD_ID") is not None:
-        options.dl2_prod_id = get_dl2_prod_id()
-    else:
-        options.dl2_prod_id = options.prod_id
-
-
 def is_night_time(hour):
     """Check if it is nighttime."""
     if 8 <= hour <= 18:
@@ -285,3 +237,168 @@ def wait_for_daytime(start=8, end=18):
     while time.localtime().tm_hour <= start or time.localtime().tm_hour >= end:
         log.info("Waiting for sunrise to not interfere with the data-taking. Sleeping.")
         time.sleep(3600)
+
+
+def get_calib_filters(run_id):
+    """Get the filters used for the calibration."""
+    if options.test:  # Run tests avoiding the access to the database
+        return 52
+
+    else:
+        mongodb = cfg.get("database", "caco_db")
+        try:
+            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
+            return find_filter_wheels(int(run_id), mongodb)
+        except IOError:
+            log.warning("No filter information found in database. Assuming positions 52.")
+            return 52
+
+
+def culmination_angle(dec: u.Quantity) -> u.Quantity:  
+    """
+    Calculate culmination angle for a given declination.
+
+    Parameters
+    ----------
+    dec: Quantity
+        declination coordinate in degrees
+
+    Returns
+    -------
+    Culmination angle in degrees
+    """
+    location = observatory_locations["cta_north"]
+    Lat = location.lat  # latitude of the LST1 site    
+    return abs(Lat - dec)
+
+
+def convert_dec_string(dec_str: str) -> u.Quantity:
+    """Return the declination angle in degrees corresponding to a 
+    given string of the form "dec_XXXX" or "dec_min_XXXX"."""
+    
+    # Check if dec_str has a valid format
+    pattern = r'^dec_(\d{3,4})$|^dec_min_(\d{3,4})$'
+    if re.match(pattern, dec_str):
+        
+        # Split the string into parts
+        parts = dec_str.split('_')
+
+        # Extract the sign, degrees, and minutes
+        sign = 1 if 'min' not in parts else -1
+        degrees = int(parts[-1])
+
+        # Calculate the numerical value
+        dec_value = sign * (degrees / 100)
+
+        return dec_value*u.deg
+
+
+def get_declinations_dict(list1: list, list2: list) -> dict:
+    """Return a dictionary created from two given lists."""
+    corresponding_dict = {}
+    for index, element in enumerate(list2):
+        corresponding_dict[element] = list1[index]
+    return corresponding_dict
+        
+
+def get_nsb_dict(rf_models_dir: Path, rf_models_prefix: str) -> dict:
+    """Return a dictionary with the NSB level of the RF models and the path to each model."""
+    rf_models = sorted(rf_models_dir.glob(f"{rf_models_prefix}*"))
+    pattern = r"nsb_tuning_([\d.]+)"
+    nsb_dict = {
+        float(re.search(pattern, str(rf_model)).group(1)): rf_model
+        for rf_model in rf_models if re.search(pattern, str(rf_model))
+    }
+    return nsb_dict
+
+
+def get_mc_nsb_dir(run_id: int, rf_models_dir: Path) -> Path:
+    """
+    Return the path of the RF models directory with the NSB level 
+    closest to that of the data for a given run.
+    """
+    additional_nsb = get_nsb_level(run_id)
+    rf_models_prefix = cfg.get("lstchain", "mc_prod")
+    nsb_dict = get_nsb_dict(rf_models_dir, rf_models_prefix)
+    closest_nsb_value = min(nsb_dict.keys(), key=lambda x: abs(float(x) - additional_nsb))
+
+    return nsb_dict[closest_nsb_value]
+
+
+def get_nsb_level(run_id):
+    """Choose the closest NSB among those that are processed with the same cleaning level."""
+    tailcuts_finder_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+    log_file = tailcuts_finder_dir / f"log_find_tailcuts_Run{run_id:05d}.log"
+    with open(log_file, "r") as file:
+        log_content = file.read()
+    match = re.search(r"Additional NSB rate \(over dark MC\): ([\d.]+)", log_content)
+    nsb = float(match.group(1))
+    
+    dl1b_config_filename = tailcuts_finder_dir / f"dl1ab_Run{run_id:05d}.json"
+    with open(dl1b_config_filename) as json_file:
+        dl1b_config = json.load(json_file)
+    picture_th = dl1b_config["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
+
+    nsb_levels = np.array([0.00, 0.07, 0.14, 0.22, 0.38, 0.50, 0.81, 1.25, 1.76, 2.34])
+    pth = np.array([8, 8, 8, 8, 10, 10, 12, 14, 16, 18])
+    candidate_nsbs = nsb_levels[pth==picture_th]
+
+    diff = abs(candidate_nsbs - nsb)
+    return candidate_nsbs[np.argsort(diff)][0]
+
+
+def get_RF_model(run_id: int) -> Path:
+    """Get the path of the RF models to be used in the DL2 production for a given run.
+    
+    The choice of the models is based on the adequate additional NSB level
+    and the proper declination line of the MC used for the training.
+    """
+    run_catalog_dir = Path(cfg.get(options.tel_id, "RUN_CATALOG"))
+    run_catalog_file = run_catalog_dir / f"RunCatalog_{date_to_dir(options.date)}.ecsv"
+    run_catalog = Table.read(run_catalog_file)
+    pointing_dec = run_catalog[run_catalog["run_id"]==run_id]["source_dec"]*u.deg  
+    # the "source_dec" given in the run catalogs is not actually the source declination, but the pointing declination
+    pointing_culmination = culmination_angle(pointing_dec)
+
+    rf_models_base_dir = Path(cfg.get(options.tel_id, "RF_MODELS"))
+    rf_models_dir = get_mc_nsb_dir(run_id, rf_models_base_dir)
+    dec_list = os.listdir(rf_models_dir)
+    dec_list = [i for i in dec_list if i.startswith("dec")]
+
+    # Convert each string in the list to numerical values
+    dec_values = [convert_dec_string(dec) for dec in dec_list]
+    dec_values = [dec for dec in dec_values if dec is not None]
+    
+    closest_declination = min(dec_values, key=lambda x: abs(x - pointing_dec))
+    closest_dec_culmination = culmination_angle(closest_declination)
+    
+    lst_location = observatory_locations["cta_north"]
+    lst_latitude = lst_location.lat  # latitude of the LST1 site    
+    closest_lines = sorted(sorted(dec_values, key=lambda x: abs(x - lst_latitude))[:2])
+
+    if pointing_dec < closest_lines[0] or pointing_dec > closest_lines[1]:
+        # If the pointing declination is between the two MC lines closest to the latitude of 
+        # the LST1 site, this check is not necessary.
+        log.debug(
+            f"The declination closest to {pointing_dec} is: {closest_declination}."
+            "Checking if the culmination angle is larger than the one of the pointing."
+        )
+        while closest_dec_culmination > pointing_culmination:
+            # If the culmination angle of the closest declination line is larger than for
+            # the pointing declination, remove it from the declination lines list and
+            # look for the second closest declination line.
+            declinations_dict = get_declinations_dict(dec_list, dec_values)
+            declination_str = declinations_dict[closest_declination]
+            dec_values.remove(closest_declination)
+            dec_list.remove(declination_str)
+            closest_declination = min(dec_values, key=lambda x: abs(x - pointing_dec))
+            closest_dec_culmination = culmination_angle(closest_declination)
+    
+    log.debug(f"The declination line to use for the DL2 production is: {closest_declination}")
+    
+    declinations_dict = get_declinations_dict(dec_list, dec_values)
+    declination_str = declinations_dict[closest_declination]
+
+    rf_model_path = rf_models_dir / declination_str
+
+    return rf_model_path.resolve()

osa/workflow/dl3.py

@@ -21,7 +21,7 @@ from osa.configs import options
 from osa.configs.config import cfg
 from osa.nightsummary.extract import build_sequences, get_source_list
 from osa.paths import destination_dir, DEFAULT_CFG, create_source_directories, analysis_path
-from osa.utils.cliopts import get_prod_id, get_dl2_prod_id
+from osa.utils.cliopts import get_prod_id
 from osa.utils.logging import myLogger
 from osa.utils.utils import stringify, YESTERDAY
 
@@ -250,7 +250,6 @@ def setup_global_options(date_obs, telescope):
     options.date = date_obs
     options.tel_id = telescope
     options.prod_id = get_prod_id()
-    options.dl2_prod_id = get_dl2_prod_id()
     options.directory = analysis_path(options.tel_id)
 
 

osa/workflow/stages.py

@@ -18,7 +18,7 @@ from osa.configs.config import cfg
 from osa.report import history
 from osa.utils.logging import myLogger
 from osa.utils.utils import stringify, date_to_dir
-from osa.paths import get_run_date
+from osa.paths import get_run_date, get_dl1_prod_id_and_config
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -87,7 +87,7 @@ class AnalysisStage:
             self._remove_drs4_baseline()
 
     def _remove_drs4_baseline(self):
-        drs4_pedestal_basedir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+        drs4_pedestal_basedir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
         date = date_to_dir(get_run_date(self.run))
         drs4_pedestal_dir = drs4_pedestal_basedir / date / lstchain.__version__
         file = drs4_pedestal_dir / "drs4_pedestal.Run{self.run}.0000.h5"
@@ -97,7 +97,7 @@ class AnalysisStage:
         drs4_pedestal_dir_pro.unlink(missing_ok=True)
 
     def _remove_calibration(self):
-        calib_basedir = Path(cfg.get("LST1", "CALIB_DIR"))
+        calib_basedir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
         date = date_to_dir(get_run_date(self.run))
         calib_dir = file = calib_basedir / date / lstchain.__version__
         file = calib_dir / f"calibration_filters_{options.filters}.Run{self.run}.0000.h5"
@@ -115,19 +115,24 @@ class AnalysisStage:
         interleaved_output_file.unlink(missing_ok=True)
 
     def _remove_dl1b_output(self, file_prefix):
-        dl1ab_subdirectory = options.directory / options.dl1_prod_id
+        dl1_prod_id = get_dl1_prod_id_and_config(int(self.run[:5]))[0]
+        dl1ab_subdirectory = options.directory / dl1_prod_id
         output_file = dl1ab_subdirectory / f"{file_prefix}{self.run}.h5"
         output_file.unlink(missing_ok=True)
 
     def _write_checkpoint(self):
         """Write the checkpoint in the history file."""
-        command_to_prod_id = {
-            cfg.get("lstchain", "r0_to_dl1"): options.prod_id,
-            cfg.get("lstchain", "dl1ab"): options.dl1_prod_id,
-            cfg.get("lstchain", "check_dl1"): options.dl1_prod_id,
-            cfg.get("lstchain", "dl1_to_dl2"): options.dl2_prod_id
-        }
-        prod_id = command_to_prod_id.get(self.command)
+        if self.command==cfg.get("lstchain", "r0_to_dl1"):
+            prod_id = options.prod_id
+        elif self.command==cfg.get("lstchain", "dl1ab"):
+            dl1_prod_id = get_dl1_prod_id_and_config(int(self.run[:5]))[0]
+            prod_id = dl1_prod_id
+        elif self.command==cfg.get("lstchain", "check_dl1"):
+            dl1_prod_id = get_dl1_prod_id_and_config(int(self.run[:5]))[0]
+            prod_id = dl1_prod_id
+        #elif self.command==cfg.get("lstchain", "dl1_to_dl2"):
+        #    dl2_prod_id = get_dl2_prod_id(int(self.run[:5]))
+        #    prod_id = dl2_prod_id
         history(
             run=self.run,
             prod_id=prod_id,

osa/workflow/tests/test_dl3.py

@@ -1,6 +1,7 @@
 import subprocess as sp
+import pytest
 
-
+@pytest.mark.skip(reason="Currently the DL3 production is not working")
 def test_dl3_stage():
     output = sp.run(
         ["dl3_stage", "-d", "2020-01-17", "-s", "LST1"], text=True, stdout=sp.PIPE, stderr=sp.PIPE

osa/workflow/tests/test_stages.py

@@ -4,7 +4,10 @@ import tenacity
 from osa.configs import options
 
 
-def test_analysis_stage(running_analysis_dir):
+def test_analysis_stage(
+        running_analysis_dir,
+        dl1b_config_files,
+    ):
     from osa.workflow.stages import AnalysisStage
 
     options.simulate = False
@@ -23,7 +26,7 @@ def test_analysis_stage(running_analysis_dir):
         f"--calibration-file={calibration_file}",
         f"--drive-file={drive_file}",
     ]
-    stage = AnalysisStage(run="01000.0001", command_args=cmd)
+    stage = AnalysisStage(run="01807.0001", command_args=cmd)
     assert stage.rc is None
     assert stage.show_command() == " ".join(cmd)
     with pytest.raises(tenacity.RetryError):
@@ -44,7 +47,7 @@ def test_analysis_stage(running_analysis_dir):
         "--input-file=dl1a_file.h5",
         "--output-file=dl1b_file.h5",
     ]
-    stage = AnalysisStage(run="01000.0001", command_args=cmd)
+    stage = AnalysisStage(run="01807.0001", command_args=cmd)
     assert stage.rc is None
     assert stage.show_command() == " ".join(cmd)
     with pytest.raises(tenacity.RetryError):
@@ -61,7 +64,7 @@ def test_analysis_stage(running_analysis_dir):
 
     # Third step
     cmd = ["lstchain_check_dl1", "--input-file=dl1_file.h5", "--batch"]
-    stage = AnalysisStage(run="01000.0001", command_args=cmd)
+    stage = AnalysisStage(run="01807.0001", command_args=cmd)
     assert stage.rc is None
     assert stage.show_command() == " ".join(cmd)
     with pytest.raises(tenacity.RetryError):