lstosa 0.10.18__py3-none-any.whl → 0.11.0__py3-none-any.whl

osa/nightsummary/extract.py

@@ -23,9 +23,9 @@ from osa.configs.datamodel import Sequence
 from osa.job import sequence_filenames
 from osa.nightsummary import database
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import sequence_calibration_files, get_run_date
+from osa.paths import sequence_calibration_files, get_run_date, get_dl1_prod_id_and_config, get_dl2_prod_id
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_iso, date_to_dir
+from osa.utils.utils import date_to_iso, date_to_dir, get_RF_model
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -263,6 +263,15 @@ def extract_sequences(date: datetime, run_obj_list: List[RunObj]) -> List[Sequen
                 f"Data sequence {sequence.seq} from run {run.run} whose parent is "
                 f"{sequence.parent} (DRS4 {required_drs4_run} & Ped-Cal {required_pedcal_run})"
             )
+            if not options.no_dl1ab and sequence.type=="DATA":
+                dl1_prod_id, dl1b_config = get_dl1_prod_id_and_config(sequence.run)
+                sequence.dl1_prod_id = dl1_prod_id
+                sequence.dl1b_config = dl1b_config
+
+            if not options.no_dl2 and not options.no_dl1ab and sequence.type=="DATA":
+                sequence.dl2_prod_id = get_dl2_prod_id(sequence.run)
+                sequence.rf_model = get_RF_model(sequence.run)
+
             sequence_list.append(sequence)
 
     # Add the calibration file names
@@ -300,7 +309,7 @@ def get_source_list(date: datetime) -> dict:
 
     # Create a dictionary of sources and their corresponding sequences
     source_dict = {
-        sequence.run: sequence.source_name
+        int(sequence.run): sequence.source_name
         for sequence in sequence_list
         if sequence.source_name is not None
     }

osa/nightsummary/tests/test_extract.py

@@ -1,5 +1,7 @@
 from datetime import datetime
+from osa.configs import options
 
+options.test = True
 obs_date = datetime.fromisoformat("2020-01-17")
 
 
@@ -11,6 +13,9 @@ def test_source_list(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Test that the list of name of sources is correct."""
     from osa.nightsummary.extract import get_source_list

osa/paths.py

@@ -2,17 +2,16 @@
 
 import logging
 import re
+import sys
 from datetime import datetime
 from pathlib import Path
 from typing import List
 import subprocess
 import time
-
-import lstchain
+import json
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
-                             find_time_calibration_file,
-                             find_filter_wheels)
+                             find_time_calibration_file)
 
 from osa.configs import options
 from osa.configs.config import DEFAULT_CFG, cfg
@@ -45,8 +44,8 @@ __all__ = [
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
-CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
-DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:
@@ -103,10 +102,12 @@ def get_drs4_pedestal_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
+    lstcam_env = Path(cfg.get("LST1", "CALIB_ENV"))
+    lstcam_calib_version = utils.get_lstcam_calib_version(lstcam_env)
     return (
         DRS4_PEDESTAL_BASEDIR
         / date
-        / f"v{lstchain.__version__}/drs4_pedestal.Run{run_id:05d}.0000.h5"
+        / f"v{lstcam_calib_version}/drs4_pedestal.Run{run_id:05d}.0000.h5"
     ).resolve()
 
 
@@ -136,26 +137,26 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
+    options.filters = utils.get_calib_filters(run_id)
 
-    if options.test:  # Run tests avoiding the access to the database
-        options.filters = 52
-
-    else:
-        mongodb = cfg.get("database", "caco_db")
-        try:
-            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = find_filter_wheels(int(run_id), mongodb)
-        except IOError:
-            log.warning("No filter information found in database. Assuming positions 52.")
-            options.filters = 52
-
+    lstcam_env = Path(cfg.get("LST1", "CALIB_ENV"))
+    lstcam_calib_version = utils.get_lstcam_calib_version(lstcam_env)
     return (
         CALIB_BASEDIR
         / date
-        / f"v{lstchain.__version__}/calibration_filters_{options.filters}.Run{run_id:05d}.0000.h5"
+        / f"v{lstcam_calib_version}/calibration_filters_{options.filters}.Run{run_id:05d}.0000.h5"
     ).resolve()
 
 
+def get_catB_calibration_filename(run_id: int) -> Path:
+    """Return the Category-B calibration filename of a given run."""
+    date = utils.date_to_dir(options.date)
+    calib_prod_id = utils.get_lstchain_version()
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / date / calib_prod_id
+    filters = utils.get_calib_filters(run_id)
+    return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+
+
 def pedestal_ids_file_exists(run_id: int) -> bool:
     """Look for the files with pedestal interleaved event identification."""
     pedestal_ids_dir = Path(cfg.get("LST1", "PEDESTAL_FINDER_DIR"))
@@ -256,15 +257,18 @@ def sequence_calibration_files(sequence_list: List[Sequence]) -> None:
 
 def get_datacheck_files(pattern: str, directory: Path) -> list:
     """Return a list of files matching the pattern."""
-    return sorted(directory.glob(pattern))
+    if pattern=="datacheck_dl1*.pdf":
+        return sorted(directory.glob("tailcut*/datacheck/"+pattern))
+    else:
+        return sorted(directory.glob(pattern))
 
 
 def datacheck_directory(data_type: str, date: str) -> Path:
     """Returns the path to the datacheck directory given the data type."""
     if data_type in {"PEDESTAL", "CALIB"}:
-        directory = Path(cfg.get("LST1", f"{data_type}_DIR")) / date / "pro/log"
+        directory = Path(cfg.get("LST1", f"CAT_A_{data_type}_DIR")) / date / "pro/log"
     elif data_type == "DL1AB":
-        directory = destination_dir("DATACHECK", create_dir=False)
+        directory = Path(cfg.get("LST1", f"{data_type}_DIR")) / date / options.prod_id
     elif data_type == "LONGTERM":
         directory = Path(cfg.get("LST1", f"{data_type}_DIR")) / options.prod_id / date
     else:
@@ -272,7 +276,7 @@ def datacheck_directory(data_type: str, date: str) -> Path:
     return directory
 
 
-def destination_dir(concept: str, create_dir: bool = True) -> Path:
+def destination_dir(concept: str, create_dir: bool = True, dl1_prod_id: str = None, dl2_prod_id: str = None) -> Path:
     """
     Create final destination directory for each data level.
     See Also osa.utils.register_run_concept_files
@@ -303,7 +307,7 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
             Path(cfg.get(options.tel_id, "DL1_DIR"))
             / nightdir
             / options.prod_id
-            / options.dl1_prod_id
+            / dl1_prod_id
             / "datacheck"
         )
     elif concept == "DL1AB":
@@ -311,13 +315,14 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
             Path(cfg.get(options.tel_id, "DL1_DIR"))
             / nightdir
             / options.prod_id
-            / options.dl1_prod_id
+            / dl1_prod_id
         )
     elif concept in {"DL2", "DL3"}:
         directory = (
             (Path(cfg.get(options.tel_id, f"{concept}_DIR")) / nightdir)
             / options.prod_id
-        ) / options.dl2_prod_id
+            / dl2_prod_id
+        ) 
     elif concept in {"PEDESTAL", "CALIB", "TIMECALIB"}:
         directory = (
             Path(cfg.get(options.tel_id, f"{concept}_DIR"))
@@ -356,7 +361,8 @@ def get_latest_version_file(longterm_files: List[str]) -> Path:
     return max(
         longterm_files,
         key=lambda path: int(path.parents[1].name.split(".")[1])
-        if path.parents[1].name.startswith("v")
+        if path.parents[1].name.startswith("v") and 
+            re.match(r'^\d+\.\d+(\.\d+)?$', path.parents[1].name[1:])
         else "",
     )
 
@@ -397,6 +403,7 @@ def create_longterm_symlink(cherenkov_job_id: str = None):
     else:
         log.warning(f"Job {cherenkov_job_id} (lstchain_cherenkov_transparency) did not finish successfully.")
 
+
 def dl1_datacheck_longterm_file_exits() -> bool:
     """Return true if the longterm DL1 datacheck file was already produced."""
     nightdir = utils.date_to_dir(options.date)
@@ -404,3 +411,79 @@ def dl1_datacheck_longterm_file_exits() -> bool:
     longterm_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5"
     return longterm_file.exists()
 
+
+def catB_closed_file_exists(run_id: int) -> bool:
+    catB_closed_file = Path(options.directory) / f"catB_{run_id:05d}.closed"
+    return catB_closed_file.exists()
+
+
+def catB_calibration_file_exists(run_id: int) -> bool:
+    catB_calib_base_dir = Path(cfg.get("LST1","CAT_B_CALIB_BASE"))
+    prod_id = utils.get_lstchain_version()
+    night_dir = utils.date_to_dir(options.date)
+    filters = utils.get_calib_filters(run_id)
+    catB_calib_dir = catB_calib_base_dir / "calibration" / night_dir / prod_id 
+    catB_calib_file = catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+    return catB_calib_file.exists()
+
+
+def get_dl1_prod_id(config_filename):
+    with open(config_filename) as json_file:
+        data = json.load(json_file)
+        
+    picture_thresh = data["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
+    boundary_thresh = data["tailcuts_clean_with_pedestal_threshold"]["boundary_thresh"]
+
+    if boundary_thresh == 4:
+        return f"tailcut{picture_thresh}{boundary_thresh}"
+    else:
+        return f"tailcut{picture_thresh}{boundary_thresh:02d}"
+
+
+def get_dl2_nsb_prod_id(rf_model: Path) -> str:
+    match = re.search(r'nsb_tuning_\d+\.\d+', str(rf_model))
+    if not match:
+        log.warning(f"No 'nsb_tuning_X.XX' pattern found in the path:\n{rf_model}")
+        sys.exit(1)
+    else:
+        return match.group(0)
+    
+
+def get_dl1_prod_id_and_config(run_id: int) -> str:
+    if not cfg.getboolean("lstchain", "apply_standard_dl1b_config"):
+        tailcuts_finder_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+        dl1b_config_file = tailcuts_finder_dir / f"dl1ab_Run{run_id:05d}.json"
+        if not dl1b_config_file.exists()  and not options.simulate:
+            log.error(
+                f"The dl1b config file was not created yet for run {run_id:05d}. "
+                "Please try again later."
+            )
+            sys.exit(1) 
+        else: 
+            dl1_prod_id = get_dl1_prod_id(dl1b_config_file)
+            return dl1_prod_id, dl1b_config_file.resolve()
+    else:
+        dl1b_config_file = Path(cfg.get("lstchain", "dl1b_config"))
+        dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
+        return dl1_prod_id, dl1b_config_file.resolve()
+    
+
+def get_dl2_prod_id(run_id: int) -> str:
+    dl1_prod_id = get_dl1_prod_id_and_config(run_id)[0]
+    rf_model = utils.get_RF_model(run_id)
+    nsb_prod_id = get_dl2_nsb_prod_id(rf_model)
+    return f"{dl1_prod_id}/{nsb_prod_id}"
+
+
+def all_dl1ab_config_files_exist(date: str) -> bool:
+    nightdir = date.replace("-","")
+    run_summary_dir =  Path(cfg.get(options.tel_id, "RUN_SUMMARY_DIR"))
+    run_summary_file = run_summary_dir / f"RunSummary_{nightdir}.ecsv"
+    summary_table = Table.read(run_summary_file)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    for run_id in data_runs["run_id"]:
+        tailcuts_finder_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+        dl1b_config_file = tailcuts_finder_dir / f"dl1ab_Run{run_id:05d}.json"
+        if not dl1b_config_file.exists():
+            return False
+    return True

osa/provenance/capture.py

@@ -53,7 +53,7 @@ LOG_FILENAME = provconfig["handlers"]["provHandler"]["filename"]
 PROV_PREFIX = provconfig["PREFIX"]
 SUPPORTED_HASH_METHOD = ["md5"]
 SUPPORTED_HASH_BUFFER = ["content", "path"]
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 # global variables
 traced_entities = {}

osa/provenance/config/definition.yaml

@@ -200,6 +200,13 @@ activities:
             #   filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/
             #   size: 128
 
+    catB_calibration:
+        description: 
+            "Create Cat-B calibration file for an observation run"
+        parameters:
+        usage:
+        generation:
+
     dl1ab:
         description:
             "Create DL1AB files for an observation run"

osa/provenance/utils.py

@@ -10,7 +10,7 @@ from osa.utils.utils import date_to_dir, get_lstchain_version
 
 __all__ = ["parse_variables", "get_log_config"]
 
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 
 def parse_variables(class_instance):
@@ -40,20 +40,18 @@ def parse_variables(class_instance):
     configfile_dl1b = cfg.get("lstchain", "dl1b_config")
     configfile_dl2 = cfg.get("lstchain", "dl2_config")
     raw_dir = Path(cfg.get("LST1", "R0_DIR"))
-    rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
+    rf_models_directory = Path(cfg.get("LST1", "RF_MODELS"))
     dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
     dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
-    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
     class_instance.SoftwareVersion = get_lstchain_version()
     class_instance.ProcessingConfigFile = str(options.configfile)
     class_instance.ObservationDate = flat_date
     if class_instance.__name__ in REDUCTION_TASKS:
         muon_dir = dl1_dir / flat_date / options.prod_id / "muons"
-        outdir_dl1 = dl1_dir / flat_date / options.prod_id / options.dl1_prod_id
-        outdir_dl2 = dl2_dir / flat_date / options.prod_id / options.dl2_prod_id
-
+        
     if class_instance.__name__ in ["drs4_pedestal", "calibrate_charge"]:
         # drs4_pedestal_run_id  [0] 1804
         # pedcal_run_id         [1] 1805
@@ -111,6 +109,7 @@ def parse_variables(class_instance):
         run = run_subrun.split(".")[0]
         class_instance.ObservationRun = run
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id
         calibration_file = Path(class_instance.args[0]).resolve()
         pedestal_file = Path(class_instance.args[1]).resolve()
         timecalibration_file = Path(class_instance.args[2]).resolve()
@@ -133,10 +132,16 @@ def parse_variables(class_instance):
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
             class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
+  
+    if class_instance.__name__ == "catB_calibration":
+        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000
+        # dl1b_config   [1]
+        # dl1_prod_id   [2]
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / class_instance.args[2]
         class_instance.Analysisconfigfile_dl1 = str(Path(configfile_dl1b))
         class_instance.ObservationRun = class_instance.args[0].split(".")[0]
         class_instance.StoreImage = cfg.getboolean("lstchain", "store_image_dl1ab")
@@ -146,9 +151,12 @@ def parse_variables(class_instance):
 
     if class_instance.__name__ == "dl1_datacheck":
         # run_str       [0] 02006.0000
+        # dl1b_prod_id  [1]
+
         run_subrun = class_instance.args[0]
         run = run_subrun.split(".")[0]
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / class_instance.args[1]
         class_instance.ObservationRun = run
         class_instance.DL1SubrunDataset = str(
             (outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
@@ -168,9 +176,16 @@ def parse_variables(class_instance):
 
     if class_instance.__name__ == "dl1_to_dl2":
         # run_str       [0] 02006.0000
+        # rf_model_path [1]
+        # dl1_prod_id   [2]
+        # dl2_prod_id   [3]
+
         run_subrun = class_instance.args[0]
         run = run_subrun.split(".")[0]
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / class_instance.args[2]
+        outdir_dl2 = dl2_dir / flat_date / options.prod_id / class_instance.args[3]
+
         class_instance.Analysisconfigfile_dl2 = configfile_dl2
         class_instance.ObservationRun = run
         class_instance.RFModelEnergyFile = str((rf_models_directory / "reg_energy.sav").resolve())

osa/scripts/autocloser.py

@@ -266,21 +266,11 @@ class Sequence:
 
     def is_100(self, no_dl2: bool):
         """Check that all analysis products are 100% complete."""
-        if (
-            no_dl2
-            and self.dict_sequence["Tel"] != "ST"
-            and self.dict_sequence["DL1%"] == "100"
-            and self.dict_sequence["DL1AB%"] == "100"
-            and self.dict_sequence["MUONS%"] == "100"
-        ):
-            return True
-
         if (
             self.dict_sequence["Tel"] != "ST"
             and self.dict_sequence["DL1%"] == "100"
             and self.dict_sequence["DL1AB%"] == "100"
             and self.dict_sequence["MUONS%"] == "100"
-            and self.dict_sequence["DL2%"] == "100"
         ):
             return True
 

osa/scripts/calibration_pipeline.py

@@ -19,6 +19,7 @@ from osa.paths import drs4_pedestal_exists, calibration_file_exists
 from osa.provenance.capture import trace
 from osa.utils.cliopts import calibration_pipeline_cliparsing
 from osa.utils.logging import myLogger
+from osa.utils import utils
 from osa.workflow.stages import DRS4PedestalStage, ChargeCalibrationStage
 
 __all__ = [
@@ -37,30 +38,36 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
     Check if both daily calibration (DRS4 baseline and
     charge calibration) files are already produced.
     """
-    return drs4_pedestal_exists(drs4_pedestal_run_id, options.prod_id) \
-        and calibration_file_exists(pedcal_run_id, options.prod_id)
+    lstcam_env = Path(cfg.get("LST1", "CALIB_ENV"))
+    lstcam_calib_version = utils.get_lstcam_calib_version(lstcam_env)
+    return drs4_pedestal_exists(drs4_pedestal_run_id, f"v{lstcam_calib_version}") \
+        and calibration_file_exists(pedcal_run_id, f"v{lstcam_calib_version}")
 
 
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     """Build the create_drs4_pedestal command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     command = cfg.get("lstchain", "drs4_baseline")
     return [
         command,
         "-r", str(drs4_pedestal_run_id),
         "-b", base_dir,
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
 
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
     command = cfg.get("lstchain", "charge_calibration")
     cmd = [
         command,
         "-p", str(drs4_pedestal_run_id),
         "-r", str(pedcal_run_id),
         "-b", base_dir,
+        f"--r0-dir={r0_dir}",
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):