lstosa 0.10.17__py3-none-any.whl → 0.10.19__py3-none-any.whl

osa/paths.py

@@ -2,17 +2,17 @@
 
 import logging
 import re
+import sys
 from datetime import datetime
 from pathlib import Path
 from typing import List
 import subprocess
 import time
-
+import json
 import lstchain
 from astropy.table import Table
 from lstchain.onsite import (find_systematics_correction_file,
-                             find_time_calibration_file,
-                             find_filter_wheels)
+                             find_time_calibration_file)
 
 from osa.configs import options
 from osa.configs.config import DEFAULT_CFG, cfg
@@ -45,8 +45,8 @@ __all__ = [
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
-CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
-DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:
@@ -136,18 +136,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
         return files[-1]  # Get the latest production among the major lstchain version
 
     date = utils.date_to_dir(get_run_date(run_id))
-
-    if options.test:  # Run tests avoiding the access to the database
-        options.filters = 52
-
-    else:
-        mongodb = cfg.get("database", "caco_db")
-        try:
-            # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = find_filter_wheels(int(run_id), mongodb)
-        except IOError:
-            log.warning("No filter information found in database. Assuming positions 52.")
-            options.filters = 52
+    options.filters = utils.get_calib_filters(run_id)
 
     return (
         CALIB_BASEDIR
@@ -156,6 +145,15 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path:
     ).resolve()
 
 
+def get_catB_calibration_filename(run_id: int) -> Path:
+    """Return the Category-B calibration filename of a given run."""
+    date = utils.date_to_dir(options.date)
+    calib_prod_id = utils.get_lstchain_version()
+    catB_calib_dir = Path(cfg.get("LST1", "CAT_B_CALIB_BASE")) / "calibration" / date / calib_prod_id
+    filters = utils.get_calib_filters(run_id)
+    return catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+
+
 def pedestal_ids_file_exists(run_id: int) -> bool:
     """Look for the files with pedestal interleaved event identification."""
     pedestal_ids_dir = Path(cfg.get("LST1", "PEDESTAL_FINDER_DIR"))
@@ -256,7 +254,10 @@ def sequence_calibration_files(sequence_list: List[Sequence]) -> None:
 
 def get_datacheck_files(pattern: str, directory: Path) -> list:
     """Return a list of files matching the pattern."""
-    return sorted(directory.glob(pattern))
+    if pattern=="datacheck_dl1*.pdf":
+        return sorted(directory.glob("tailcut*/datacheck/"+pattern))
+    else:
+        return sorted(directory.glob(pattern))
 
 
 def datacheck_directory(data_type: str, date: str) -> Path:
@@ -264,7 +265,7 @@ def datacheck_directory(data_type: str, date: str) -> Path:
     if data_type in {"PEDESTAL", "CALIB"}:
         directory = Path(cfg.get("LST1", f"{data_type}_DIR")) / date / "pro/log"
     elif data_type == "DL1AB":
-        directory = destination_dir("DATACHECK", create_dir=False)
+        directory = Path(cfg.get("LST1", f"{data_type}_DIR")) / date / options.prod_id
     elif data_type == "LONGTERM":
         directory = Path(cfg.get("LST1", f"{data_type}_DIR")) / options.prod_id / date
     else:
@@ -272,7 +273,7 @@ def datacheck_directory(data_type: str, date: str) -> Path:
     return directory
 
 
-def destination_dir(concept: str, create_dir: bool = True) -> Path:
+def destination_dir(concept: str, create_dir: bool = True, dl1_prod_id: str = None, dl2_prod_id: str = None) -> Path:
     """
     Create final destination directory for each data level.
     See Also osa.utils.register_run_concept_files
@@ -303,7 +304,7 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
             Path(cfg.get(options.tel_id, "DL1_DIR"))
             / nightdir
             / options.prod_id
-            / options.dl1_prod_id
+            / dl1_prod_id
             / "datacheck"
         )
     elif concept == "DL1AB":
@@ -311,13 +312,14 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
             Path(cfg.get(options.tel_id, "DL1_DIR"))
             / nightdir
             / options.prod_id
-            / options.dl1_prod_id
+            / dl1_prod_id
         )
     elif concept in {"DL2", "DL3"}:
         directory = (
             (Path(cfg.get(options.tel_id, f"{concept}_DIR")) / nightdir)
             / options.prod_id
-        ) / options.dl2_prod_id
+            / dl2_prod_id
+        ) 
     elif concept in {"PEDESTAL", "CALIB", "TIMECALIB"}:
         directory = (
             Path(cfg.get(options.tel_id, f"{concept}_DIR"))
@@ -397,6 +399,7 @@ def create_longterm_symlink(cherenkov_job_id: str = None):
     else:
         log.warning(f"Job {cherenkov_job_id} (lstchain_cherenkov_transparency) did not finish successfully.")
 
+
 def dl1_datacheck_longterm_file_exits() -> bool:
     """Return true if the longterm DL1 datacheck file was already produced."""
     nightdir = utils.date_to_dir(options.date)
@@ -404,3 +407,79 @@ def dl1_datacheck_longterm_file_exits() -> bool:
     longterm_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5"
     return longterm_file.exists()
 
+
+def catB_closed_file_exists(run_id: int) -> bool:
+    catB_closed_file = Path(options.directory) / f"catB_{run_id:05d}.closed"
+    return catB_closed_file.exists()
+
+
+def catB_calibration_file_exists(run_id: int) -> bool:
+    catB_calib_base_dir = Path(cfg.get("LST1","CAT_B_CALIB_BASE"))
+    prod_id = utils.get_lstchain_version()
+    night_dir = utils.date_to_dir(options.date)
+    filters = utils.get_calib_filters(run_id)
+    catB_calib_dir = catB_calib_base_dir / "calibration" / night_dir / prod_id 
+    catB_calib_file = catB_calib_dir / f"cat_B_calibration_filters_{filters}.Run{run_id:05d}.h5"
+    return catB_calib_file.exists()
+
+
+def get_dl1_prod_id(config_filename):
+    with open(config_filename) as json_file:
+        data = json.load(json_file)
+        
+    picture_thresh = data["tailcuts_clean_with_pedestal_threshold"]["picture_thresh"]
+    boundary_thresh = data["tailcuts_clean_with_pedestal_threshold"]["boundary_thresh"]
+
+    if boundary_thresh == 4:
+        return f"tailcut{picture_thresh}{boundary_thresh}"
+    else:
+        return f"tailcut{picture_thresh}{boundary_thresh:02d}"
+
+
+def get_dl2_nsb_prod_id(rf_model: Path) -> str:
+    match = re.search(r'nsb_tuning_\d+\.\d+', str(rf_model))
+    if not match:
+        log.warning(f"No 'nsb_tuning_X.XX' pattern found in the path:\n{rf_model}")
+        sys.exit(1)
+    else:
+        return match.group(0)
+    
+
+def get_dl1_prod_id_and_config(run_id: int) -> str:
+    if not cfg.getboolean("lstchain", "apply_standard_dl1b_config"):
+        tailcuts_finder_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+        dl1b_config_file = tailcuts_finder_dir / f"dl1ab_Run{run_id:05d}.json"
+        if not dl1b_config_file.exists()  and not options.simulate:
+            log.error(
+                f"The dl1b config file was not created yet for run {run_id:05d}. "
+                "Please try again later."
+            )
+            sys.exit(1) 
+        else: 
+            dl1_prod_id = get_dl1_prod_id(dl1b_config_file)
+            return dl1_prod_id, dl1b_config_file.resolve()
+    else:
+        dl1b_config_file = Path(cfg.get("lstchain", "dl1b_config"))
+        dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
+        return dl1_prod_id, dl1b_config_file.resolve()
+    
+
+def get_dl2_prod_id(run_id: int) -> str:
+    dl1_prod_id = get_dl1_prod_id_and_config(run_id)[0]
+    rf_model = utils.get_RF_model(run_id)
+    nsb_prod_id = get_dl2_nsb_prod_id(rf_model)
+    return f"{dl1_prod_id}/{nsb_prod_id}"
+
+
+def all_dl1ab_config_files_exist(date: str) -> bool:
+    nightdir = date.replace("-","")
+    run_summary_dir =  Path(cfg.get(options.tel_id, "RUN_SUMMARY_DIR"))
+    run_summary_file = run_summary_dir / f"RunSummary_{nightdir}.ecsv"
+    summary_table = Table.read(run_summary_file)
+    data_runs = summary_table[summary_table["run_type"] == "DATA"]
+    for run_id in data_runs["run_id"]:
+        tailcuts_finder_dir = Path(cfg.get(options.tel_id, "TAILCUTS_FINDER_DIR"))
+        dl1b_config_file = tailcuts_finder_dir / f"dl1ab_Run{run_id:05d}.json"
+        if not dl1b_config_file.exists():
+            return False
+    return True

osa/provenance/capture.py

@@ -53,7 +53,7 @@ LOG_FILENAME = provconfig["handlers"]["provHandler"]["filename"]
 PROV_PREFIX = provconfig["PREFIX"]
 SUPPORTED_HASH_METHOD = ["md5"]
 SUPPORTED_HASH_BUFFER = ["content", "path"]
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 # global variables
 traced_entities = {}

osa/provenance/config/definition.yaml

@@ -200,6 +200,13 @@ activities:
             #   filepath: /fefs/aswg/data/real/DL1/20200218/v0.4.3_v00/
             #   size: 128
 
+    catB_calibration:
+        description: 
+            "Create Cat-B calibration file for an observation run"
+        parameters:
+        usage:
+        generation:
+
     dl1ab:
         description:
             "Create DL1AB files for an observation run"

osa/provenance/utils.py

@@ -10,7 +10,7 @@ from osa.utils.utils import date_to_dir, get_lstchain_version
 
 __all__ = ["parse_variables", "get_log_config"]
 
-REDUCTION_TASKS = ["r0_to_dl1", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
+REDUCTION_TASKS = ["r0_to_dl1", "catB_calibration", "dl1ab", "dl1_datacheck", "dl1_to_dl2"]
 
 
 def parse_variables(class_instance):
@@ -40,20 +40,18 @@ def parse_variables(class_instance):
     configfile_dl1b = cfg.get("lstchain", "dl1b_config")
     configfile_dl2 = cfg.get("lstchain", "dl2_config")
     raw_dir = Path(cfg.get("LST1", "R0_DIR"))
-    rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
+    rf_models_directory = Path(cfg.get("LST1", "RF_MODELS"))
     dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
     dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
-    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CAT_A_CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "CAT_A_PEDESTAL_DIR"))
 
     class_instance.SoftwareVersion = get_lstchain_version()
     class_instance.ProcessingConfigFile = str(options.configfile)
     class_instance.ObservationDate = flat_date
     if class_instance.__name__ in REDUCTION_TASKS:
         muon_dir = dl1_dir / flat_date / options.prod_id / "muons"
-        outdir_dl1 = dl1_dir / flat_date / options.prod_id / options.dl1_prod_id
-        outdir_dl2 = dl2_dir / flat_date / options.prod_id / options.dl2_prod_id
-
+        
     if class_instance.__name__ in ["drs4_pedestal", "calibrate_charge"]:
         # drs4_pedestal_run_id  [0] 1804
         # pedcal_run_id         [1] 1805
@@ -111,6 +109,7 @@ def parse_variables(class_instance):
         run = run_subrun.split(".")[0]
         class_instance.ObservationRun = run
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id
         calibration_file = Path(class_instance.args[0]).resolve()
         pedestal_file = Path(class_instance.args[1]).resolve()
         timecalibration_file = Path(class_instance.args[2]).resolve()
@@ -133,10 +132,16 @@ def parse_variables(class_instance):
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
             class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
+  
+    if class_instance.__name__ == "catB_calibration":
+        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000
+        # dl1b_config   [1]
+        # dl1_prod_id   [2]
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / class_instance.args[2]
         class_instance.Analysisconfigfile_dl1 = str(Path(configfile_dl1b))
         class_instance.ObservationRun = class_instance.args[0].split(".")[0]
         class_instance.StoreImage = cfg.getboolean("lstchain", "store_image_dl1ab")
@@ -146,9 +151,12 @@ def parse_variables(class_instance):
 
     if class_instance.__name__ == "dl1_datacheck":
         # run_str       [0] 02006.0000
+        # dl1b_prod_id  [1]
+
         run_subrun = class_instance.args[0]
         run = run_subrun.split(".")[0]
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / class_instance.args[1]
         class_instance.ObservationRun = run
         class_instance.DL1SubrunDataset = str(
             (outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
@@ -168,9 +176,16 @@ def parse_variables(class_instance):
 
     if class_instance.__name__ == "dl1_to_dl2":
         # run_str       [0] 02006.0000
+        # rf_model_path [1]
+        # dl1_prod_id   [2]
+        # dl2_prod_id   [3]
+
         run_subrun = class_instance.args[0]
         run = run_subrun.split(".")[0]
 
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / class_instance.args[2]
+        outdir_dl2 = dl2_dir / flat_date / options.prod_id / class_instance.args[3]
+
         class_instance.Analysisconfigfile_dl2 = configfile_dl2
         class_instance.ObservationRun = run
         class_instance.RFModelEnergyFile = str((rf_models_directory / "reg_energy.sav").resolve())

osa/scripts/autocloser.py

@@ -266,21 +266,11 @@ class Sequence:
 
     def is_100(self, no_dl2: bool):
         """Check that all analysis products are 100% complete."""
-        if (
-            no_dl2
-            and self.dict_sequence["Tel"] != "ST"
-            and self.dict_sequence["DL1%"] == "100"
-            and self.dict_sequence["DL1AB%"] == "100"
-            and self.dict_sequence["MUONS%"] == "100"
-        ):
-            return True
-
         if (
             self.dict_sequence["Tel"] != "ST"
             and self.dict_sequence["DL1%"] == "100"
             and self.dict_sequence["DL1AB%"] == "100"
             and self.dict_sequence["MUONS%"] == "100"
-            and self.dict_sequence["DL2%"] == "100"
         ):
             return True
 

osa/scripts/calibration_pipeline.py

@@ -44,22 +44,27 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
     """Build the create_drs4_pedestal command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
+    command = cfg.get("lstchain", "drs4_baseline")
     return [
-        "onsite_create_drs4_pedestal_file",
-        f"--run_number={drs4_pedestal_run_id}",
-        f"--base_dir={base_dir}",
+        command,
+        "-r", str(drs4_pedestal_run_id),
+        "-b", base_dir,
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
 
-
 def calibration_file_command(drs4_pedestal_run_id: int, pedcal_run_id: int) -> list:
     """Build the create_calibration_file command."""
     base_dir = Path(cfg.get("LST1", "BASE")).resolve()
+    r0_dir = Path(cfg.get("LST1", "R0_DIR")).resolve()
+    command = cfg.get("lstchain", "charge_calibration")
     cmd = [
-        "onsite_create_calibration_file",
-        f"--pedestal_run={drs4_pedestal_run_id}",
-        f"--run_number={pedcal_run_id}",
-        f"--base_dir={base_dir}",
+        command,
+        "-p", str(drs4_pedestal_run_id),
+        "-r", str(pedcal_run_id),
+        "-b", base_dir,
+        f"--r0-dir={r0_dir}",
     ]
     # In case of problems with trigger tagging:
     if cfg.getboolean("lstchain", "use_ff_heuristic_id"):

osa/scripts/closer.py

@@ -169,14 +169,14 @@ def post_process(seq_tuple):
         # Extract the provenance info
         extract_provenance(seq_list)
 
-        # Merge DL1b files run-wise
-        merge_files(seq_list, data_level="DL1AB")
-
         merge_muon_files(seq_list)
 
-        # Merge DL2 files run-wise
-        if not options.no_dl2:
-            merge_files(seq_list, data_level="DL2")
+        # Merge DL1b files run-wise
+        for sequence in seq_list:
+            dl1_merge_job_id = merge_files(sequence, data_level="DL1AB")
+            # Produce DL2 files run-wise
+            if not options.no_dl2 and sequence.type=="DATA":
+                dl1_to_dl2(sequence, dl1_merge_job_id)
 
         # Merge DL1 datacheck files and produce PDFs. It also produces
         # the daily datacheck report using the longterm script, and updates
@@ -215,6 +215,66 @@ def post_process(seq_tuple):
     return False
 
 
+def dl1_to_dl2(sequence, dl1_merge_job_id) -> int:
+    """
+    It prepares and execute the dl1 to dl2 lstchain scripts that applies
+    the already trained RFs models to DL1 files. It identifies the
+    primary particle, reconstructs its energy and direction.
+
+    Parameters
+    ----------
+    run_str: str
+
+    Returns
+    -------
+    rc: int
+    """
+    nightdir = date_to_dir(options.date)
+    dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
+    dl2_subdirectory = dl2_dir / nightdir / options.prod_id / sequence.dl2_prod_id
+    dl2_file = dl2_subdirectory / f"dl2_LST-1.Run{sequence.run_str[:5]}.h5"
+    dl2_config = Path(cfg.get("lstchain", "dl2_config"))
+    dl1ab_subdirectory = Path(cfg.get("LST1", "DL1AB_DIR"))
+    dl1_file = dl1ab_subdirectory / nightdir / options.prod_id / sequence.dl1_prod_id / f"dl1_LST-1.Run{sequence.run_str[:5]}.h5"
+    log_dir = Path(options.directory) / "log"
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
+
+    if dl2_file.exists():
+        log.debug(f"The dl2 file {dl2_file} already exists.")
+        return 0
+    
+    command = cfg.get("lstchain", "dl1_to_dl2")
+    cmd = [
+        "sbatch",
+        "--parsable",
+        "--mem-per-cpu=60GB",
+        f"--account={slurm_account}",
+        "-o", f"{log_dir}/Run{sequence.run_str[:5]}_dl2_%j.out",
+        "-e", f"{log_dir}/Run{sequence.run_str[:5]}_dl2_%j.err",
+        f"--dependency=afterok:{dl1_merge_job_id}",
+        command,
+        f"--input-file={dl1_file}",
+        f"--output-dir={dl2_subdirectory}",
+        f"--path-models={sequence.rf_model}",
+        f"--config={dl2_config}",
+    ]
+    log.info(f"executing {cmd}")
+
+    if options.simulate:
+        return 0
+    
+    if not options.test and shutil.which("sbatch") is not None:
+        job = subprocess.run(
+            cmd,
+            encoding="utf-8",
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        job_id = job.stdout.strip()
+        return job_id
+
+
 def post_process_files(seq_list: list):
     """
     Identify the different types of files, try to close the sequences
@@ -226,9 +286,7 @@ def post_process_files(seq_list: list):
         list of sequences
     """
 
-    output_files_set = set(Path(options.directory).rglob("*Run*"))
-
-    DL1AB_RE = re.compile(rf"{options.dl1_prod_id}/dl1.*.(?:h5|hdf5|hdf)")
+    DL1AB_RE = re.compile(r"tailcut.*/dl1.*.(?:h5|hdf5|hdf)")
     MUONS_RE = re.compile(r"muons.*.fits")
     DATACHECK_RE = re.compile(r"datacheck_dl1.*.(?:h5|hdf5|hdf)")
     INTERLEAVED_RE = re.compile(r"interleaved.*.(?:h5|hdf5|hdf)")
@@ -243,27 +301,36 @@ def post_process_files(seq_list: list):
     )
 
     if not options.no_dl2:
-        DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
+        DL2_RE = re.compile("tailcut.*/nsb_tuning_.*/dl2.*.(?:h5|hdf5|hdf)")
         pattern_files["DL2"] = DL2_RE
 
     for concept, pattern_re in pattern_files.items():
-        log.info(f"Post processing {concept} files, {len(output_files_set)} files left")
+        for sequence in seq_list:
+            output_files_set = set(Path(options.directory).rglob(f"*Run{sequence.run:05d}*"))
+            log.info(f"Post processing {concept} files, {len(output_files_set)} files left")
+            
+            if sequence.type=="DATA":
+                dst_path = destination_dir(
+                    concept,
+                    create_dir=True,
+                    dl1_prod_id=sequence.dl1_prod_id,
+                    dl2_prod_id=sequence.dl2_prod_id
+                )
 
-        dst_path = destination_dir(concept, create_dir=True)
+                log.debug(f"Checking if {concept} files need to be moved to {dst_path}")
 
-        log.debug(f"Checking if {concept} files need to be moved to {dst_path}")
+                for file_path in output_files_set.copy():
 
-        for file_path in output_files_set.copy():
+                    file = str(file_path)
 
-            file = str(file_path)
-            # If seqtoclose is set, we only want to close that sequence
-            if options.seqtoclose is not None and options.seqtoclose not in file:
-                continue
+                    # If seqtoclose is set, we only want to close that sequence
+                    if options.seqtoclose is not None and options.seqtoclose not in file:
+                        continue
 
-            if pattern_found := pattern_re.search(file):
-                log.debug(f"Pattern {concept} found, {pattern_found} in {file}")
-                registered_file = register_found_pattern(file_path, seq_list, concept, dst_path)
-                output_files_set.remove(registered_file)
+                    if pattern_found := pattern_re.search(file):
+                        log.debug(f"Pattern {concept} found, {pattern_found} in {file}")
+                        registered_file = register_found_pattern(file_path, seq_list, concept, dst_path)
+                        output_files_set.remove(registered_file)
 
 
 def set_closed_with_file():
@@ -335,13 +402,13 @@ def merge_dl1_datacheck(seq_list) -> List[str]:
     log.debug("Merging dl1 datacheck files and producing PDFs")
 
     muons_dir = destination_dir("MUON", create_dir=False)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
     slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     list_job_id = []
 
     for sequence in seq_list:
         if sequence.type == "DATA":
+            datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id=sequence.dl1_prod_id)
             cmd = [
                 "sbatch",
                 "--parsable",
@@ -387,7 +454,7 @@ def extract_provenance(seq_list):
     """
     log.info("Extract provenance run wise")
 
-    nightdir = date_to_dir(options.date)
+    nightdir = date_to_iso(options.date)
     slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     for sequence in seq_list:
@@ -431,40 +498,52 @@ def get_pattern(data_level) -> Tuple[str, str]:
     raise ValueError(f"Unknown data level {data_level}")
 
 
-def merge_files(sequence_list, data_level="DL2"):
+def merge_files(sequence, data_level="DL2"):
     """Merge DL1b or DL2 h5 files run-wise."""
     log.info(f"Looping over the sequences and merging the {data_level} files")
-
-    data_dir = destination_dir(data_level, create_dir=False)
     pattern, prefix = get_pattern(data_level)
     slurm_account = cfg.get("SLURM", "ACCOUNT")
+    
+    if sequence.type == "DATA":
+        data_dir = destination_dir(
+            data_level,
+            create_dir=False,
+            dl1_prod_id=sequence.dl1_prod_id,
+            dl2_prod_id=sequence.dl2_prod_id
+            )
+        merged_file = Path(data_dir) / f"{prefix}_LST-1.Run{sequence.run:05d}.h5"
 
-    for sequence in sequence_list:
-        if sequence.type == "DATA":
-            merged_file = Path(data_dir) / f"{prefix}_LST-1.Run{sequence.run:05d}.h5"
-
-            cmd = [
-                "sbatch",
-                f"--account={slurm_account}",
-                "-D",
-                options.directory,
-                "-o",
-                f"log/merge_{prefix}_{sequence.run:05d}_%j.log",
-                "lstchain_merge_hdf5_files",
-                f"--input-dir={data_dir}",
-                f"--output-file={merged_file}",
-                "--no-image",
-                "--no-progress",
-                f"--run-number={sequence.run}",
-                f"--pattern={pattern}",
-            ]
+        cmd = [
+            "sbatch",
+            "--parsable",
+            f"--account={slurm_account}",
+            "-D",
+            options.directory,
+            "-o",
+            f"log/merge_{prefix}_{sequence.run:05d}_%j.log",
+            "lstchain_merge_hdf5_files",
+            f"--input-dir={data_dir}",
+            f"--output-file={merged_file}",
+            "--no-image",
+            "--no-progress",
+            f"--run-number={sequence.run}",
+            f"--pattern={pattern}",
+        ]
 
-            log.debug(f"Executing {stringify(cmd)}")
+        log.debug(f"Executing {stringify(cmd)}")
 
-            if not options.simulate and not options.test and shutil.which("sbatch") is not None:
-                subprocess.run(cmd, check=True)
-            else:
-                log.debug("Simulate launching scripts")
+        if not options.simulate and not options.test and shutil.which("sbatch") is not None:
+            job = subprocess.run(
+                cmd,
+                encoding="utf-8",
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            job_id = job.stdout.strip()
+            return job_id
+        else:
+            log.debug("Simulate launching scripts")
 
 
 def merge_muon_files(sequence_list):
@@ -503,7 +582,7 @@ def merge_muon_files(sequence_list):
 def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     """Build the daily longterm command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
@@ -548,7 +627,7 @@ def daily_datacheck(cmd: List[str]):
 def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
     """Build the cherenkov transparency command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
     slurm_account = cfg.get("SLURM", "ACCOUNT")