lstosa 0.10.18__py3-none-any.whl → 0.11.0__py3-none-any.whl

osa/scripts/closer.py

@@ -160,7 +160,8 @@ def post_process(seq_tuple):
     seq_list = seq_tuple[1]
     
     if dl1_datacheck_longterm_file_exits() and not options.test:
-        create_longterm_symlink()
+        if cfg.getboolean("lstchain", "create_longterm_symlink"):
+            create_longterm_symlink()
 
     else:
         # Close the sequences
@@ -169,14 +170,14 @@ def post_process(seq_tuple):
         # Extract the provenance info
         extract_provenance(seq_list)
 
-        # Merge DL1b files run-wise
-        merge_files(seq_list, data_level="DL1AB")
-
         merge_muon_files(seq_list)
 
-        # Merge DL2 files run-wise
-        if not options.no_dl2:
-            merge_files(seq_list, data_level="DL2")
+        # Merge DL1b files run-wise
+        for sequence in seq_list:
+            dl1_merge_job_id = merge_files(sequence, data_level="DL1AB")
+            # Produce DL2 files run-wise
+            if not options.no_dl2 and sequence.type=="DATA":
+                dl1_to_dl2(sequence, dl1_merge_job_id)
 
         # Merge DL1 datacheck files and produce PDFs. It also produces
         # the daily datacheck report using the longterm script, and updates
@@ -185,7 +186,8 @@ def post_process(seq_tuple):
             list_job_id = merge_dl1_datacheck(seq_list)
             longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
             cherenkov_job_id = cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
-            create_longterm_symlink(cherenkov_job_id)
+            if cfg.getboolean("lstchain", "create_longterm_symlink"):
+                create_longterm_symlink(cherenkov_job_id)
 
         time.sleep(600)
 
@@ -215,6 +217,66 @@ def post_process(seq_tuple):
     return False
 
 
+def dl1_to_dl2(sequence, dl1_merge_job_id) -> int:
+    """
+    It prepares and execute the dl1 to dl2 lstchain scripts that applies
+    the already trained RFs models to DL1 files. It identifies the
+    primary particle, reconstructs its energy and direction.
+
+    Parameters
+    ----------
+    run_str: str
+
+    Returns
+    -------
+    rc: int
+    """
+    nightdir = date_to_dir(options.date)
+    dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
+    dl2_subdirectory = dl2_dir / nightdir / options.prod_id / sequence.dl2_prod_id
+    dl2_file = dl2_subdirectory / f"dl2_LST-1.Run{sequence.run_str[:5]}.h5"
+    dl2_config = Path(cfg.get("lstchain", "dl2_config"))
+    dl1ab_subdirectory = Path(cfg.get("LST1", "DL1AB_DIR"))
+    dl1_file = dl1ab_subdirectory / nightdir / options.prod_id / sequence.dl1_prod_id / f"dl1_LST-1.Run{sequence.run_str[:5]}.h5"
+    log_dir = Path(options.directory) / "log"
+    slurm_account = cfg.get("SLURM", "ACCOUNT")
+
+    if dl2_file.exists():
+        log.debug(f"The dl2 file {dl2_file} already exists.")
+        return 0
+    
+    command = cfg.get("lstchain", "dl1_to_dl2")
+    cmd = [
+        "sbatch",
+        "--parsable",
+        "--mem-per-cpu=60GB",
+        f"--account={slurm_account}",
+        "-o", f"{log_dir}/Run{sequence.run_str[:5]}_dl2_%j.out",
+        "-e", f"{log_dir}/Run{sequence.run_str[:5]}_dl2_%j.err",
+        f"--dependency=afterok:{dl1_merge_job_id}",
+        command,
+        f"--input-file={dl1_file}",
+        f"--output-dir={dl2_subdirectory}",
+        f"--path-models={sequence.rf_model}",
+        f"--config={dl2_config}",
+    ]
+    log.info(f"executing {cmd}")
+
+    if options.simulate:
+        return 0
+    
+    if not options.test and shutil.which("sbatch") is not None:
+        job = subprocess.run(
+            cmd,
+            encoding="utf-8",
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        job_id = job.stdout.strip()
+        return job_id
+
+
 def post_process_files(seq_list: list):
     """
     Identify the different types of files, try to close the sequences
@@ -226,9 +288,7 @@ def post_process_files(seq_list: list):
         list of sequences
     """
 
-    output_files_set = set(Path(options.directory).rglob("*Run*"))
-
-    DL1AB_RE = re.compile(rf"{options.dl1_prod_id}/dl1.*.(?:h5|hdf5|hdf)")
+    DL1AB_RE = re.compile(r"tailcut.*/dl1.*.(?:h5|hdf5|hdf)")
     MUONS_RE = re.compile(r"muons.*.fits")
     DATACHECK_RE = re.compile(r"datacheck_dl1.*.(?:h5|hdf5|hdf)")
     INTERLEAVED_RE = re.compile(r"interleaved.*.(?:h5|hdf5|hdf)")
@@ -243,27 +303,36 @@ def post_process_files(seq_list: list):
     )
 
     if not options.no_dl2:
-        DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
+        DL2_RE = re.compile("tailcut.*/nsb_tuning_.*/dl2.*.(?:h5|hdf5|hdf)")
         pattern_files["DL2"] = DL2_RE
 
     for concept, pattern_re in pattern_files.items():
-        log.info(f"Post processing {concept} files, {len(output_files_set)} files left")
+        for sequence in seq_list:
+            output_files_set = set(Path(options.directory).rglob(f"*Run{sequence.run:05d}*"))
+            log.info(f"Post processing {concept} files, {len(output_files_set)} files left")
+            
+            if sequence.type=="DATA":
+                dst_path = destination_dir(
+                    concept,
+                    create_dir=True,
+                    dl1_prod_id=sequence.dl1_prod_id,
+                    dl2_prod_id=sequence.dl2_prod_id
+                )
 
-        dst_path = destination_dir(concept, create_dir=True)
+                log.debug(f"Checking if {concept} files need to be moved to {dst_path}")
 
-        log.debug(f"Checking if {concept} files need to be moved to {dst_path}")
+                for file_path in output_files_set.copy():
 
-        for file_path in output_files_set.copy():
+                    file = str(file_path)
 
-            file = str(file_path)
-            # If seqtoclose is set, we only want to close that sequence
-            if options.seqtoclose is not None and options.seqtoclose not in file:
-                continue
+                    # If seqtoclose is set, we only want to close that sequence
+                    if options.seqtoclose is not None and options.seqtoclose not in file:
+                        continue
 
-            if pattern_found := pattern_re.search(file):
-                log.debug(f"Pattern {concept} found, {pattern_found} in {file}")
-                registered_file = register_found_pattern(file_path, seq_list, concept, dst_path)
-                output_files_set.remove(registered_file)
+                    if pattern_found := pattern_re.search(file):
+                        log.debug(f"Pattern {concept} found, {pattern_found} in {file}")
+                        registered_file = register_found_pattern(file_path, seq_list, concept, dst_path)
+                        output_files_set.remove(registered_file)
 
 
 def set_closed_with_file():
@@ -335,13 +404,13 @@ def merge_dl1_datacheck(seq_list) -> List[str]:
     log.debug("Merging dl1 datacheck files and producing PDFs")
 
     muons_dir = destination_dir("MUON", create_dir=False)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
     slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     list_job_id = []
 
     for sequence in seq_list:
         if sequence.type == "DATA":
+            datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id=sequence.dl1_prod_id)
             cmd = [
                 "sbatch",
                 "--parsable",
@@ -387,7 +456,7 @@ def extract_provenance(seq_list):
     """
     log.info("Extract provenance run wise")
 
-    nightdir = date_to_dir(options.date)
+    nightdir = date_to_iso(options.date)
     slurm_account = cfg.get("SLURM", "ACCOUNT")
 
     for sequence in seq_list:
@@ -431,40 +500,52 @@ def get_pattern(data_level) -> Tuple[str, str]:
     raise ValueError(f"Unknown data level {data_level}")
 
 
-def merge_files(sequence_list, data_level="DL2"):
+def merge_files(sequence, data_level="DL2"):
     """Merge DL1b or DL2 h5 files run-wise."""
     log.info(f"Looping over the sequences and merging the {data_level} files")
-
-    data_dir = destination_dir(data_level, create_dir=False)
     pattern, prefix = get_pattern(data_level)
     slurm_account = cfg.get("SLURM", "ACCOUNT")
+    
+    if sequence.type == "DATA":
+        data_dir = destination_dir(
+            data_level,
+            create_dir=False,
+            dl1_prod_id=sequence.dl1_prod_id,
+            dl2_prod_id=sequence.dl2_prod_id
+            )
+        merged_file = Path(data_dir) / f"{prefix}_LST-1.Run{sequence.run:05d}.h5"
 
-    for sequence in sequence_list:
-        if sequence.type == "DATA":
-            merged_file = Path(data_dir) / f"{prefix}_LST-1.Run{sequence.run:05d}.h5"
-
-            cmd = [
-                "sbatch",
-                f"--account={slurm_account}",
-                "-D",
-                options.directory,
-                "-o",
-                f"log/merge_{prefix}_{sequence.run:05d}_%j.log",
-                "lstchain_merge_hdf5_files",
-                f"--input-dir={data_dir}",
-                f"--output-file={merged_file}",
-                "--no-image",
-                "--no-progress",
-                f"--run-number={sequence.run}",
-                f"--pattern={pattern}",
-            ]
+        cmd = [
+            "sbatch",
+            "--parsable",
+            f"--account={slurm_account}",
+            "-D",
+            options.directory,
+            "-o",
+            f"log/merge_{prefix}_{sequence.run:05d}_%j.log",
+            "lstchain_merge_hdf5_files",
+            f"--input-dir={data_dir}",
+            f"--output-file={merged_file}",
+            "--no-image",
+            "--no-progress",
+            f"--run-number={sequence.run}",
+            f"--pattern={pattern}",
+        ]
 
-            log.debug(f"Executing {stringify(cmd)}")
+        log.debug(f"Executing {stringify(cmd)}")
 
-            if not options.simulate and not options.test and shutil.which("sbatch") is not None:
-                subprocess.run(cmd, check=True)
-            else:
-                log.debug("Simulate launching scripts")
+        if not options.simulate and not options.test and shutil.which("sbatch") is not None:
+            job = subprocess.run(
+                cmd,
+                encoding="utf-8",
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            job_id = job.stdout.strip()
+            return job_id
+        else:
+            log.debug("Simulate launching scripts")
 
 
 def merge_muon_files(sequence_list):
@@ -503,7 +584,7 @@ def merge_muon_files(sequence_list):
 def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     """Build the daily longterm command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
@@ -548,7 +629,7 @@ def daily_datacheck(cmd: List[str]):
 def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
     """Build the cherenkov transparency command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False)
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
     slurm_account = cfg.get("SLURM", "ACCOUNT")

osa/scripts/copy_datacheck.py

@@ -5,12 +5,13 @@ directories whenever they are needed.
 """
 
 import logging
+from pathlib import Path
 
 from osa.configs import options
+from osa.configs.config import cfg
 from osa.paths import (
     datacheck_directory,
     get_datacheck_files,
-    destination_dir,
 )
 from osa.utils.cliopts import copy_datacheck_parsing
 from osa.utils.logging import myLogger
@@ -90,8 +91,9 @@ def get_number_of_runs():
     Get the run sequence processed list for the given date by globbing the
     run-wise DL1 files.
     """
-    dl1_directory = destination_dir("DL1AB", create_dir=False)
-    list_files = list(dl1_directory.glob("dl1_LST-1.Run?????.h5"))
+    nightdir = date_to_dir(options.date)
+    dl1_directory = Path(cfg.get("LST1", "DL1_DIR")) / nightdir / options.prod_id
+    list_files = list(dl1_directory.glob("tailcut*/dl1_LST-1.Run?????.h5"))
     return len(list_files)
 
 

osa/scripts/datasequence.py

@@ -9,11 +9,13 @@ from osa.configs.config import cfg
 from osa.job import historylevel
 from osa.workflow.stages import AnalysisStage
 from osa.provenance.capture import trace
+from osa.paths import get_catB_calibration_filename
 from osa.utils.cliopts import data_sequence_cli_parsing
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir
+from osa.paths import catB_closed_file_exists
 
-__all__ = ["data_sequence", "r0_to_dl1", "dl1_to_dl2", "dl1ab", "dl1_datacheck"]
+__all__ = ["data_sequence", "r0_to_dl1", "dl1ab", "dl1_datacheck"]
 
 log = myLogger(logging.getLogger())
 
@@ -27,6 +29,8 @@ def data_sequence(
     run_summary: Path,
     pedestal_ids_file: Path,
     run_str: str,
+    dl1b_config: Path,
+    dl1_prod_id: str,
 ):
     """
     Performs all the steps to process a whole run.
@@ -50,10 +54,10 @@ def data_sequence(
     history_file = Path(options.directory) / f"sequence_{options.tel_id}_{run_str}.history"
     # Set the starting level and corresponding return code from last analysis step
     # registered in the history file.
-    level, rc = (4, 0) if options.simulate else historylevel(history_file, "DATA")
+    level, rc = (3, 0) if options.simulate else historylevel(history_file, "DATA")
     log.info(f"Going to level {level}")
 
-    if level == 4:
+    if level == 3:
         rc = r0_to_dl1(
             calibration_file,
             pedestal_file,
@@ -67,32 +71,23 @@ def data_sequence(
         level -= 1
         log.info(f"Going to level {level}")
 
-    if level == 3:
-        rc = dl1ab(run_str)
-        if cfg.getboolean("lstchain", "store_image_dl1ab"):
-            level -= 1
-            log.info(f"Going to level {level}")
-        else:
-            level -= 2
-            log.info(f"No images stored in dl1ab. Producing DL2. Going to level {level}")
-
     if level == 2:
-        rc = dl1_datacheck(run_str)
-        if options.no_dl2:
+        if options.no_dl1ab:
             level = 0
-            log.info(f"No DL2 are going to be produced. Going to level {level}")
+            log.info(f"No DL1B are going to be produced. Going to level {level}")
         else:
-            level -= 1
-            log.info(f"Going to level {level}")
+            rc = dl1ab(run_str, dl1b_config, dl1_prod_id)
+            if cfg.getboolean("lstchain", "store_image_dl1ab"):
+                level -= 1
+                log.info(f"Going to level {level}")
+            else:
+                level -= 2
+                log.info(f"No images stored in dl1ab. Going to level {level}")
 
     if level == 1:
-        if options.no_dl2:
-            level = 0
-            log.info(f"No DL2 are going to be produced. Going to level {level}")
-        else:
-            rc = dl1_to_dl2(run_str)
-            level -= 1
-            log.info(f"Going to level {level}")
+        rc = dl1_datacheck(run_str, dl1_prod_id)
+        level -= 1
+        log.info(f"Going to level {level}")
 
     if level == 0:
         log.info(f"Job for sequence {run_str} finished without fatal errors")
@@ -166,7 +161,7 @@ def r0_to_dl1(
 
 
 @trace
-def dl1ab(run_str: str) -> int:
+def dl1ab(run_str: str, dl1b_config: Path, dl1_prod_id: str) -> int:
     """
     Prepare and launch the actual lstchain script that is performing
     the image cleaning considering the interleaved pedestal information
@@ -181,26 +176,39 @@ def dl1ab(run_str: str) -> int:
     rc: int
         Return code of the executed command.
     """
+    
+    # Prepare and launch the actual lstchain script
+    command = cfg.get("lstchain", "dl1ab")
+    
     # Create a new subdirectory for the dl1ab output
-    dl1ab_subdirectory = Path(options.directory) / options.dl1_prod_id
+    dl1ab_subdirectory = Path(options.directory) / dl1_prod_id
     dl1ab_subdirectory.mkdir(parents=True, exist_ok=True)
-    dl1b_config = Path(cfg.get("lstchain", "dl1b_config"))
     # DL1a input file from base running_analysis directory
     input_dl1_datafile = Path(options.directory) / f"dl1_LST-1.Run{run_str}.h5"
     # DL1b output file to be stored in the dl1ab subdirectory
     output_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
 
-    # Prepare and launch the actual lstchain script
-    command = cfg.get("lstchain", "dl1ab")
     cmd = [
         command,
         f"--input-file={input_dl1_datafile}",
         f"--output-file={output_dl1_datafile}",
         f"--config={dl1b_config}",
     ]
+    
     if not cfg.getboolean("lstchain", "store_image_dl1ab"):
         cmd.append("--no-image=True")
 
+    if cfg.getboolean("lstchain", "apply_catB_calibration"):
+        if catB_closed_file_exists(int(run_str[:5])):
+            catB_calibration_file = get_catB_calibration_filename(int(run_str[:5]))
+            cmd.append(f"--catB-calibration-file={catB_calibration_file}")
+        else:
+            log.info(
+                f"Cat-B calibration did not finish yet for run {run_str[:5]}. "
+                "Please try again later."
+            )
+            sys.exit(1)
+
     if options.simulate:
         return 0
 
@@ -210,7 +218,7 @@ def dl1ab(run_str: str) -> int:
 
 
 @trace
-def dl1_datacheck(run_str: str) -> int:
+def dl1_datacheck(run_str: str, dl1_prod_id: str) -> int:
     """
     Run datacheck script
 
@@ -223,9 +231,9 @@ def dl1_datacheck(run_str: str) -> int:
     rc: int
     """
     # Create a new subdirectory for the dl1ab output
-    dl1ab_subdirectory = Path(options.directory) / options.dl1_prod_id
+    dl1ab_subdirectory = Path(options.directory) / dl1_prod_id
     input_dl1_datafile = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
-    output_directory = Path(options.directory) / options.dl1_prod_id
+    output_directory = Path(options.directory) / dl1_prod_id
     output_directory.mkdir(parents=True, exist_ok=True)
 
     # Prepare and launch the actual lstchain script
@@ -247,46 +255,8 @@ def dl1_datacheck(run_str: str) -> int:
     return analysis_step.rc
 
 
-@trace
-def dl1_to_dl2(run_str: str) -> int:
-    """
-    It prepares and execute the dl1 to dl2 lstchain scripts that applies
-    the already trained RFs models to DL1 files. It identifies the
-    primary particle, reconstructs its energy and direction.
-
-    Parameters
-    ----------
-    run_str: str
-
-    Returns
-    -------
-    rc: int
-    """
-    dl1ab_subdirectory = Path(options.directory) / options.dl1_prod_id
-    dl2_subdirectory = Path(options.directory) / options.dl2_prod_id
-    dl2_config = Path(cfg.get("lstchain", "dl2_config"))
-    rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
-    dl1_file = dl1ab_subdirectory / f"dl1_LST-1.Run{run_str}.h5"
-
-    command = cfg.get("lstchain", "dl1_to_dl2")
-    cmd = [
-        command,
-        f"--input-file={dl1_file}",
-        f"--output-dir={dl2_subdirectory}",
-        f"--path-models={rf_models_directory}",
-        f"--config={dl2_config}",
-    ]
-
-    if options.simulate:
-        return 0
-
-    analysis_step = AnalysisStage(run=run_str, command_args=cmd, config_file=dl2_config.name)
-    analysis_step.execute()
-    return analysis_step.rc
-
-
 def main():
-    """Performs the analysis steps to convert raw data into DL2 files."""
+    """Performs the analysis steps to convert raw data into DL1b files."""
     (
         calibration_file,
         drs4_ped_file,
@@ -296,6 +266,8 @@ def main():
         run_summary_file,
         pedestal_ids_file,
         run_number,
+        dl1b_config,
+        dl1_prod_id,
     ) = data_sequence_cli_parsing()
 
     if options.verbose:
@@ -313,6 +285,8 @@ def main():
         run_summary_file,
         pedestal_ids_file,
         run_number,
+        dl1b_config,
+        dl1_prod_id,
     )
     sys.exit(rc)
 

osa/scripts/gain_selection.py

@@ -276,11 +276,10 @@ def apply_gain_selection(date: datetime, start: int, end: int, tool: str = None,
     data_runs = summary_table[summary_table["run_type"] == "DATA"]
     log.info(f"Found {len(data_runs)} DATA runs to which apply the gain selection")
 
-    base_dir = Path(cfg.get("LST1", "BASE"))
     date_str = date_to_dir(date)
-    r0_dir = base_dir / "R0" / date_str
-    output_dir = base_dir / f"R0G/{date_str}"
-    log_dir = base_dir / f"R0G/log/{date_str}"
+    r0_dir = Path(cfg.get("LST1", "RAW_R0_DIR")) / date_str
+    output_dir = Path(cfg.get("LST1", "R0_DIR")) / date_str
+    log_dir = Path(cfg.get("LST1", "R0_DIR")) / f"log/{date_str}"
     if not simulate:
         output_dir.mkdir(parents=True, exist_ok=True)
         log_dir.mkdir(parents=True, exist_ok=True)
@@ -356,9 +355,10 @@ def update_history_file(run_id: str, subrun: str, log_dir: Path, history_file: P
 
 def is_run_already_copied(date: datetime, run_id: int) -> bool:
     """Check if the R0 files of a given run have already been copied to the R0G directory."""
-    base_dir = Path(cfg.get("LST1", "BASE"))
-    r0_files = glob.glob(f"{base_dir}/R0/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
-    r0g_files = glob.glob(f"{base_dir}/R0G/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    r0_dir = Path(cfg.get("LST1", "RAW_R0_DIR"))
+    r0g_dir = Path(cfg.get("LST1", "R0_DIR"))
+    r0_files = glob.glob(f"{r0_dir}/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
+    r0g_files = glob.glob(f"{r0g_dir}/{date_to_dir(date)}/LST-1.?.Run{run_id:05d}.????.fits.fz")
     return len(r0_files)==len(r0g_files)
 
 
@@ -465,9 +465,10 @@ def check_failed_jobs(date: datetime):
     missing_runs = []
 
     date_str = date_to_dir(date)
-    base_dir = Path(cfg.get("LST1", "BASE"))
-    r0_files = glob.glob(f"{base_dir}/R0/{date_str}/LST-1.?.Run?????.????.fits.fz")
-    r0g_files = glob.glob(f"{base_dir}/R0G/{date_str}/LST-1.?.Run?????.????.fits.fz")
+    r0_dir = Path(cfg.get("LST1", "RAW_R0_DIR")) / date_str
+    r0g_dir = Path(cfg.get("LST1", "R0_DIR")) / date_str
+    r0_files = glob.glob(f"{r0_dir}/LST-1.?.Run?????.????.fits.fz")
+    r0g_files = glob.glob(f"{r0g_dir}/LST-1.?.Run?????.????.fits.fz")
     all_r0_runs = [parse_r0_filename(i).run for i in r0_files]
     all_r0g_runs = [parse_r0_filename(i).run for i in r0g_files]
 
@@ -478,17 +479,15 @@ def check_failed_jobs(date: datetime):
 
     missing_runs.sort()
     if missing_runs:
-        output_dir = base_dir / f"R0G/{date_str}/"
         log.info(
             f"Some runs are missing. Copying R0 files of runs {pd.Series(missing_runs).unique()} "
-            f"directly to {output_dir}"
+            f"directly to {r0g_dir}"
         )
 
         for run in missing_runs:
-            
-            files = base_dir.glob(f"R0/{date_str}/LST-1.?.Run{run:05d}.????.fits.fz")
+            files = r0_dir.glob(f"LST-1.?.Run{run:05d}.????.fits.fz")
             for file in files:
-                sp.run(["cp", file, output_dir])
+                sp.run(["cp", file, r0g_dir])
 
     GainSel_dir = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR"))
     flagfile_dir = GainSel_dir / date_str