lstosa 0.11.6__py3-none-any.whl → 0.11.7__py3-none-any.whl

{lstosa-0.11.6.dist-info → lstosa-0.11.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.11.6
+Version: 0.11.7
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>

{lstosa-0.11.6.dist-info → lstosa-0.11.7.dist-info}/RECORD

@@ -1,9 +1,9 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=_nuaDP9J6XpAZCjMK8MqVIMfgWOB6l17LU-S9ZDw0Lg,706
-osa/conftest.py,sha256=AdSHdWqjeMTgIeZEF8fdw6eSliySouivMWCPIqUV2os,24667
+osa/_version.py,sha256=KU0yNPBwd_J_8Kv0fRgFC7mC3xqwbIvT7rTRw-5uF-A,706
+osa/conftest.py,sha256=ziqbE03yLFiBw4bEQv-1jEYz0rNpTLwfsvxBYEri0TE,24608
 osa/job.py,sha256=CunwW7xA4mWEocS6KkDZ1K6h_LYh_ePZMyGHyCa6CKg,27863
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
-osa/paths.py,sha256=B8-HA39YmThl2Dd9p3OFuYSYJf9MWg4Qdc0zmAhWjOE,18100
+osa/paths.py,sha256=tgFXTMyDKVLMLuoXGYFcfeTmOcVcnZXLw1GnMOruvhc,19963
 osa/raw.py,sha256=ZNIsuqfx5ljoz_hwhSuafdKf-wr8-cxRJmel-A2endg,1337
 osa/report.py,sha256=sL2V7n8Y_UUaSDbWJY2o4UxDb4FU5AaFIRR8R25DB8o,4634
 osa/version.py,sha256=9T2TtuGBQeOy5PJDxMCeGlqx5baxLaq47VmFTDc09z8,796
@@ -12,7 +12,7 @@ osa/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/configs/config.py,sha256=cX0Vr4sorBVQ2KRqPIRSEGENKH_uC5lrgVHx1hp6YTk,1148
 osa/configs/datamodel.py,sha256=L_WRM91PBlMrtuE30akh7YR-56P0g9D994qzKSfhNJc,1950
 osa/configs/options.py,sha256=WbsyKhOs1Ud4Yt21O8KcISUvJYmhipa8vl_pxD_TZf4,558
-osa/configs/sequencer.cfg,sha256=49m-umbLo9w4gCv1oEdvFqukCh1py2nJDduOu28b_Qc,5652
+osa/configs/sequencer.cfg,sha256=CEqegb_Irbx0fM6KcQgbvoMXIAI1TyYTKjR8AToSr8k,5690
 osa/high_level/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/high_level/selection_cuts.toml,sha256=ReSmcKtOPZY5JsZ9ExnxYdz7OrJEB8gghCbzHmeOyFg,128
 osa/high_level/significance.py,sha256=mMeG_y2wDnt0O2lSosYkSjaGZQl0V4GnuFrqKwSKSbE,9066
@@ -36,13 +36,14 @@ osa/provenance/config/logger.yaml,sha256=hy_lH3DfbRFh2VM_iawI-c-3wE0cjTRHy465C2e
 osa/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/scripts/autocloser.py,sha256=yjROBHDMbKHu0HAb0UW_3pZ-K9R20gQJTlMzKCA9dvU,14401
 osa/scripts/calibration_pipeline.py,sha256=oWX2L9WIofVHp5qYsX3YMfwoslfiUgnXtWA2EI8KuG4,6056
-osa/scripts/closer.py,sha256=vhoe9wkvPvMc5O_kXqAiBuWeqm_KrBpYzgb9zs9ZORw,21718
+osa/scripts/closer.py,sha256=lvaNH2l_fO8PvzThtJ0SDcf9i05vnc-ZfKp4xtj4s2A,21722
 osa/scripts/copy_datacheck.py,sha256=rAJu5-5oJxspynjYqVWLDKnXTF7Ib9xtzSf3_DiIHsk,3212
 osa/scripts/datasequence.py,sha256=0EZYlkX7ouaD66Ia2a5SWKbL0Fhlk8p4knKh0TG-2Gw,8592
-osa/scripts/gain_selection.py,sha256=1-wUCKuwI0W7yjja951sTXTKsGBl5YunLYcT1TwhCMQ,23240
-osa/scripts/gainsel_webmaker.py,sha256=40_DX7RUmImX-31iqRTFPEUvdVXXPSKp0xMEJWuAu80,5006
+osa/scripts/gain_selection.py,sha256=hgA1o7l8wi6SiGNiG7gQlR2kJ50c3_56ZtaN6DvZeBw,23296
+osa/scripts/gainsel_webmaker.py,sha256=nCu4x7O3xnsSIOBSQt5j5J-fxlSrBKUlb2A8hI2dWec,5253
+osa/scripts/interleaved_date.py,sha256=J5aKbPlUdx-dRhYE48SPgYv7B0AxTibUwXVuwyWY8FY,8019
 osa/scripts/provprocess.py,sha256=Zv8sHTOgGNuEzu1QPUF142VQbHyfOsrxO073-xA6KG8,19107
-osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
+osa/scripts/reprocess_longterm.py,sha256=a8gc7iiBB16T2sxwYlQrqp5Kgda-WZcKsqYq7dYC6Fc,2137
 osa/scripts/reprocessing.py,sha256=IL30e2McFA0EGbzvkTXdzmKJGBTbafYlCA-PsafyzyY,4571
 osa/scripts/sequencer.py,sha256=hqkTXuYKvHHD6-s_wbzBulb2qpMF-pTLif0UOcIXu70,13150
 osa/scripts/sequencer_catB_tailcuts.py,sha256=zVBuCBiiIoO3HKHbJ0zqaDtuwxkxcziR4T5i-2WyDX8,10476
@@ -51,7 +52,7 @@ osa/scripts/show_run_summary.py,sha256=ofznLdoFHKQB-TELD6g5zMqD_TE9M9BuJR12zHvNd
 osa/scripts/simulate_processing.py,sha256=D3b0kjyUS6Lm7pkQNJ-tJzQMveTwoPabfBBW80RMqN0,6828
 osa/scripts/update_source_catalog.py,sha256=Po4KSBOQCAT2Do1DUu5wnKV48Dq_pONfvtD47hh8fYI,8277
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=HaKVW-E2Gs6vvJoozIeCAg8-sxyu9k5zwNLFDPpCVrs,15116
+osa/scripts/tests/test_osa_scripts.py,sha256=tKcsjkV7y_YXPCqLir_gXuyAwrqLwoIJxeJ8IpobMZI,15098
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/tests/test_jobs.py,sha256=ozrMBjHujDhEMzEvJE7WynChqjbBTcORHsHmFXozeWk,16112
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
@@ -70,15 +71,15 @@ osa/utils/utils.py,sha256=a--RvA0P2JHVkuyG8uBy1BSFINLi33Ek-rNZ9jg3Fs8,13514
 osa/utils/tests/test_iofile.py,sha256=e35_EqJerp-dEOrOqwXEUZCc5P_9llf2QfveltagfIk,399
 osa/utils/tests/test_utils.py,sha256=ybngpeUyzHgr96Gcx8r2g5oHFTo3aDQIwkC5aE9Ztic,2357
 osa/webserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/webserver/utils.py,sha256=ymB2wTzsFkPH4ebUcz_zK_zyolpnBzEbWGYwCvbaHf0,2155
+osa/webserver/utils.py,sha256=NcxGQ3fDVuQvZ9d99E4LmIrY3KK49e2qJsTad6jjvGc,2135
 osa/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/workflow/dl3.py,sha256=ZMXPrdJA0WOxDiHEW9sdM2vmYix8R3eSMTGc6o42yxg,9860
 osa/workflow/stages.py,sha256=ILg91VyNXcy--si7CpDa1UFRiugqIH6nKl10Ru2zZVc,7422
 osa/workflow/tests/test_dl3.py,sha256=81Vt4lNxNjdKi_ehzymqfFWFjncw7GuQcix9S0Yigaw,384
 osa/workflow/tests/test_stages.py,sha256=aslfOIjf-rvf3r9N7NtimaOKkVB6K1M3bidgHOzRkMs,3985
-lstosa-0.11.6.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.11.6.dist-info/METADATA,sha256=vl7USgduk5P1dGlyqfRIEfmZ2w4d4lev6r4yiXn5BEo,7302
-lstosa-0.11.6.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
-lstosa-0.11.6.dist-info/entry_points.txt,sha256=CzDKpLjZZQm8jJBxOVpMR8Czpgg_Yh-k6IPETp30VZE,1048
-lstosa-0.11.6.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.11.6.dist-info/RECORD,,
+lstosa-0.11.7.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.11.7.dist-info/METADATA,sha256=NPpGT5dYeuo0baO6qjNuzqE0PCLzFdpdBa4OsqkREgk,7302
+lstosa-0.11.7.dist-info/WHEEL,sha256=WnJ8fYhv8N4SYVK2lLYNI6N0kVATA7b0piVUNvqIIJE,91
+lstosa-0.11.7.dist-info/entry_points.txt,sha256=CzDKpLjZZQm8jJBxOVpMR8Czpgg_Yh-k6IPETp30VZE,1048
+lstosa-0.11.7.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.11.7.dist-info/RECORD,,

{lstosa-0.11.6.dist-info → lstosa-0.11.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.2)
+Generator: setuptools (75.3.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

osa/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.11.6'
-__version_tuple__ = version_tuple = (0, 11, 6)
+__version__ = version = '0.11.7'
+__version_tuple__ = version_tuple = (0, 11, 7)
 
 __commit_id__ = commit_id = None

osa/configs/sequencer.cfg

@@ -26,11 +26,12 @@ DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
+DATACHECK_DIR: %(DL1_DIR)s/datacheck_files
 RF_MODELS: %(BASE)s/models/AllSky
 OSA_DIR: %(BASE)s/OSA
 CLOSER_DIR: %(OSA_DIR)s/Closer
 HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
-LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
+LONGTERM_DIR: %(DATACHECK_DIR)s/night_wise
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
 SOURCE_CATALOG: %(OSA_DIR)s/Catalog
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb

osa/conftest.py

@@ -511,15 +511,15 @@ def datacheck_dl1_files(base_test_dir):
 
 
 @pytest.fixture(scope="session")
-def longterm_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date)
+def longterm_dir(datacheck_dir):
+    directory = datacheck_dir / prod_id / date_to_dir(date)
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
 
 @pytest.fixture(scope="session")
-def longterm_link_latest_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all"
+def datacheck_dir(base_test_dir):
+    directory = base_test_dir / "DL1" / "datacheck_files" / "night_wise"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

osa/paths.py

@@ -388,18 +388,66 @@ def is_job_completed(job_id: str):
     return False
 
 
-def create_longterm_symlink(cherenkov_job_id: str = None):
+def create_longterm_symlink():
     """If the created longterm DL1 datacheck file corresponds to the latest 
     version available, make symlink to it in the "all" common directory."""
-    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
-        nightdir = utils.date_to_dir(options.date)
-        longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
-        linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
-        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+    
+    nightdir = utils.date_to_dir(options.date)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR"))
+    
+    extensions = ["h5", "log", "html"]
+    
+    for ext in extensions:
+        linked_longterm_file = output_dir / f"night_wise/DL1_datacheck_{nightdir}.{ext}"
+        all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.{ext}")
         latest_version_file = get_latest_version_file(all_longterm_files)
-        log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
         linked_longterm_file.unlink(missing_ok=True)
         linked_longterm_file.symlink_to(latest_version_file)
+
+
+def create_runwise_datacheck_symlinks():
+    """Create symlinks of the run-wise datacheck files in the "datacheck" directory."""
+    nightdir = utils.date_to_dir(options.date)
+    dl1_dir = Path(cfg.get("LST1", "DL1_DIR")) / nightdir / options.prod_id
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR")) / nightdir
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    patterns = [
+        "tailcut*/datacheck/datacheck_dl1_LST-1.Run?????.pdf",
+        "tailcut*/datacheck/datacheck_dl1_LST-1.Run?????.h5",
+    ]
+
+    for pattern in patterns:
+        for input_file in dl1_dir.rglob(pattern):
+            output_file = output_dir / input_file.name
+            if not output_file.is_symlink():
+                output_file.symlink_to(input_file.resolve())
+
+
+def create_muons_symlinks():
+    """Create symlinks of the muon files in the "datacheck" directory."""
+    nightdir = utils.date_to_dir(options.date)
+    muons_dir = destination_dir("MUON", create_dir=False)
+    muons_file_list = muons_dir.rglob("muons_LST-1*.fits")
+    output_dir = Path(cfg.get("LST1", "DATACHECK_DIR")) / nightdir / "muons"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    for input_file in muons_file_list:
+        output_file = output_dir / input_file.name
+        if not output_file.is_symlink():
+            print(f"input file exists: {input_file.exists()}")
+            output_file.symlink_to(input_file.resolve())
+
+
+def create_datacheck_symlinks(cherenkov_job_id: str=None):
+    """Once all steps of autocloser have finished successfully, create symlinks of the run-wise
+    and night-wise datacheck files, and of the muon files in the "datacheck" directory."""
+    if not cherenkov_job_id or is_job_completed(cherenkov_job_id):
+        log.info("Creating symlinks of the datacheck and muon files in the common directory.")
+        create_longterm_symlink()
+        create_runwise_datacheck_symlinks()
+        create_muons_symlinks()
     else:
         log.warning(f"Job {cherenkov_job_id} (lstchain_cherenkov_transparency) did not finish successfully.")
 

osa/scripts/closer.py

@@ -26,7 +26,7 @@ from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
 from osa.paths import (
     destination_dir,
-    create_longterm_symlink,
+    create_datacheck_symlinks,
     dl1_datacheck_longterm_file_exits
 )
 from osa.raw import is_raw_data_available
@@ -161,7 +161,7 @@ def post_process(seq_tuple):
     
     if dl1_datacheck_longterm_file_exits() and not options.test:
         if cfg.getboolean("lstchain", "create_longterm_symlink"):
-            create_longterm_symlink()
+            create_datacheck_symlinks()
 
     else:
         # Close the sequences
@@ -187,7 +187,7 @@ def post_process(seq_tuple):
             longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
             cherenkov_job_id = cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
             if cfg.getboolean("lstchain", "create_longterm_symlink"):
-                create_longterm_symlink(cherenkov_job_id)
+                create_datacheck_symlinks(cherenkov_job_id)
 
         time.sleep(600)
 
@@ -584,7 +584,7 @@ def merge_muon_files(sequence_list):
 def daily_longterm_cmd(parent_job_ids: List[str]) -> List[str]:
     """Build the daily longterm command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut*")
     muons_dir = destination_dir("MUON", create_dir=False)
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_output_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
@@ -629,7 +629,7 @@ def daily_datacheck(cmd: List[str]):
 def cherenkov_transparency_cmd(longterm_job_id: str) -> List[str]:
     """Build the cherenkov transparency command."""
     nightdir = date_to_dir(options.date)
-    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut84")
+    datacheck_dir = destination_dir("DATACHECK", create_dir=False, dl1_prod_id="tailcut*")
     longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir
     longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5"
     slurm_account = cfg.get("SLURM", "ACCOUNT")

osa/scripts/gain_selection.py

@@ -334,21 +334,23 @@ def update_history_file(run_id: str, subrun: str, log_dir: Path, history_file: P
         log.debug(f"Cannot find a job_id for the run {run_id:05d}.{subrun:04d}")
     else:
         job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
         if job_status.item() in ["RUNNING", "PENDING"]:
             log.info(f"Job {job_id} is still running.")
             return
-            
+
         elif job_status.item() == "COMPLETED":
             log.debug(f"Job {job_id} finished successfully, updating history file.")
             string_to_write = (
-                f"{run_id:05d}.{subrun:04d} gain_selection 0\n"
+                f"{now} | {run_id:05d}.{subrun:04d} gain_selection 0\n"
             )
             append_to_file(history_file, string_to_write)
-        
+
         else:
             log.info(f"Job {job_id} failed, updating history file.")
             string_to_write = (
-                f"{run_id:05d}.{subrun:04d} gain_selection 1\n"
+                f"{now} | {run_id:05d}.{subrun:04d} gain_selection 1\n"
             )
             append_to_file(history_file, string_to_write)
 

osa/scripts/gainsel_webmaker.py

@@ -144,6 +144,10 @@ def main():
         content = "<p>No data found</p>"
         log.warning(f"No data found for date {date}, creating an empty HTML file.")
 
+    elif len(Table.read(run_summary_file)[Table.read(run_summary_file)["run_type"] == "DATA"]) == 0:
+        content = "<p>Only calibration events were taken</p>"
+        log.warning(f"No DATA runs for date {date}, creating an empty HTML file.")
+
     else:
         # Get the table with the gain selection check report in HTML format:
         table_gain_selection_jobs = check_failed_jobs(options.date)

osa/scripts/interleaved_date.py

@@ -0,0 +1,212 @@
+"""
+Remove interleaved directories for data runs that have already been processed.
+
+Given an input date (<YYYYMMDD>), the script searches for interleaved directories
+within the last month. For each date in that period, it retrieves the runs taken
+and identifies the corresponding observed sources. 
+It creates a shell file to remove all of the interleaved files identified
+(those that do not correspond to Crab observations). Then, it have to be removed
+using SLURM command sbatch. 
+
+At the moment, this shell files to be removed are saved in my workspace:
+fefs/aswg/workspace/maria.rivero/remove_sh
+"""
+
+import os
+import sys
+import csv
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+import glob
+
+# Data directories to look for links of interleaved directories
+base_dirs = [
+    "/fefs/onsite/data/lst-pipe/LSTN-01/running_analysis",
+    "/fefs/aswg/data/real/running_analysis"
+]
+
+# Directory of RunSummary files
+summary_dir = "/fefs/aswg/data/real/monitoring/RunSummary"
+backup_summary_dir = "/fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunSummary"
+
+# Directories to check run's sources in RunCatalogs
+catalog_dir = "/fefs/aswg/data/real/monitoring/RunCatalog"
+backup_catalog_dir = "/fefs/onsite/data/lst-pipe/LSTN-01/monitoring/RunCatalog"
+
+
+def find_interleaved(target_date_str):
+    """Look into base directories to find interleaved directories to be removed.
+    Args:
+        target_date_str (str): Last date to search within a month.  
+    Returns: 
+        interleaved_paths (list): Paths to interleaved directory.
+        interleaved_dates (list): Observation dates of each interleaved.
+    """
+
+    # Change date format (YYYYMMDD to python date)
+    try:
+        target_date = datetime.strptime(target_date_str, "%Y%m%d").date()
+    except ValueError:
+        print("Invalid format. Use YYYYMMDD")
+        sys.exit(1)
+
+    interleaved_paths = []
+    interleaved_dates = []
+    # Let's look into both dirs and check if both exist
+    for base_dir in base_dirs:
+        if not os.path.isdir(base_dir):
+            print(f"Path not found: {base_dir}")
+            continue
+        # Look in each date directory
+        for date_dir in sorted(os.listdir(base_dir)):
+            date_path = os.path.join(base_dir, date_dir) # given date path
+            if not os.path.isdir(date_path):
+                continue
+
+            if not (len(date_dir) == 8 and date_dir.isdigit()):
+                continue
+
+            try:
+                date_obj = datetime.strptime(date_dir, "%Y%m%d").date() #save it as date python object
+            except ValueError:
+                continue
+            # search only in the last month to the input date
+            if date_obj > target_date or date_obj < (target_date - relativedelta(months=1)):
+                continue
+
+            # look for interleaved directory (save path and date)
+            for root, dirs, _ in os.walk(date_path):
+                for d in dirs:
+                    if d == "interleaved":
+                        interleaved_paths.append(os.path.join(root, d))
+                        interleaved_dates.append(date_dir)
+    return interleaved_paths, interleaved_dates
+
+def info_dates(date, catalog_dir, runs_id):
+    """Given an observation date, it classifies runs in RunCatalog by their sources (Crab or not).
+    Args:
+        date (str): <YYYYMMDD> format.
+        catalog_dir (str): path to RunCatalog files.
+        runs_id (list): DATA runs taken from RunSummary
+    Returns:
+        entry (dict): stores runs by source (Crab or other) and saves other sources' names.   
+    """
+
+    entry = {"crab": [], "other_source": [], "others_names": []}
+    filename = f"RunCatalog_{date}.ecsv"
+    catalog_file = os.path.join(catalog_dir,filename)
+    if not os.path.isfile(catalog_file):
+        print(f"File not found: {catalog_file}")
+        return entry
+
+    else:
+        with open(catalog_file, "r") as f:
+            reader = csv.reader(f)
+            for row in reader:
+                if not row or row[0].startswith("#") or row[0].startswith("run"):
+                    continue # skip comment and header lines
+                if int(row[0]) in runs_id:
+                    if "Crab" in row[1]:
+                        entry["crab"].append(int(row[0]))
+                    elif "crab" in row[1]:
+                        entry["crab"].append(int(row[0]))
+                    else:
+                        entry["other_source"].append(int(row[0]))
+                        entry["others_names"].append(row[1])
+                else:
+                    continue
+        return(entry)
+
+
+def summary_dates(date, summary_dir):
+    """Given an observation date, it stores run_ids and types.
+    Args:
+        date (str): <YYYYMMDD> format.
+        summary_dir (str): path to RunSummary files.
+    Returns:
+        entry (dict): stores run_id and run_type.
+    """
+
+    entry = {"run_id": [], "run_type": []}
+    filename = f"RunSummary_{date}.ecsv"
+    summary_file = os.path.join(summary_dir, filename)
+    if not os.path.isfile(summary_file):
+         print(f"File not found: {summary_file}")
+         return entry
+    else:
+        with open(summary_file, "r") as f:
+            reader = csv.reader(f)
+            for row in reader:
+                if not row or row[0].startswith("#"):
+                    continue # skip comment lines
+                try:
+                    entry["run_id"].append(int(row[0]))
+                    entry["run_type"].append(row[2])
+
+                except ValueError:
+                    continue
+        return(entry)
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Use: python interleaved_date.py <YYYYMMDD>")
+        sys.exit(1)
+
+    date_arg = sys.argv[1]
+    found_paths, found_dates = find_interleaved(date_arg)
+    month = date_arg[:6]
+    recordfile = f'/fefs/aswg/workspace/maria.rivero/remove_sh/entries_rm{month}.sh'
+    with open(recordfile, 'w') as file:
+        file.write('#!/bin/bash \n')
+
+    dl1_paths = [
+        p.replace("/running_analysis/", "/DL1/")
+        for p in found_paths
+    ]
+    print('Interleaved path: ' , dl1_paths)
+
+    for path,link_path, date in zip(dl1_paths, found_paths, found_dates):
+        summary = summary_dates(date, summary_dir)
+        if not summary['run_id']:
+            summary = summary_dates(date, backup_summary_dir)
+        if not summary['run_id']:
+            continue
+
+        data_runs = [
+            run_id
+            for run_id, run_type in zip(summary["run_id"], summary["run_type"])
+            if run_type == "DATA"]
+
+        entry = info_dates(date, catalog_dir, data_runs)
+        if entry["crab"] == [] and entry["other_source"] == []:
+            entry = info_dates(date, backup_catalog_dir, data_runs)
+        if not entry["other_source"]:
+            continue
+
+        print('\n Dates with interleaved: ' , date)
+
+        print('RunSummary info:')
+        print(summary)
+        print('Run info (Crab or not):')
+        print(entry)
+
+        found_dataruns = sorted(entry["crab"] + entry["other_source"])
+        if not entry["crab"] and len(found_dataruns) == len(data_runs):
+            print(f"rm -r {path}")
+            with open(recordfile, 'a') as file:
+                file.write(f"rm -r {path} \n")
+                file.write(f"rm -r {link_path} \n")
+        else:
+            for runid in entry["other_source"]:
+                run_str = f"{runid:05d}" # run_ids must be always five digits
+                filename = f"interleaved_LST-1.Run{run_str}.*.h5"
+                filepath = os.path.join(path, filename)
+                link_filepath = os.path.join(link_path, filename)
+                matching_files = glob.glob(filepath) # check that exist files with run_id
+                if matching_files:
+                    print(f"rm {filepath}")
+                    with open(recordfile, 'a') as file:
+                        file.write(f"rm {filepath} \n")
+                        file.write(f"rm {link_filepath} \n")
+                else:
+                    continue

osa/scripts/reprocess_longterm.py

@@ -30,7 +30,7 @@ def run_longterm(date: str, prod_id: str, new_prod_id: str, log_dir: Path):
     log_dir : Path
         Path to the directory where job logs will be stored.
     """
-    dl1_dir = ANALYSIS_PATH / "DL1" / date / prod_id / "tailcut84" / "datacheck"
+    dl1_dir = ANALYSIS_PATH / "DL1" / date / prod_id / "tailcut*" / "datacheck"
     muons_dir = ANALYSIS_PATH / "DL1" / date / prod_id / "muons"
     new_longterm_dir = LONGTERM_PATH / new_prod_id / date
     longterm_output_file = new_longterm_dir / f"DL1_datacheck_{date}.h5"

osa/scripts/tests/test_osa_scripts.py

@@ -198,7 +198,7 @@ def test_closer(
     systematic_correction_files,
     merged_run_summary,
     longterm_dir,
-    longterm_link_latest_dir,
+    datacheck_dir,
     daily_datacheck_dl1_files,
     dl1b_config_files,
     tailcuts_log_files,
@@ -223,7 +223,7 @@ def test_closer(
         assert obs_file.exists()
     assert merged_run_summary.exists()
     assert longterm_dir.exists()
-    assert longterm_link_latest_dir.exists()
+    assert datacheck_dir.exists()
     for check_file in daily_datacheck_dl1_files:
         assert check_file.exists()
     assert rf_models[2].exists()
@@ -389,8 +389,8 @@ def test_daily_longterm_cmd():
         "log/longterm_daily_%j.log",
         "--dependency=afterok:12345,54321",
         "lstchain_longterm_dl1_check",
-        "--input-dir=test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/datacheck",
-        "--output-file=test_osa/test_files0/OSA/DL1DataCheck_LongTerm/v0.1.0/20200117/DL1_datacheck_20200117.h5",
+        "--input-dir=test_osa/test_files0/DL1/20200117/v0.1.0/tailcut*/datacheck",
+        "--output-file=test_osa/test_files0/DL1/datacheck_files/night_wise/v0.1.0/20200117/DL1_datacheck_20200117.h5",
         "--muons-dir=test_osa/test_files0/DL1/20200117/v0.1.0/muons",
         "--batch",
     ]

osa/webserver/utils.py

@@ -37,8 +37,8 @@ def directory_in_webserver(host: str, datacheck_type: str, date: str, prod_id: s
     DATACHECK_WEB_DIRS = {
         "PEDESTAL": f"drs4/{prod_id}/{date}",
         "CALIB": f"enf_calibration/{prod_id}/{date}",
-        "DL1AB": f"dl1/{prod_id}/{date}/pdf",
-        "LONGTERM": f"dl1/{prod_id}/{date}",
+        "DL1AB": f"dl1/{date}/pdf",
+        "LONGTERM": f"dl1/{date}",
         "HIGH_LEVEL": f"high_level/{prod_id}/{date}",
     }