lstosa 0.10.4__py3-none-any.whl → 0.10.6__py3-none-any.whl

{lstosa-0.10.4.dist-info → lstosa-0.10.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.4
+Version: 0.10.6
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -54,20 +54,20 @@ Requires-Dist: freezegun ; extra == 'test'
   [![ci](https://github.com/cta-observatory/lstosa/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/cta-observatory/lstosa/actions/workflows/ci.yml)
   [![Documentation Status](https://readthedocs.org/projects/lstosa/badge/?version=latest)](https://lstosa.readthedocs.io/en/latest/?badge=latest)
   [![coverage](https://codecov.io/gh/cta-observatory/lstosa/branch/main/graph/badge.svg?token=Zjk1U1ytaG)](https://codecov.io/gh/cta-observatory/lstosa)
-  [![quality](https://app.codacy.com/project/badge/Grade/a8743a706e7c45fc989d5ebc4d61d54f)](https://www.codacy.com/gh/cta-observatory/lstosa/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=cta-observatory/lstosa&amp;utm_campaign=Badge_Grade)
+  [![Codacy Badge](https://app.codacy.com/project/badge/Grade/a8743a706e7c45fc989d5ebc4d61d54f)](https://app.codacy.com/gh/cta-observatory/lstosa/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
   [![pypi](https://img.shields.io/pypi/v/lstosa)](https://pypi.org/project/lstosa/)
   [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.6567234.svg)](https://doi.org/10.5281/zenodo.6567234)
 
 
-Prototype onsite processing pipeline for the Large Size Telescope prototype (LST-1) of [CTA](https://www.cta-observatory.org/) (Cherenkov Telescope Array) based on [cta-lstchain](https://github.com/cta-observatory/cta-lstchain) running on the LST-1 IT onsite center at ORM (La Palma, Spain). It automatically carries out the next-day analysis of observed data using cron jobs, parallelizing the processing using the job scheduler SLURM. It provides data quality monitoring and tracking of analysis products' provenance.
-Moreover, it also massively reprocesses the entire LST-1 dataset with each cta-lstchain major release:
+Onsite processing pipeline for the Large-Sized Telescope prototype (LST-1) of [CTAO](https://www.cta-observatory.org/) (Cherenkov Telescope Array Observatory) based on [cta-lstchain](https://github.com/cta-observatory/cta-lstchain) running on the LST-1 IT onsite data center at Observatorio Roque de los Muchachos (La Palma, Spain). It automatically carries out the next-day analysis of observed data using cron jobs, parallelizing the processing using the job scheduler SLURM. It provides data quality monitoring and tracking of analysis products' provenance. Moreover, it also massively reprocesses the entire LST-1 dataset with each cta-lstchain major release.
+
  - Code: <https://github.com/cta-observatory/lstosa>
  - Docs: <https://lstosa.readthedocs.io/>
  - License: [BSD-3-Clause](https://github.com/cta-observatory/lstosa/blob/main/LICENSE)
 
 ## Install
 We recommend using an isolated conda environment.
- - Install miniconda first.
+ - Install mamba/miniconda first.
  - Clone the repository, create and activate the conda environment using the `environment.yml` file:
 
     ```bash
@@ -77,7 +77,7 @@ We recommend using an isolated conda environment.
     conda activate osa
     ```
 
-Then install `lstosa` as a **user** with: `pip install lstosa`, or as a **developer** with: `pip install -e .`.
+Then install `lstosa` as a **user** with: `pip install lstosa`, or as a **developer** with: `pip install -e .`. To install test, docs dependencies use `pip install -e .[test]`, `pip install -e .[doc]` or simply `pip install -e .[all]`
 
 In case you want to install the lstchain development version instead of a fixed tag, you can run inside the `osa` environment:
 

{lstosa-0.10.4.dist-info → lstosa-0.10.6.dist-info}/RECORD

@@ -1,9 +1,9 @@
 osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
-osa/_version.py,sha256=HXKqJR44DhKjW27JxX9-utoVrog1s0wcE7R-Ggh4Ats,413
-osa/conftest.py,sha256=7U1hVv-sCPfAQ27UR__Ij--F-AU51hMOe7rsT1-7CJE,19026
+osa/_version.py,sha256=0ek8_w_jF04UB6FpW1K7d1RgH0zQMd77H5TbKzdthjI,413
+osa/conftest.py,sha256=b_26FciV8NZAdHj2S10E8cE4bettinLJY83Roe1a5L4,19358
 osa/job.py,sha256=acdiaRlHsc2GLyz2Oz_RBiawJo9QKK-tsajYdM7X0_Q,25393
 osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
-osa/paths.py,sha256=P8ArRxdOghiGufYrcuJJmla0OP5YQjseNxpbOig1Qpg,9998
+osa/paths.py,sha256=nkXPUdQAzIlvB71lOERiPjgET58fj5DEbWazWqKbIJw,11869
 osa/raw.py,sha256=ZNIsuqfx5ljoz_hwhSuafdKf-wr8-cxRJmel-A2endg,1337
 osa/report.py,sha256=sL2V7n8Y_UUaSDbWJY2o4UxDb4FU5AaFIRR8R25DB8o,4634
 osa/version.py,sha256=9T2TtuGBQeOy5PJDxMCeGlqx5baxLaq47VmFTDc09z8,796
@@ -11,8 +11,8 @@ osa/veto.py,sha256=HMaojHSRT0HG0FBGG38su3xz2_ejsVKU1QktAzx28aA,3090
 osa/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/configs/config.py,sha256=cX0Vr4sorBVQ2KRqPIRSEGENKH_uC5lrgVHx1hp6YTk,1148
 osa/configs/datamodel.py,sha256=L_WRM91PBlMrtuE30akh7YR-56P0g9D994qzKSfhNJc,1950
-osa/configs/options.py,sha256=q8NGEZiLkwulogp35vhK8mHcYN1AZBVUY8CwnOrrOF4,545
-osa/configs/sequencer.cfg,sha256=BY5GqemJsg1CJUynRWnJQjwEU2ktIWQIAcSx0plBoM8,4803
+osa/configs/options.py,sha256=CyL7WnHiC_pvB3mnjRF7Wg43uPzQgmwlbvIqkRzlDLA,524
+osa/configs/sequencer.cfg,sha256=cysPy6q8vShh0V0LiEBfip-rUmwyNlNVDKhGPMiKH5g,4784
 osa/high_level/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/high_level/selection_cuts.toml,sha256=ReSmcKtOPZY5JsZ9ExnxYdz7OrJEB8gghCbzHmeOyFg,128
 osa/high_level/significance.py,sha256=Y1jokkHCo-D_qSqxKiQzc6KJSmivznaJRS2xY-txNIo,9039
@@ -29,18 +29,18 @@ osa/nightsummary/tests/test_source_coordinates.py,sha256=doyF2e93RnFScPbG79aUmtH
 osa/provenance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/provenance/capture.py,sha256=PFN1SDU1HN4yW5Y7V5PshuS_IdoioktXCpKYWXzjH18,22348
 osa/provenance/io.py,sha256=8VAVKmE2S3YHcxwDC8PJ4hO6ogfz08EflAeprLGkp3Y,9471
-osa/provenance/utils.py,sha256=3LZ1dcdMShXBdhJTT1-nt27XKediYXHKQwh_XsyqLFc,8884
+osa/provenance/utils.py,sha256=yohpB1XHMY9iZQGxw8rEVk_yzv77UeYWEit5GgRvUjU,8738
 osa/provenance/config/definition.yaml,sha256=DSwqNln1jEXV8aUh7ca2r7ArMkpaMJi1xbHEz0hUcao,20538
 osa/provenance/config/environment.yaml,sha256=kPM6ucPyLZLDFzkwFWoY0C6vmAArG98U-P1UAl89bgE,246
 osa/provenance/config/logger.yaml,sha256=hy_lH3DfbRFh2VM_iawI-c-3wE0cjTRHy465C2eFfnQ,510
 osa/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/scripts/autocloser.py,sha256=j7nRvglIv_Ol_GJzbcfqPHquBIWwQU515GkDRfc6_ho,14540
-osa/scripts/calibration_pipeline.py,sha256=PySKtspCATdykaXclzeTkdwwvKm1CpWGlaVz44vyiCY,5625
-osa/scripts/closer.py,sha256=n0GRxQ6DHYykyL4FvT24Qar80-FbX6b0ymJOEez-r3c,15091
+osa/scripts/calibration_pipeline.py,sha256=g9o1chqCRRSo7GNuQZRABjGnJYjZyfhkPgRLPmuV994,5703
+osa/scripts/closer.py,sha256=eZLUlqot4EXwL9IlU21tKRZ2GvZ4i8ill0c2QdEkoDY,15219
 osa/scripts/copy_datacheck.py,sha256=tfDs6oTdPbii4BOXp6bTHuED0xNJeqaPFrv6Ed7ZnWc,3104
 osa/scripts/datasequence.py,sha256=gXAp8arbLPEK-sca9VnME6-2XfUzBFIoEFchlUZYrXI,9260
-osa/scripts/gain_selection.py,sha256=rHPqJEk9rvKRe13whOPOZEKCSF54OG0BvYyqlcBPqNQ,7776
-osa/scripts/provprocess.py,sha256=02vJmQvl7W3e4oAzzzSwOVo9okXLCwYPNqe6jKeaoLA,18480
+osa/scripts/gain_selection.py,sha256=BmNjtfeCFw5OGymaLG1PFPDcn8-yDOG6fWaifOCOHLM,8095
+osa/scripts/provprocess.py,sha256=pKR3wf3krAGmeHen_au2RmqJJtNBcX29J3XPQYKrHRE,18536
 osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
 osa/scripts/reprocessing.py,sha256=D-J8Rl3GrkWpxYkk6ci79oJOMewgGdxLkQgaHCAZuqs,3417
 osa/scripts/sequencer.py,sha256=6Cg-eIExk0eN8-HXkO6DBeFEWOI-FznDueqJEpbEzos,8163
@@ -50,34 +50,34 @@ osa/scripts/show_run_summary_tcu.py,sha256=SoDLVKdQHOJkfenFguBOfXf10Gyv7heXSQAFn
 osa/scripts/simulate_processing.py,sha256=NiRVYiwZENt_mnKncytgJT23_-tJMb1B5PswM12nnX4,6941
 osa/scripts/update_source_catalog.py,sha256=lJXYWI92YB0SrafsND4do0kjPOV_mP2MzuyMfBJbY-0,6265
 osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/scripts/tests/test_osa_scripts.py,sha256=NwUo6l4V_srnfDHJEn5hO57te0Xa47G7tF7UDw42KY4,12488
+osa/scripts/tests/test_osa_scripts.py,sha256=UiUTXw5uK2iMZt7thRS1UfqoAY5b9s8k8TB143BnXcw,12587
 osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/tests/test_jobs.py,sha256=F0jsHZ9BYB_cCHHxlXyO9v1E5_-mBJhuFtshtsAtnXo,15260
 osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
 osa/tests/test_osadb.py,sha256=pJHV1dxxblGH2sjS-JPDPTkMn-ew1MzbioCFyg7wbB8,1599
-osa/tests/test_paths.py,sha256=fH169s05DJTYPvOfzuHwx-37Z65TRueoyCSzssuBllc,3161
+osa/tests/test_paths.py,sha256=sFCxG5uPGLcto76E7X1I26-kRx5faxgHGh9z8LvHz2M,3173
 osa/tests/test_raw.py,sha256=WkgwEc_vY0D6nREo-BSm6F-5xDpqidMC0DkS86pXlRU,1058
 osa/tests/test_report.py,sha256=OY-EsrXytoS6esfjUeLnIAmCMIw9EzoGD-elySafyhE,1365
 osa/tests/test_veto.py,sha256=UIsooji_5Z8TtAhc0UlD2VqheVd9DBufuMxinJ3e0w8,1066
 osa/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-osa/utils/cliopts.py,sha256=9IAEhFK-RO11C3EvJ-2gch9RYBZKj5zQQRfPepY4fng,13716
+osa/utils/cliopts.py,sha256=Xqh-9w2o1iSHfwO0jHEW0AXQ9VeEw6X6IGLNAPKoNyc,13699
 osa/utils/iofile.py,sha256=kJ7KB1suynhS2cTf7EeHwhMXq3delC_ls2HFpCzvsZo,2021
 osa/utils/logging.py,sha256=1WcNPjjslo3y25jcEY_fe0yXOeJ6frZrGLAy1GJpu_k,1491
 osa/utils/mail.py,sha256=uQfqPQdiOVHTvEAXr9H15a7-g9DtYVNKjMEb9GnI0oY,554
 osa/utils/register.py,sha256=bQdZeawUSoe1xdaWQ2rp7itZSlzLMuilQiRTywbcdBc,6202
-osa/utils/utils.py,sha256=5Yxe4LLt042-63IyPliMEZxCV_73qv0XNUZWYYuLGcI,7959
+osa/utils/utils.py,sha256=KYNOln6t3J8DzZ31YjmK6TnGhHI9n3doptQTvERrUpI,7341
 osa/utils/tests/test_iofile.py,sha256=e35_EqJerp-dEOrOqwXEUZCc5P_9llf2QfveltagfIk,399
-osa/utils/tests/test_utils.py,sha256=ekD1p_lN85iqSZSFt6JVAlqDPI0-G5RoX7pA76qmmfE,2139
+osa/utils/tests/test_utils.py,sha256=4S4OESdF3wHPOMA5A_B6Zp_O4FZKizL581SmlMUVm2I,1959
 osa/webserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/webserver/utils.py,sha256=ymB2wTzsFkPH4ebUcz_zK_zyolpnBzEbWGYwCvbaHf0,2155
 osa/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 osa/workflow/dl3.py,sha256=kz7L5jcKHFJ--UdQ8HQKLzWO6nxc2LLOTz42ExcqzTk,9921
-osa/workflow/stages.py,sha256=CqyBSt3sre7dd5izKvcW4Bi9AD7GQTJS8jKHKW6SQZM,6244
+osa/workflow/stages.py,sha256=WYgUM2XDIaUjCc4_Zs_VSGW6gk73EaKcHk6ZMnPds74,6692
 osa/workflow/tests/test_dl3.py,sha256=aY5bb-8OcZGAXG3JPCZihChzkA_GsWjRIa31BHZn3Dg,299
 osa/workflow/tests/test_stages.py,sha256=TmC00XFACWZp740TQeFaokWi3C50ovj_XGiySWrrdZk,3944
-lstosa-0.10.4.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
-lstosa-0.10.4.dist-info/METADATA,sha256=9XiE_4H5kKI4-1GDzgUocaCQ5i8c8EtfdfcetCiz9Mk,7220
-lstosa-0.10.4.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-lstosa-0.10.4.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
-lstosa-0.10.4.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
-lstosa-0.10.4.dist-info/RECORD,,
+lstosa-0.10.6.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.10.6.dist-info/METADATA,sha256=Z1GxQGKizr5McPZEq-mhPLuYRmNjzLV8teV87W62kdA,7354
+lstosa-0.10.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+lstosa-0.10.6.dist-info/entry_points.txt,sha256=e5x7xddaqZhfdZPsErhHInqR4UGHsxXIlylEbTie0_8,928
+lstosa-0.10.6.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.10.6.dist-info/RECORD,,

{lstosa-0.10.4.dist-info → lstosa-0.10.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

osa/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.4'
-__version_tuple__ = version_tuple = (0, 10, 4)
+__version__ = version = '0.10.6'
+__version_tuple__ = version_tuple = (0, 10, 6)

osa/configs/options.py

@@ -16,7 +16,6 @@ warning = None
 nocheck = None
 no_dl2 = None
 prod_id = None
-calib_prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None
 append = None

osa/configs/sequencer.cfg

@@ -32,7 +32,6 @@ SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
 PROD_ID: v0.1.0
-CALIB_PROD_ID: v01
 # Change this to produce a different DL1b or DL2 sub-productions.
 # Otherwise, keep it empty to use the common PROD-ID
 DL1_PROD_ID: tailcut84

osa/conftest.py

@@ -26,6 +26,7 @@ from osa.nightsummary.nightsummary import run_summary_table
 from osa.scripts.tests.test_osa_scripts import run_program
 from osa.utils.utils import date_to_dir
 from datetime import datetime
+import lstchain
 
 date = datetime.fromisoformat("2020-01-17")
 nightdir = date_to_dir(date)
@@ -68,16 +69,25 @@ def calibration_base_dir(monitoring_dir):
     return base_dir
 
 
+@pytest.fixture(scope="session")
+def drive_log(monitoring_dir):
+    drive_dir = monitoring_dir / "DrivePositioning"
+    drive_file = drive_dir / "DrivePosition_log_20200117.txt"
+    drive_dir.mkdir(parents=True, exist_ok=True)
+    drive_file.touch()
+    return drive_file
+
+
 @pytest.fixture(scope="session")
 def calibration_dir(calibration_base_dir):
-    directory = calibration_base_dir / "calibration" / nightdir / "pro"
+    directory = calibration_base_dir / "calibration" / nightdir / f"v{lstchain.__version__}"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
 
 @pytest.fixture(scope="session")
 def drs4_baseline_dir(calibration_base_dir):
-    directory = calibration_base_dir / "drs4_baseline" / nightdir / "pro"
+    directory = calibration_base_dir / "drs4_baseline" / nightdir / f"v{lstchain.__version__}"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

osa/paths.py

@@ -1,17 +1,19 @@
 """Handle the paths of the analysis products."""
 
 import logging
+import re
+from datetime import datetime
 from pathlib import Path
 from typing import List
-from datetime import datetime
 
+import lstchain
 from astropy.table import Table
-from lstchain.onsite import find_systematics_correction_file, find_time_calibration_file
-from lstchain.scripts.onsite.onsite_create_calibration_file import search_filter
+from lstchain.onsite import (find_systematics_correction_file,
+                             find_time_calibration_file,
+                             find_filter_wheels)
 
 from osa.configs import options
-from osa.configs.config import DEFAULT_CFG
-from osa.configs.config import cfg
+from osa.configs.config import DEFAULT_CFG, cfg
 from osa.configs.datamodel import Sequence
 from osa.utils import utils
 from osa.utils.logging import myLogger
@@ -19,8 +21,8 @@ from osa.utils.logging import myLogger
 log = myLogger(logging.getLogger(__name__))
 
 __all__ = [
-    "get_calibration_file",
-    "get_drs4_pedestal_file",
+    "get_calibration_filename",
+    "get_drs4_pedestal_filename",
     "pedestal_ids_file_exists",
     "get_run_date",
     "drs4_pedestal_exists",
@@ -40,6 +42,8 @@ __all__ = [
 
 
 DATACHECK_WEB_BASEDIR = Path(cfg.get("WEBSERVER", "DATACHECK"))
+CALIB_BASEDIR = Path(cfg.get("LST1", "CALIB_DIR"))
+DRS4_PEDESTAL_BASEDIR = Path(cfg.get("LST1", "PEDESTAL_DIR"))
 
 
 def analysis_path(tel) -> Path:
@@ -86,18 +90,25 @@ def get_run_date(run_id: int) -> datetime:
     return datetime.strptime(date_string, "%Y-%m-%d")
 
 
-def get_drs4_pedestal_file(run_id: int) -> Path:
+def get_drs4_pedestal_filename(run_id: int, prod_id: str) -> Path:
     """
     Return the drs4 pedestal file corresponding to a given run id
     regardless of the date when the run was taken.
     """
-    drs4_pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+    if drs4_pedestal_exists(run_id, prod_id):
+        files = search_drs4_files(run_id, prod_id)
+        return files[-1]  # Get the latest production among the major lstchain version
+
     date = utils.date_to_dir(get_run_date(run_id))
-    file = drs4_pedestal_dir / date / f"pro/drs4_pedestal.Run{run_id:05d}.0000.h5"
-    return file.resolve()
+    return (
+        DRS4_PEDESTAL_BASEDIR
+        / date
+        / f"v{lstchain.__version__}/drs4_pedestal.Run{run_id:05d}.0000.h5"
+    ).resolve()
 
 
-def get_calibration_file(run_id: int) -> Path:
+def get_calibration_filename(run_id: int, prod_id: str) -> Path:
+    # sourcery skip: remove-unnecessary-cast
     """
     Return the calibration file corresponding to a given run_id.
 
@@ -109,14 +120,18 @@ def get_calibration_file(run_id: int) -> Path:
     Notes
     -----
     The file path will be built regardless of the date when the run was taken.
-    We follow the naming convention of the calibration files produced by the lstchain script
-    which depends on the filter wheels position. Therefore, we need to try to fetch the filter
-    position from the CaCo database. If the filter position is not found, we assume the default
-    filter position 5-2. Filter information is not available in the database for runs taken before
+    We follow the naming convention of the calibration files produced by the
+    lstchain script which depends on the filter wheels position. Therefore, we
+    need to try to fetch the filter position from the CaCo database. If the
+    filter position is not found, we assume the default filter position 5-2.
+    Filter information is not available in the database for runs taken before
     mid 2021 approx.
     """
 
-    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
+    if calibration_file_exists(run_id, prod_id):
+        files = search_calibration_files(run_id, prod_id)
+        return files[-1]  # Get the latest production among the major lstchain version
+
     date = utils.date_to_dir(get_run_date(run_id))
 
     if options.test:  # Run tests avoiding the access to the database
@@ -126,13 +141,16 @@ def get_calibration_file(run_id: int) -> Path:
         mongodb = cfg.get("database", "caco_db")
         try:
             # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB
-            options.filters = search_filter(int(run_id), mongodb)
+            options.filters = find_filter_wheels(int(run_id), mongodb)
         except IOError:
             log.warning("No filter information found in database. Assuming positions 52.")
             options.filters = 52
 
-    file = calib_dir / date / f"pro/calibration_filters_{options.filters}.Run{run_id:05d}.0000.h5"
-    return file.resolve()
+    return (
+        CALIB_BASEDIR
+        / date
+        / f"v{lstchain.__version__}/calibration_filters_{options.filters}.Run{run_id:05d}.0000.h5"
+    ).resolve()
 
 
 def pedestal_ids_file_exists(run_id: int) -> bool:
@@ -142,16 +160,51 @@ def pedestal_ids_file_exists(run_id: int) -> bool:
     return bool(file_list)
 
 
-def drs4_pedestal_exists(run_id: int) -> bool:
+def drs4_pedestal_exists(run_id: int, prod_id: str) -> bool:
     """Return true if drs4 pedestal file was already produced."""
-    file = get_drs4_pedestal_file(run_id)
-    return file.exists()
+    files = search_drs4_files(run_id, prod_id)
+
+    return len(files) != 0
 
 
-def calibration_file_exists(run_id: int) -> bool:
+def calibration_file_exists(run_id: int, prod_id: str) -> bool:
     """Return true if calibration file was already produced."""
-    file = get_calibration_file(run_id)
-    return file.exists()
+    files = search_calibration_files(run_id, prod_id)
+
+    return len(files) != 0
+
+
+def search_drs4_files(run_id: int, prod_id: str) -> list:
+    """
+    Find DRS4 baseline correction files corresponding to a run ID
+    and major lstchain production version
+    """
+    date = utils.date_to_dir(get_run_date(run_id))
+    version = get_major_version(prod_id)
+    drs4_dir = DRS4_PEDESTAL_BASEDIR / date
+    return sorted(
+        drs4_dir.glob(f"{version}*/drs4_pedestal.Run{run_id:05d}.0000.h5")
+    )
+
+
+def get_major_version(prod_id):
+    """Given a version as vX.Y.Z return vX.Y"""
+    # First check that the given version is in the correct format
+    if prod_id.startswith("v") and len(prod_id.split(".")) >= 2:
+        return re.search(r"\D\d+\.\d+", prod_id)[0]
+
+    raise ValueError("Format of the version is not in the form vW.X.Y.Z")
+
+
+def search_calibration_files(run_id: int, prod_id: str) -> list:
+    """
+    Search charge calibration files corresponding to a run ID and major lstchain production version
+    """
+    date = utils.date_to_dir(get_run_date(run_id))
+    version = get_major_version(prod_id)
+    return sorted(
+        (CALIB_BASEDIR / date).glob(f"{version}*/calibration_filters_*.Run{run_id:05d}.0000.h5")
+    )
 
 
 def get_drive_file(date: str) -> Path:
@@ -180,11 +233,16 @@ def sequence_calibration_files(sequence_list: List[Sequence]) -> None:
     """Build names of the calibration files for each sequence in the list."""
     flat_date = utils.date_to_dir(options.date)
     base_dir = Path(cfg.get("LST1", "BASE"))
+    prod_id = options.prod_id
 
     for sequence in sequence_list:
         # Assign the calibration files to the sequence object
-        sequence.drs4_file = get_drs4_pedestal_file(sequence.drs4_run)
-        sequence.calibration_file = get_calibration_file(sequence.pedcal_run)
+        sequence.drs4_file = get_drs4_pedestal_filename(
+            sequence.drs4_run, prod_id
+        )
+        sequence.calibration_file = get_calibration_filename(
+            sequence.pedcal_run, prod_id
+        )
         sequence.time_calibration_file = find_time_calibration_file(
             "pro", sequence.pedcal_run, base_dir=base_dir
         )
@@ -235,10 +293,7 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
         directory = Path(cfg.get(options.tel_id, "DL1_DIR")) / nightdir / options.prod_id / "muons"
     elif concept == "INTERLEAVED":
         directory = (
-            Path(cfg.get(options.tel_id, "DL1_DIR"))
-            / nightdir
-            / options.prod_id
-            / "interleaved"
+            Path(cfg.get(options.tel_id, "DL1_DIR")) / nightdir / options.prod_id / "interleaved"
         )
     elif concept == "DATACHECK":
         directory = (
@@ -257,17 +312,21 @@ def destination_dir(concept: str, create_dir: bool = True) -> Path:
         )
     elif concept in {"DL2", "DL3"}:
         directory = (
-            Path(cfg.get(options.tel_id, concept + "_DIR"))
-            / nightdir
+            (Path(cfg.get(options.tel_id, f"{concept}_DIR")) / nightdir)
             / options.prod_id
-            / options.dl2_prod_id
-        )
+        ) / options.dl2_prod_id
     elif concept in {"PEDESTAL", "CALIB", "TIMECALIB"}:
         directory = (
-            Path(cfg.get(options.tel_id, concept + "_DIR")) / nightdir / options.calib_prod_id
+            Path(cfg.get(options.tel_id, f"{concept}_DIR"))
+            / nightdir
+            / options.prod_id
         )
     elif concept == "HIGH_LEVEL":
-        directory = Path(cfg.get(options.tel_id, concept + "_DIR")) / nightdir / options.prod_id
+        directory = (
+            Path(cfg.get(options.tel_id, f"{concept}_DIR"))
+            / nightdir
+            / options.prod_id
+        )
     else:
         log.warning(f"Concept {concept} not known")
         directory = None

osa/provenance/utils.py

@@ -1,7 +1,6 @@
 """Utility functions for OSA pipeline provenance."""
 
 
-import os
 from importlib.resources import files
 from pathlib import Path
 
@@ -27,10 +26,11 @@ def parse_variables(class_instance):
     # datasequence.py
     # -c sequencer.cfg
     # -d 2020_02_18
-    # --prod-id v0.4.3_v00
-    # --pedcal-file .../20200218/pro/calibration_filters_52.Run02006.0000.h5
-    # --drs4-pedestal-file .../20200218/pro/drs4_pedestal.Run02005.0000.h5
+    # --prod-id v0.4
+    # --pedcal-file .../20200218/v0.4.3/calibration_filters_52.Run02006.0000.h5
+    # --drs4-pedestal-file .../20200218/v0.4.3/drs4_pedestal.Run02005.0000.h5
     # --time-calib-file .../20191124/pro/time_calibration.Run01625.0000.h5
+    # --systematic_correction_file .../20200725/pro/ffactor_systematics_20200725.h5
     # --drive-file .../DrivePositioning/DrivePosition_20200218.txt
     # --run-summary .../monitoring/RunSummary/RunSummary_20200101.ecsv
     # 02006.0000
@@ -39,22 +39,20 @@ def parse_variables(class_instance):
     flat_date = date_to_dir(options.date)
     configfile_dl1b = cfg.get("lstchain", "dl1b_config")
     configfile_dl2 = cfg.get("lstchain", "dl2_config")
-    raw_dir = cfg.get("LST1", "R0_DIR")
-    rf_models_directory = cfg.get("lstchain", "RF_MODELS")
-    dl1_dir = cfg.get("LST1", "DL1_DIR")
-    dl2_dir = cfg.get("LST1", "DL2_DIR")
-    calib_dir = cfg.get("LST1", "CALIB_DIR")
-    pedestal_dir = cfg.get("LST1", "PEDESTAL_DIR")
-    # summary_dir = cfg.get("LST1", "RUN_SUMMARY_DIR")
-    # calib_base_dir = cfg.get("LST1", "CALIB_BASE_DIR")
-    # sys_dir = calib_base_dir / "ffactor_systematics"
+    raw_dir = Path(cfg.get("LST1", "R0_DIR"))
+    rf_models_directory = Path(cfg.get("lstchain", "RF_MODELS"))
+    dl1_dir = Path(cfg.get("LST1", "DL1_DIR"))
+    dl2_dir = Path(cfg.get("LST1", "DL2_DIR"))
+    calib_dir = Path(cfg.get("LST1", "CALIB_DIR"))
+    pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+
     class_instance.SoftwareVersion = get_lstchain_version()
-    class_instance.ProcessingConfigFile = options.configfile
+    class_instance.ProcessingConfigFile = str(options.configfile)
     class_instance.ObservationDate = flat_date
     if class_instance.__name__ in REDUCTION_TASKS:
-        muon_dir = Path(dl1_dir) / flat_date / options.prod_id
-        outdir_dl1 = Path(dl1_dir) / flat_date / options.prod_id / options.dl1_prod_id
-        outdir_dl2 = Path(dl2_dir) / flat_date / options.prod_id / options.dl2_prod_id
+        muon_dir = dl1_dir / flat_date / options.prod_id / "muons"
+        outdir_dl1 = dl1_dir / flat_date / options.prod_id / options.dl1_prod_id
+        outdir_dl2 = dl2_dir / flat_date / options.prod_id / options.dl2_prod_id
 
     if class_instance.__name__ in ["drs4_pedestal", "calibrate_charge"]:
         # drs4_pedestal_run_id  [0] 1804
@@ -63,123 +61,133 @@ def parse_variables(class_instance):
         class_instance.PedestalRun = f"{class_instance.args[0]:05d}"
         class_instance.CalibrationRun = f"{class_instance.args[1]:05d}"
 
-        pro = "pro"
-        # TODO - massive reprocessing vs. next day processing
+        version = get_lstchain_version()
 
         # according to code in onsite scripts in lstchain
-        class_instance.RawObservationFilePedestal = os.path.realpath(
-            f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[0]:05d}.fits.fz"
-        )
-        class_instance.RawObservationFileCalibration = os.path.realpath(
-            f"{raw_dir}/{flat_date}/LST-1.1.Run{class_instance.args[1]:05d}.fits.fz"
-        )
-        class_instance.PedestalCheckPlot = os.path.realpath(
-            f"{pedestal_dir}/{flat_date}/{pro}/"
-            f"log/drs4_pedestal.Run{class_instance.args[0]:05d}.0000.pdf"
-        )
-        class_instance.CalibrationCheckPlot = os.path.realpath(
-            f"{calib_dir}/{flat_date}/{pro}/"
-            f"log/calibration_filters_52.Run{class_instance.args[1]:05d}.0000.pdf"
-        )
+        class_instance.RawObservationFilePedestal = str((
+            raw_dir / flat_date / f"LST-1.1.Run{class_instance.args[0]:05d}.fits.fz"
+        ).resolve())
+        class_instance.RawObservationFileCalibration = str((
+            raw_dir / flat_date / f"LST-1.1.Run{class_instance.args[1]:05d}.fits.fz"
+        ).resolve())
+        class_instance.PedestalCheckPlot = str((
+            pedestal_dir
+            / flat_date
+            / version
+            / f"log/drs4_pedestal.Run{class_instance.args[0]:05d}.0000.pdf"
+        ).resolve())
+        class_instance.CalibrationCheckPlot = str((
+            calib_dir
+            / flat_date
+            / version
+            / f"log/calibration_filters_52.Run{class_instance.args[1]:05d}.0000.pdf"
+        ).resolve())
 
         # according to code in sequence_calibration_files from paths.py
-        class_instance.PedestalFile = os.path.realpath(
-            f"{pedestal_dir}/{flat_date}/{pro}/"
-            f"drs4_pedestal.Run{class_instance.args[0]:05d}.0000.h5"
-        )
-        class_instance.CoefficientsCalibrationFile = os.path.realpath(
-            f"{calib_dir}/{flat_date}/{pro}/"
-            f"calibration_filters_52.Run{class_instance.args[1]:05d}.0000.h5"
-        )
+        class_instance.PedestalFile = str((
+            pedestal_dir
+            / flat_date
+            / version
+            / f"drs4_pedestal.Run{class_instance.args[0]:05d}.0000.h5"
+        ).resolve())
+        class_instance.CoefficientsCalibrationFile = str((
+            calib_dir
+            / flat_date
+            / version
+            / f"calibration_filters_52.Run{class_instance.args[1]:05d}.0000.h5"
+        ).resolve())
 
     if class_instance.__name__ == "r0_to_dl1":
-        # calibration_file   [0] .../20200218/pro/calibration_filters_52.Run02006.0000.h5
-        # drs4_pedestal_file [1] .../20200218/pro/drs4_pedestal.Run02005.0000.h5
-        # time_calib_file    [2] .../20191124/pro/time_calibration.Run01625.0000.h5
-        # systematic_corr    [3] .../20200101/pro/no_sys_corrected_calib_20210514.0000.h5
+        # calibration_file   [0] .../20200218/v0.4.3/calibration_filters_52.Run02006.0000.h5
+        # drs4_pedestal_file [1] .../20200218/v0.4.3/drs4_pedestal.Run02005.0000.h5
+        # time_calib_file    [2] .../20191124/v0.4.3/time_calibration.Run01625.0000.h5
+        # systematic_corr    [3] .../20200101/v0.4.3/no_sys_corrected_calib_20210514.0000.h5
         # drive_file         [4] .../DrivePositioning/DrivePosition_20200218.txt
         # run_summary_file   [5] .../RunSummary/RunSummary_20200101.ecsv
         # pedestal_ids_file  [6] .../path/to/interleaved/pedestal/events.h5
         # run_str            [7] 02006.0000
 
-        run_subrun_id = class_instance.args[7]
-        class_instance.ObservationRun = run_subrun_id.split(".")[0]
-        # use realpath to resolve symbolic links and return abspath
-        calibration_file = os.path.realpath(class_instance.args[0])
-        pedestal_file = os.path.realpath(class_instance.args[1])
-        timecalibration_file = os.path.realpath(class_instance.args[2])
-        systematic_correction_file = os.path.realpath(class_instance.args[3])
-        class_instance.R0SubrunDataset = os.path.realpath(
-            f"{raw_dir}/{flat_date}/LST-1.1.Run{run_subrun_id}.fits.fz"
-        )
-        class_instance.CoefficientsCalibrationFile = calibration_file
-        class_instance.PedestalFile = pedestal_file
-        class_instance.TimeCalibrationFile = timecalibration_file
-        class_instance.SystematicCorrectionFile = systematic_correction_file
-        class_instance.PointingFile = os.path.realpath(class_instance.args[4])
-        class_instance.RunSummaryFile = os.path.realpath(class_instance.args[5])
-        class_instance.DL1SubrunDataset = os.path.realpath(
-            f"{outdir_dl1}/dl1_LST-1.Run{run_subrun_id}.h5"
-        )
-        class_instance.MuonsSubrunDataset = os.path.realpath(
-            f"{muon_dir}/muons_LST-1.Run{run_subrun_id}.fits"
-        )
+        run_subrun = class_instance.args[7]
+        run = run_subrun.split(".")[0]
+        class_instance.ObservationRun = run
+
+        calibration_file = Path(class_instance.args[0]).resolve()
+        pedestal_file = Path(class_instance.args[1]).resolve()
+        timecalibration_file = Path(class_instance.args[2]).resolve()
+        systematic_correction_file = Path(class_instance.args[3]).resolve()
+        class_instance.R0SubrunDataset = str((
+            raw_dir / flat_date / f"LST-1.1.Run{run_subrun}.fits.fz"
+        ).resolve())
+        class_instance.CoefficientsCalibrationFile = str(calibration_file)
+        class_instance.PedestalFile = str(pedestal_file)
+        class_instance.TimeCalibrationFile = str(timecalibration_file)
+        class_instance.SystematicCorrectionFile = str(systematic_correction_file)
+        class_instance.PointingFile = str(Path(class_instance.args[4]).resolve())
+        class_instance.RunSummaryFile = str(Path(class_instance.args[5]).resolve())
+        class_instance.DL1SubrunDataset = str(
+            (outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
+        )
+        class_instance.MuonsSubrunDataset = str((
+            muon_dir / f"muons_LST-1.Run{run_subrun}.fits"
+        ).resolve())
         class_instance.InterleavedPedestalEventsFile = None
         if class_instance.args[6] is not None:
-            class_instance.InterleavedPedestalEventsFile = os.path.realpath(class_instance.args[6])
+            class_instance.InterleavedPedestalEventsFile = str(Path(class_instance.args[6]))
 
     if class_instance.__name__ == "dl1ab":
         # run_str       [0] 02006.0000
 
-        class_instance.Analysisconfigfile_dl1 = os.path.realpath(configfile_dl1b)
+        class_instance.Analysisconfigfile_dl1 = str(Path(configfile_dl1b))
         class_instance.ObservationRun = class_instance.args[0].split(".")[0]
         class_instance.StoreImage = cfg.getboolean("lstchain", "store_image_dl1ab")
-        class_instance.DL1SubrunDataset = os.path.realpath(
-            f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5"
-        )
+        class_instance.DL1SubrunDataset = str((
+            outdir_dl1 / f"dl1_LST-1.Run{class_instance.args[0]}.h5"
+        ).resolve())
 
     if class_instance.__name__ == "dl1_datacheck":
         # run_str       [0] 02006.0000
-
-        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
-        class_instance.DL1SubrunDataset = os.path.realpath(
-            f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5"
-        )
-        class_instance.MuonsSubrunDataset = os.path.realpath(
-            f"{muon_dir}/muons_LST-1.Run{class_instance.args[0]}.fits"
-        )
-        class_instance.DL1CheckSubrunDataset = os.path.realpath(
-            f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.args[0]}.h5"
-        )
-        class_instance.DL1CheckHDF5File = os.path.realpath(
-            f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.ObservationRun}.h5"
-        )
-        class_instance.DL1CheckPDFFile = os.path.realpath(
-            f"{outdir_dl1}/datacheck_dl1_LST-1.Run{class_instance.ObservationRun}.pdf"
-        )
+        run_subrun = class_instance.args[0]
+        run = run_subrun.split(".")[0]
+
+        class_instance.ObservationRun = run
+        class_instance.DL1SubrunDataset = str(
+            (outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
+        )
+        class_instance.MuonsSubrunDataset = str((
+            muon_dir / f"muons_LST-1.Run{run_subrun}.fits"
+        ).resolve())
+        class_instance.DL1CheckSubrunDataset = str((
+            outdir_dl1 / f"datacheck_dl1_LST-1.Run{run_subrun}.h5"
+        ).resolve())
+        class_instance.DL1CheckHDF5File = str((
+            outdir_dl1 / f"datacheck_dl1_LST-1.Run{run}.h5"
+        ).resolve())
+        class_instance.DL1CheckPDFFile = str((
+            outdir_dl1 / f"datacheck_dl1_LST-1.Run{run}.pdf"
+        ).resolve())
 
     if class_instance.__name__ == "dl1_to_dl2":
         # run_str       [0] 02006.0000
+        run_subrun = class_instance.args[0]
+        run = run_subrun.split(".")[0]
 
         class_instance.Analysisconfigfile_dl2 = configfile_dl2
-        class_instance.ObservationRun = class_instance.args[0].split(".")[0]
-        class_instance.RFModelEnergyFile = os.path.realpath(f"{rf_models_directory}/reg_energy.sav")
-        class_instance.RFModelDispNormFile = os.path.realpath(
-            f"{rf_models_directory}/reg_disp_norm.sav"
-        )
-        class_instance.RFModelDispSignFile = os.path.realpath(
-            f"{rf_models_directory}/reg_disp_sign.sav"
+        class_instance.ObservationRun = run
+        class_instance.RFModelEnergyFile = str((rf_models_directory / "reg_energy.sav").resolve())
+        class_instance.RFModelDispNormFile = str(
+            (rf_models_directory / "reg_disp_norm.sav").resolve()
         )
-        class_instance.RFModelGammanessFile = os.path.realpath(f"{rf_models_directory}/cls_gh.sav")
-        class_instance.DL1SubrunDataset = os.path.realpath(
-            f"{outdir_dl1}/dl1_LST-1.Run{class_instance.args[0]}.h5"
+        class_instance.RFModelDispSignFile = str(
+            (rf_models_directory / "reg_disp_sign.sav").resolve()
         )
-        class_instance.DL2SubrunDataset = os.path.realpath(
-            f"{outdir_dl2}/dl2_LST-1.Run{class_instance.args[0]}.h5"
+        class_instance.RFModelGammanessFile = str((rf_models_directory / "cls_gh.sav").resolve())
+        class_instance.DL1SubrunDataset = str(
+            (outdir_dl1 / f"dl1_LST-1.Run{run_subrun}.h5").resolve()
         )
-        class_instance.DL2MergedFile = os.path.realpath(
-            f"{outdir_dl2}/dl2_LST-1.Run{class_instance.ObservationRun}.h5"
+        class_instance.DL2SubrunDataset = str(
+            (outdir_dl2 / f"dl2_LST-1.Run{run_subrun}.h5").resolve()
         )
+        class_instance.DL2MergedFile = str((outdir_dl2 / f"dl2_LST-1.Run{run}.h5").resolve())
 
     if class_instance.__name__ in REDUCTION_TASKS:
         class_instance.session_name = class_instance.ObservationRun
@@ -189,5 +197,5 @@ def parse_variables(class_instance):
 
 def get_log_config():
     """Get logging configuration from provenance logger config file."""
-    std_logger_file = files("osa.provenance").joinpath("config/logger.yaml")
+    std_logger_file = files("osa.provenance") / "config/logger.yaml"
     return std_logger_file.read_text()

osa/scripts/calibration_pipeline.py

@@ -37,7 +37,8 @@ def is_calibration_produced(drs4_pedestal_run_id: int, pedcal_run_id: int) -> bo
     Check if both daily calibration (DRS4 baseline and
     charge calibration) files are already produced.
     """
-    return drs4_pedestal_exists(drs4_pedestal_run_id) and calibration_file_exists(pedcal_run_id)
+    return drs4_pedestal_exists(drs4_pedestal_run_id, options.prod_id) \
+        and calibration_file_exists(pedcal_run_id, options.prod_id)
 
 
 def drs4_pedestal_command(drs4_pedestal_run_id: int) -> list:
@@ -122,7 +123,7 @@ def drs4_pedestal(
     rc : int
         Return code
     """
-    if options.simulate or drs4_pedestal_exists(drs4_pedestal_run_id):
+    if options.simulate or drs4_pedestal_exists(drs4_pedestal_run_id, options.prod_id):
         return 0
 
     cmd = drs4_pedestal_command(drs4_pedestal_run_id)
@@ -154,7 +155,7 @@ def calibrate_charge(
     rc: int
         Return code
     """
-    if options.simulate or calibration_file_exists(pedcal_run_id):
+    if options.simulate or calibration_file_exists(pedcal_run_id, options.prod_id):
         return 0
 
     cmd = calibration_file_command(

osa/scripts/closer.py

@@ -193,7 +193,6 @@ def post_process_files(seq_list: list):
     output_files_set = set(Path(options.directory).rglob("*Run*"))
 
     DL1AB_RE = re.compile(rf"{options.dl1_prod_id}/dl1.*.(?:h5|hdf5|hdf)")
-    DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
     MUONS_RE = re.compile(r"muons.*.fits")
     DATACHECK_RE = re.compile(r"datacheck_dl1.*.(?:h5|hdf5|hdf)")
     INTERLEAVED_RE = re.compile(r"interleaved.*.(?:h5|hdf5|hdf)")
@@ -201,13 +200,16 @@ def post_process_files(seq_list: list):
     pattern_files = dict(
         [
             ("DL1AB", DL1AB_RE),
-            ("DL2", DL2_RE),
             ("MUON", MUONS_RE),
             ("DATACHECK", DATACHECK_RE),
             ("INTERLEAVED", INTERLEAVED_RE),
         ]
     )
 
+    if not options.no_dl2:
+        DL2_RE = re.compile(f"{options.dl2_prod_id}/dl2.*.(?:h5|hdf5|hdf)")
+        pattern_files["DL2"] = DL2_RE
+
     for concept, pattern_re in pattern_files.items():
         log.info(f"Post processing {concept} files, {len(output_files_set)} files left")
 
@@ -368,6 +370,9 @@ def extract_provenance(seq_list):
                 nightdir,
                 options.prod_id,
             ]
+            if options.no_dl2:
+                cmd.append("--no-dl2")
+                
             if not options.simulate and not options.test and shutil.which("sbatch") is not None:
                 subprocess.run(cmd, check=True)
             else:

osa/scripts/gain_selection.py

@@ -41,7 +41,7 @@ def get_sbatch_script(
     )
 
 
-def apply_gain_selection(date: str, output_basedir: Path = None):
+def apply_gain_selection(date: str, start: int, end: int, output_basedir: Path = None):
     """
     Submit the jobs to apply the gain selection to the data for a given date
     on a subrun-by-subrun basis.
@@ -64,7 +64,7 @@ def apply_gain_selection(date: str, output_basedir: Path = None):
         check_job_status_and_wait(max_jobs=1500)
 
         # Avoid running jobs while it is still night time
-        wait_for_daytime(start=12, end=18)
+        wait_for_daytime(start, end)
 
         run_id = run["run_id"]
         ref_time = run["dragon_reference_time"]
@@ -121,7 +121,7 @@ def apply_gain_selection(date: str, output_basedir: Path = None):
 
     for run in calib_runs:
         # Avoid copying files while it is still night time
-        wait_for_daytime(start=12, end=18)
+        wait_for_daytime(start, end)
 
         run_id = run["run_id"]
         r0_files = r0_dir.glob(f"LST-1.?.Run{run_id:05d}.????.fits.fz")
@@ -206,7 +206,15 @@ def check_failed_jobs(date: str, output_basedir: Path = None):
 @click.option("--check", is_flag=True, default=False, help="Check for failed jobs.")
 @click.argument("dates-file", type=click.Path(exists=True, path_type=Path))
 @click.argument("output-basedir", type=click.Path(path_type=Path))
-def main(dates_file: Path = None, output_basedir: Path = None, check: bool = False):
+@click.option("-s", "--start-time", type=int, default=10, help="Time to (re)start gain selection in HH format.")
+@click.option("-e", "--end-time", type=int, default=18, help="Time to stop gain selection in HH format.")
+def main(
+    dates_file: Path = None, 
+    output_basedir: Path = None, 
+    check: bool = False, 
+    start_time: int = 10, 
+    end_time: int = 18
+):
     """
     Loop over the dates listed in the input file and launch the gain selection
     script for each of them. The input file should list the dates in the format
@@ -221,7 +229,7 @@ def main(dates_file: Path = None, output_basedir: Path = None, check: bool = Fal
             check_failed_jobs(date, output_basedir)
     else:
         for date in list_of_dates:
-            apply_gain_selection(date, output_basedir)
+            apply_gain_selection(date, start_time, end_time, output_basedir)
         log.info("Done! No more dates to process.")
 
 

osa/scripts/provprocess.py

@@ -442,15 +442,17 @@ def produce_provenance(session_log_filename, base_filename):
         calibration_to_dl1 = define_paths(
             "calibration_to_dl1", PATH_DL1, options.dl1_prod_id, base_filename
         )
-        calibration_to_dl2 = define_paths(
-            "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename
-        )
         calibration_to_dl1_lines = calibration_lines + dl1_lines[1:]
         lines_dl1 = copy.deepcopy(calibration_to_dl1_lines)
-        calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:]
-        lines_dl2 = copy.deepcopy(calibration_to_dl2_lines)
         produce_provenance_files(lines_dl1, calibration_to_dl1)
-        produce_provenance_files(lines_dl2, calibration_to_dl2)
+
+        if not options.no_dl2:
+            calibration_to_dl2 = define_paths(
+                "calibration_to_dl2", PATH_DL2, options.dl2_prod_id, base_filename
+            )
+            calibration_to_dl2_lines = calibration_to_dl1_lines + dl1_dl2_lines[1:]
+            lines_dl2 = copy.deepcopy(calibration_to_dl2_lines)
+            produce_provenance_files(lines_dl2, calibration_to_dl2)
 
 
 def main():

osa/scripts/tests/test_osa_scripts.py

@@ -63,6 +63,7 @@ def test_simulate_processing(
     run_summary_file,
     r0_data,
     merged_run_summary,
+    drive_log
 ):
 
     for file in drs4_time_calibration_files:
@@ -75,6 +76,8 @@ def test_simulate_processing(
         assert r0_file.exists()
 
     assert run_summary_file.exists()
+    assert merged_run_summary.exists()
+    assert drive_log.exists()
 
     remove_provlog()
     rc = run_program("simulate_processing", "-p", "--force")
@@ -257,7 +260,7 @@ def test_datasequence(running_analysis_dir):
 
 
 def test_calibration_pipeline(running_analysis_dir):
-    prod_id = "v0.1.0"
+    options.prod_id = "v0.1.0"
     drs4_run_number = "01804"
     pedcal_run_number = "01805"
     options.directory = running_analysis_dir
@@ -266,7 +269,7 @@ def test_calibration_pipeline(running_analysis_dir):
         "calibration_pipeline",
         "--date=2020-01-17",
         "--simulate",
-        f"--prod-id={prod_id}",
+        f"--prod-id={options.prod_id}",
         f"--drs4-pedestal-run={drs4_run_number}",
         f"--pedcal-run={pedcal_run_number}",
         "LST1",

osa/tests/test_paths.py

@@ -6,23 +6,24 @@ from osa.configs.config import cfg
 from osa.utils.utils import date_to_dir
 
 options.date = datetime.fromisoformat("2020-01-17")
+options.prod_id = "v0.1.1"
 
 
 def test_get_calibration_file(r0_data, merged_run_summary):
-    from osa.paths import get_calibration_file
+    from osa.paths import get_calibration_filename
 
     for file in r0_data:
         assert file.exists()
-    file = get_calibration_file(1809)
+    file = get_calibration_filename(1809, options.prod_id)
     file.exists()
 
 
 def test_get_drs4_pedestal_file(r0_data, merged_run_summary):
-    from osa.paths import get_drs4_pedestal_file
+    from osa.paths import get_drs4_pedestal_filename
 
     for file in r0_data:
         assert file.exists()
-    file = get_drs4_pedestal_file(1804)
+    file = get_drs4_pedestal_filename(1804, options.prod_id)
     file.exists()
 
 
@@ -58,6 +59,7 @@ def test_destination_dir():
     base_path = Path(base_directory)
 
     data_types = {
+        "INTERLEAVED": "DL1",
         "DL1AB": "DL1",
         "DATACHECK": "DL1",
         "MUON": "DL1",
@@ -86,8 +88,6 @@ def test_destination_dir():
             expected_directory = (
                 base_path / dst_dir / datedir / options.prod_id / options.dl2_prod_id
             )
-        else:
-            expected_directory = base_path / dst_dir / datedir / options.prod_id
 
         assert directory == expected_directory
 

osa/utils/cliopts.py

@@ -10,7 +10,6 @@ from osa.configs.config import cfg
 from osa.paths import analysis_path, DEFAULT_CFG
 from osa.utils.logging import myLogger
 from osa.utils.utils import (
-    get_calib_prod_id,
     get_dl1_prod_id,
     get_dl2_prod_id,
     get_prod_id,
@@ -35,7 +34,6 @@ __all__ = [
     "get_prod_id",
     "get_dl1_prod_id",
     "get_dl2_prod_id",
-    "get_calib_prod_id",
     "calibration_pipeline_cliparsing",
     "calibration_pipeline_argparser",
     "autocloser_cli_parser",
@@ -171,10 +169,7 @@ def calibration_pipeline_cliparsing():
     # setting the default date and directory if needed
     options.date = set_default_date_if_needed()
     options.directory = analysis_path(options.tel_id)
-    if cfg.get("LST1", "CALIB_PROD_ID") is not None:
-        options.calib_prod_id = get_calib_prod_id()
-    else:
-        options.calib_prod_id = options.prod_id
+
     return opts.drs4_pedestal_run, opts.pedcal_run
 
 
@@ -331,6 +326,12 @@ def provprocess_argparser():
         dest="quit",
         help="use this flag to reset session and remove log file",
     )
+    parser.add_argument(
+        "--no-dl2",
+        action="store_true",
+        default=False,
+        help="Do not produce DL2 files (default False)",
+    )
     parser.add_argument(
         "drs4_pedestal_run_id", help="Number of the drs4_pedestal used in the calibration"
     )
@@ -359,6 +360,7 @@ def provprocessparsing():
     options.configfile = opts.config.resolve()
     options.filter = opts.filter
     options.quit = opts.quit
+    options.no_dl2 = opts.no_dl2
     set_prod_ids()
 
 

osa/utils/tests/test_utils.py

@@ -30,13 +30,6 @@ def test_get_prod_id():
     assert get_prod_id() == prod_id
 
 
-def test_get_calib_prod_id():
-    from osa.utils.utils import get_calib_prod_id
-
-    prod_id = cfg.get(options.tel_id, "CALIB_PROD_ID")
-    assert get_calib_prod_id() == prod_id
-
-
 def test_date_to_dir():
     from osa.utils.utils import date_to_dir
 

osa/utils/utils.py

@@ -26,7 +26,6 @@ __all__ = [
     "create_lock",
     "stringify",
     "gettag",
-    "get_calib_prod_id",
     "get_dl1_prod_id",
     "get_dl2_prod_id",
     "time_to_seconds",
@@ -86,19 +85,6 @@ def get_prod_id():
     return options.prod_id
 
 
-def get_calib_prod_id() -> str:
-    """Build calibration production ID."""
-    if not options.calib_prod_id:
-        if cfg.get("LST1", "CALIB_PROD_ID") is not None:
-            options.calib_prod_id = cfg.get("LST1", "CALIB_PROD_ID")
-        else:
-            options.calib_prod_id = get_lstchain_version()
-
-    log.debug(f"Getting prod ID for calibration products: {options.calib_prod_id}")
-
-    return options.calib_prod_id
-
-
 def get_dl1_prod_id():
     """
     Get the prod ID for the dl1 products provided
@@ -262,11 +248,6 @@ def set_prod_ids():
     """Set the product IDs."""
     options.prod_id = get_prod_id()
 
-    if cfg.get("LST1", "CALIB_PROD_ID") is not None:
-        options.calib_prod_id = get_calib_prod_id()
-    else:
-        options.calib_prod_id = options.prod_id
-
     if cfg.get("LST1", "DL1_PROD_ID") is not None:
         options.dl1_prod_id = get_dl1_prod_id()
     else:

osa/workflow/stages.py

@@ -11,6 +11,7 @@ from pathlib import Path
 from typing import List, Union
 
 from tenacity import retry, stop_after_attempt
+import lstchain
 
 from osa.configs import options
 from osa.configs.config import cfg
@@ -75,36 +76,48 @@ class AnalysisStage:
         """
 
         if self.command == "lstchain_data_r0_to_dl1":
-            dl1_output_file = options.directory / f"dl1_LST-1.Run{self.run}.h5"
-            muon_output_file = options.directory / f"muons_LST-1.Run{self.run}.fits"
-            interleaved_output_file = options.directory / f"interleaved_LST-1.Run{self.run}.h5"
-            dl1_output_file.unlink(missing_ok=True)
-            muon_output_file.unlink(missing_ok=True)
-            interleaved_output_file.unlink(missing_ok=True)
-
+            self._remove_dl1a_output()
         elif self.command == "lstchain_dl1ab":
-            dl1ab_subdirectory = options.directory / options.dl1_prod_id
-            output_file = dl1ab_subdirectory / f"dl1_LST-1.Run{self.run}.h5"
-            output_file.unlink(missing_ok=True)
-
+            self._remove_dl1b_output('dl1_LST-1.Run')
         elif self.command == "lstchain_check_dl1":
-            dl1ab_subdirectory = options.directory / options.dl1_prod_id
-            output_file = dl1ab_subdirectory / f"datacheck_dl1_LST-1.Run{self.run}.h5"
-            output_file.unlink(missing_ok=True)
-
+            self._remove_dl1b_output('datacheck_dl1_LST-1.Run')
         elif self.command == "onsite_create_calibration_file":
-            calib_base = Path(cfg.get("LST1", "CALIB_DIR"))
-            date = date_to_dir(get_run_date(self.run))
-            cal_dir = calib_base / date / "pro"
-            output_file = cal_dir / f"calibration_filters_{options.filters}.Run{self.run}.0000.h5"
-            output_file.unlink(missing_ok=True)
-
+            self._remove_calibration()
         elif self.command == "onsite_create_drs4_pedestal_file":
-            drs4_pedestal_dir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
-            date = date_to_dir(get_run_date(self.run))
-            output_file = drs4_pedestal_dir / date / f"pro/drs4_pedestal.Run{self.run}.0000.h5"
-            output_file.unlink(missing_ok=True)
-
+            self._remove_drs4_baseline()
+
+    def _remove_drs4_baseline(self):
+        drs4_pedestal_basedir = Path(cfg.get("LST1", "PEDESTAL_DIR"))
+        date = date_to_dir(get_run_date(self.run))
+        drs4_pedestal_dir = drs4_pedestal_basedir / date / lstchain.__version__
+        file = drs4_pedestal_dir / "drs4_pedestal.Run{self.run}.0000.h5"
+        file.unlink(missing_ok=True)
+        # Also remove the link to "pro" directory
+        drs4_pedestal_dir_pro = drs4_pedestal_basedir / date / "pro"
+        drs4_pedestal_dir_pro.unlink(missing_ok=True)
+
+    def _remove_calibration(self):
+        calib_basedir = Path(cfg.get("LST1", "CALIB_DIR"))
+        date = date_to_dir(get_run_date(self.run))
+        calib_dir = file = calib_basedir / date / lstchain.__version__
+        file = calib_dir / f"calibration_filters_{options.filters}.Run{self.run}.0000.h5"
+        file.unlink(missing_ok=True)
+        # Also remove the link to "pro" directory
+        calib_dir_pro = file = calib_basedir / date / "pro"
+        calib_dir_pro.unlink(missing_ok=True)
+
+    def _remove_dl1a_output(self):
+        dl1_output_file = options.directory / f"dl1_LST-1.Run{self.run}.h5"
+        muon_output_file = options.directory / f"muons_LST-1.Run{self.run}.fits"
+        interleaved_output_file = options.directory / f"interleaved_LST-1.Run{self.run}.h5"
+        dl1_output_file.unlink(missing_ok=True)
+        muon_output_file.unlink(missing_ok=True)
+        interleaved_output_file.unlink(missing_ok=True)
+
+    def _remove_dl1b_output(self, file_prefix):
+        dl1ab_subdirectory = options.directory / options.dl1_prod_id
+        output_file = dl1ab_subdirectory / f"{file_prefix}{self.run}.h5"
+        output_file.unlink(missing_ok=True)
 
     def _write_checkpoint(self):
         """Write the checkpoint in the history file."""
@@ -137,7 +150,6 @@ class DRS4PedestalStage(AnalysisStage):
         config_file: Union[str, None] = None,
     ):
         super().__init__(run, command_args, config_file)
-        self.prod_id = options.calib_prod_id
         self.run_pedcal = run_pedcal
         self.history_file = (
             Path(options.directory) / f"sequence_{options.tel_id}_{self.run_pedcal}.history"
@@ -147,7 +159,7 @@ class DRS4PedestalStage(AnalysisStage):
         """Write the checkpoint in the history file."""
         history(
             run=self.run,
-            prod_id=self.prod_id,
+            prod_id=options.prod_id,
             stage=self.command,
             return_code=self.rc,
             history_file=self.history_file,
@@ -164,7 +176,6 @@ class ChargeCalibrationStage(AnalysisStage):
         config_file: Union[str, None] = None,
     ):
         super().__init__(run, command_args, config_file)
-        self.prod_id = options.calib_prod_id
         self.history_file = (
             Path(options.directory) / f"sequence_{options.tel_id}_{self.run}.history"
         )
@@ -173,7 +184,7 @@ class ChargeCalibrationStage(AnalysisStage):
         """Write the checkpoint in the history file."""
         history(
             run=self.run,
-            prod_id=self.prod_id,
+            prod_id=options.prod_id,
             stage=self.command,
             return_code=self.rc,
             history_file=self.history_file,