lstosa 0.10.17__tar.gz → 0.10.19__tar.gz

{lstosa-0.10.17 → lstosa-0.10.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.17
+Version: 0.10.19
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -13,14 +13,13 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Classifier: Topic :: Scientific/Engineering :: Physics
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain>=0.10.7
+Requires-Dist: lstchain>=0.10.15
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
@@ -34,6 +33,7 @@ Requires-Dist: toml
 Requires-Dist: pymongo
 Requires-Dist: gammapy~=1.1
 Requires-Dist: tenacity
+Requires-Dist: lstcam_calib
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"

{lstosa-0.10.17 → lstosa-0.10.19}/environment.yml

@@ -21,12 +21,13 @@ dependencies:
   - click
   - pymongo
   - scikit-learn=1.2
-  - protobuf=3.20
-  - ctapipe_io_lst=0.22
+  - ctapipe_io_lst >=0.25.1,<0.26a0
   - ctaplot~=0.6.4
   - pyirf~=0.10
-  - lstchain>=0.10.7
+  - lstchain>=0.10.15
   - tenacity
+  - pip:
+      - lstcam_calib
   # dev dependencies
   - pytest
   - pytest-cov

lstosa-0.10.19/extra/history_files/sequence_LST1_04183.history

@@ -0,0 +1,2 @@
+04179 lstcam_calib_onsite_create_drs4_pedestal_file v0.7.0 2021-12-19 22:11 None None 0
+04183 lstcam_calib_onsite_create_calibration_file v0.7.0 2021-12-19 22:11 None None 0

lstosa-0.10.19/extra/history_files/sequence_LST1_04185.0001.history

@@ -0,0 +1,3 @@
+04185.0001 lstchain_data_r0_to_dl1 v0.7.0 Thu Mar 25 12:53:40 UTC 2021 calibration.Run04183.0000.hdf5 drs4_pedestal.Run04179.0000.fits 0
+04185.0001 lstchain_dl1ab tailcut105 Thu Mar 25 13:06:09 UTC 2021 dl1_LST-1.Run04185.0001.h5 lstchain_standard_config_tailcut84.json 0
+04185.0001 lstchain_check_dl1 tailcut105 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0001.h5 None 0

{lstosa-0.10.17 → lstosa-0.10.19}/extra/history_files/sequence_LST1_04185.0010.history

@@ -1,4 +1,4 @@
 04185.0010 lstchain_data_r0_to_dl1 v0.7.0 Thu Mar 25 12:53:40 UTC 2021 calibration.Run04183.0000.hdf5 drs4_pedestal.Run04179.0000.fits 0
 04185.0010 lstchain_dl1ab tailcut84 Thu Mar 25 13:06:09 UTC 2021 dl1_LST-1.Run04185.0010.h5 lstchain_standard_config_tailcut84.json 0
 04185.0010 lstchain_check_dl1 tailcut84 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0
-04185.0010 lstchain_dl1_to_dl2 model1 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0
+04185.0010 lstchain_dl1_to_dl2 tailcut84/nsb_tuning_0.14 Thu Mar 25 13:07:02 UTC 2021 dl1_LST-1.Run04185.0010.h5 None 0

{lstosa-0.10.17 → lstosa-0.10.19}/pyproject.toml

@@ -26,15 +26,14 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Astronomy",
     "Topic :: Scientific/Engineering :: Physics",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
 ]
 
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "astropy~=5.0",
-    "lstchain>=0.10.7",
+    "lstchain>=0.10.15",
     "matplotlib",
     "numpy",
     "pandas",
@@ -48,6 +47,7 @@ dependencies = [
     "pymongo",
     "gammapy~=1.1",
     "tenacity",
+    "lstcam_calib",
 ]
 
 dynamic = ["version"]
@@ -76,6 +76,7 @@ reprocess_longterm = "osa.scripts.reprocess_longterm:main"
 gain_selection = "osa.scripts.gain_selection:main"
 update_source_catalog = "osa.scripts.update_source_catalog:main"
 gainsel_webmaker = "osa.scripts.gainsel_webmaker:main"
+sequencer_catB_tailcuts = "osa.scripts.sequencer_catB_tailcuts:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]

{lstosa-0.10.17 → lstosa-0.10.19}/src/lstosa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.17
+Version: 0.10.19
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -13,14 +13,13 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: Astronomy
 Classifier: Topic :: Scientific/Engineering :: Physics
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: astropy~=5.0
-Requires-Dist: lstchain>=0.10.7
+Requires-Dist: lstchain>=0.10.15
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
@@ -34,6 +33,7 @@ Requires-Dist: toml
 Requires-Dist: pymongo
 Requires-Dist: gammapy~=1.1
 Requires-Dist: tenacity
+Requires-Dist: lstcam_calib
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"

{lstosa-0.10.17 → lstosa-0.10.19}/src/lstosa.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ extra/squeue_output.csv
 extra/history_files/sequence_LST1_04183.history
 extra/history_files/sequence_LST1_04183_failed.history
 extra/history_files/sequence_LST1_04183_oneline.history
+extra/history_files/sequence_LST1_04185.0001.history
 extra/history_files/sequence_LST1_04185.0010.history
 src/lstosa.egg-info/PKG-INFO
 src/lstosa.egg-info/SOURCES.txt
@@ -95,6 +96,7 @@ src/osa/scripts/provprocess.py
 src/osa/scripts/reprocess_longterm.py
 src/osa/scripts/reprocessing.py
 src/osa/scripts/sequencer.py
+src/osa/scripts/sequencer_catB_tailcuts.py
 src/osa/scripts/sequencer_webmaker.py
 src/osa/scripts/show_run_summary.py
 src/osa/scripts/simulate_processing.py

{lstosa-0.10.17 → lstosa-0.10.19}/src/lstosa.egg-info/entry_points.txt

@@ -11,6 +11,7 @@ provprocess = osa.scripts.provprocess:main
 reprocess_longterm = osa.scripts.reprocess_longterm:main
 reprocessing = osa.scripts.reprocessing:main
 sequencer = osa.scripts.sequencer:main
+sequencer_catB_tailcuts = osa.scripts.sequencer_catB_tailcuts:main
 sequencer_webmaker = osa.scripts.sequencer_webmaker:main
 show_run_summary = osa.scripts.show_run_summary:main
 show_run_summary_tcu = osa.scripts.show_run_summary_tcu:main

{lstosa-0.10.17 → lstosa-0.10.19}/src/lstosa.egg-info/requires.txt

@@ -1,5 +1,5 @@
 astropy~=5.0
-lstchain>=0.10.7
+lstchain>=0.10.15
 matplotlib
 numpy
 pandas
@@ -13,6 +13,7 @@ toml
 pymongo
 gammapy~=1.1
 tenacity
+lstcam_calib
 
 [all]
 lstosa[dev,doc,test]

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.17'
-__version_tuple__ = version_tuple = (0, 10, 17)
+__version__ = version = '0.10.19'
+__version_tuple__ = version_tuple = (0, 10, 19)

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/configs/options.py

@@ -15,6 +15,8 @@ verbose = None
 warning = None
 nocheck = None
 no_dl2 = None
+no_dl1ab = None
+no_gainsel = None
 prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/configs/sequencer.cfg

@@ -14,14 +14,18 @@ DRIVE_DIR: %(MONITORING)s/DrivePositioning
 RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
+TAILCUTS_FINDER_DIR: %(BASE)s/auxiliary/TailCuts
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
+RF_MODELS: %(BASE)s/models/AllSky
 OSA_DIR: %(BASE)s/OSA
 CLOSER_DIR: %(OSA_DIR)s/Closer
 HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
@@ -41,8 +45,11 @@ DL2_PROD_ID: model2
 
 [lstchain]
 # Calibration steps in calibration pipeline script
-drs4_baseline: onsite_create_drs4_pedestal_file
-charge_calibration: onsite_create_calibration_file
+#drs4_baseline: onsite_create_drs4_pedestal_file
+#charge_calibration: onsite_create_calibration_file
+drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file
+charge_calibration: lstcam_calib_onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
@@ -50,15 +57,18 @@ r0_to_dl1: lstchain_data_r0_to_dl1
 dl1ab: lstchain_dl1ab
 check_dl1: lstchain_check_dl1
 dl1_to_dl2: lstchain_dl1_to_dl2
+tailcuts_finder: lstchain_find_tailcuts
 
 # To be set by the user
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+apply_catB_calibration: False
+apply_standard_dl1b_config: False
 use_ff_heuristic_gain_selection: False
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
-rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3
+mc_prod: 20240918_v0.10.12_allsky_
 dl3_config: /software/lstchain/data/dl3_std_config.json
 max_tries: 3
 

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/conftest.py

@@ -31,8 +31,8 @@ import lstchain
 date = datetime.fromisoformat("2020-01-17")
 nightdir = date_to_dir(date)
 prod_id = "v0.1.0"
-dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
-dl2_prod_id = cfg.get("LST1", "DL2_PROD_ID")
+dl1_prod_id = "tailcut84"
+dl2_prod_id = "tailcut84/nsb_tuning_0.14"
 
 
 @pytest.fixture(scope="session")
@@ -412,6 +412,9 @@ def sequence_list(
     r0_data,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Creates a sequence list from a run summary file."""
     options.directory = running_analysis_dir
@@ -574,10 +577,131 @@ def database(osa_dir):
 
 
 @pytest.fixture(scope="session")
-def gain_selection_flag_file(osa_dir):
+def rf_models_allsky_basedir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def rf_model_path(rf_models_allsky_basedir):
+    mc_prod = "20240131_allsky_v0.10.5_all_dec_base"
+    declination_str = "dec_2276"
+    rf_model_path = rf_models_allsky_basedir / mc_prod / declination_str
+    rf_model_path.mkdir(parents=True, exist_ok=True)
+    return rf_model_path
 
+  
+@pytest.fixture(scope="session")
+def gain_selection_flag_file(osa_dir):
     GainSel_dir = osa_dir / "GainSel" / "20200117"
     GainSel_dir.mkdir(parents=True, exist_ok=True)
     file = GainSel_dir / "GainSelFinished.txt"
     file.touch()
     return file
+
+
+@pytest.fixture(scope="session")
+def catB_closed_file(running_analysis_dir):
+
+    catB_closed_file = running_analysis_dir / "catB_00003.closed"
+    catB_closed_file.touch()
+    return catB_closed_file
+
+
+@pytest.fixture(scope="session")
+def catB_calib_base_dir(monitoring_dir):
+
+    catB_calib_base_dir = monitoring_dir / "PixelCalibration" / "Cat-B"
+    catB_calib_base_dir.mkdir(parents=True, exist_ok=True)
+    return catB_calib_base_dir
+
+
+@pytest.fixture(scope="session")
+def catB_calibration_file(catB_calib_dir):
+
+    catB_calib_dir = catB_calib_base_dir / "calibration" / nightdir / prod_id
+    catB_calib_file = catB_calib_dir / "cat_B_calibration_filters_52.Run00003.h5"
+    catB_calib_file.touch()
+    return catB_calib_file
+
+
+@pytest.fixture(scope="session")
+def tailcuts_finder_dir(base_test_dir):
+    tailcuts_finder_dir = base_test_dir / "auxiliary" / "TailCuts"
+    tailcuts_finder_dir.mkdir(parents=True, exist_ok=True)
+    return tailcuts_finder_dir
+
+
+@pytest.fixture(scope="session")
+def dl1b_config_files(tailcuts_finder_dir):
+    config_information = dedent(
+        """\
+            {
+            "tailcuts_clean_with_pedestal_threshold": {
+                "picture_thresh": 8,
+                "boundary_thresh": 4,
+                "sigma": 2.5,
+                "keep_isolated_pixels": false,
+                "min_number_picture_neighbors": 2,
+                "use_only_main_island": false,
+                "delta_time": 2
+            },
+            "dynamic_cleaning": {
+                "apply": true,
+                "threshold": 267,
+                "fraction_cleaning_intensity": 0.03
+            }
+        }"""
+    )
+    config_file1 = tailcuts_finder_dir / "dl1ab_Run01807.json"
+    config_file1.touch()
+    config_file1.write_text(config_information)
+    config_file2 = tailcuts_finder_dir / "dl1ab_Run01808.json"
+    config_file2.touch()
+    config_file2.write_text(config_information)
+    config_file3 = tailcuts_finder_dir / "dl1ab_Run04185.json"
+    config_file3.touch()
+    config_file3.write_text(config_information)
+    return config_file1, config_file2, config_file3
+
+
+@pytest.fixture(scope="session")
+def tailcuts_log_files(tailcuts_finder_dir):
+    log_information = dedent(
+        """\
+    Median of 95% quantile of pedestal charge: 5.416 p.e.
+
+    Additional NSB rate (over dark MC): 0.2221 p.e./ns
+    lstchain_find_tailcuts finished successfully!
+    """
+    )
+    log_file1 = tailcuts_finder_dir / "log_find_tailcuts_Run01807.log"
+    log_file1.touch()
+    log_file1.write_text(log_information)
+    log_file2 = tailcuts_finder_dir / "log_find_tailcuts_Run01808.log"
+    log_file2.touch()
+    log_file2.write_text(log_information)
+    log_file3 = tailcuts_finder_dir / "log_find_tailcuts_Run04185.log"
+    log_file3.touch()
+    log_file3.write_text(log_information)
+    return log_file1, log_file2, log_file3
+
+
+@pytest.fixture(scope="session")
+def rf_models_base_dir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def rf_models(rf_models_base_dir):
+    rf_models_prefix = cfg.get("lstchain", "mc_prod")
+    rf_models_path1 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.00"
+    rf_models_path1.mkdir(parents=True, exist_ok=True)
+    rf_models_path2 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_2276"
+    rf_models_path2.mkdir(parents=True, exist_ok=True)
+    rf_models_path3 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_4822"
+    rf_models_path3.mkdir(parents=True, exist_ok=True)
+    return rf_models_path1, rf_models_path2, rf_models_path3

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/high_level/significance.py

@@ -26,7 +26,8 @@ from osa.configs import options
 from osa.configs.config import cfg
 from osa.nightsummary.extract import get_source_list
 from osa.paths import DEFAULT_CFG, destination_dir, analysis_path
-from osa.utils.cliopts import get_prod_id, get_dl2_prod_id
+from osa.utils.cliopts import get_prod_id
+from osa.paths import get_dl2_prod_id
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir, YESTERDAY
 
@@ -209,7 +210,6 @@ def main(
     flat_date = date_to_dir(date)
     options.tel_id = telescope
     options.prod_id = get_prod_id()
-    options.dl2_prod_id = get_dl2_prod_id()
     options.directory = analysis_path(options.tel_id)
     dl2_directory = Path(cfg.get("LST1", "DL2_DIR"))
     highlevel_directory = destination_dir("HIGH_LEVEL", create_dir=True)
@@ -232,7 +232,7 @@ def main(
                 dl2_directory
                 / flat_date
                 / options.prod_id
-                / options.dl2_prod_id
+                / get_dl2_prod_id(run)
                 / f"dl2_LST-1.Run{run:05d}.h5"
             )
             df = pd.concat([df, pd.read_hdf(input_file, key=dl2_params_lstcam_key)])

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/high_level/tests/test_significance.py

@@ -10,6 +10,9 @@ def test_significance(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     output = sp.run(
         ["theta2_significance", "-d", "2020-01-17", "-s", "LST1"],

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/job.py

@@ -5,6 +5,7 @@ import logging
 import shutil
 import subprocess as sp
 import time
+import re
 from io import StringIO
 from pathlib import Path
 from textwrap import dedent
@@ -20,10 +21,16 @@ from osa.paths import (
     get_drive_file,
     get_summary_file,
     get_pedestal_ids_file,
+    get_dl1_prod_id_and_config,
 )
 from osa.utils.iofile import write_to_file
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir, time_to_seconds, stringify, date_to_iso
+from osa.utils.utils import (
+    date_to_dir,
+    time_to_seconds,
+    stringify,
+    date_to_iso,
+)
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -92,7 +99,15 @@ def are_all_jobs_correctly_finished(sequence_list):
     flag = True
     analysis_directory = Path(options.directory)
     for sequence in sequence_list:
-        history_files_list = analysis_directory.rglob(f"*{sequence.seq}*.history")
+        history_files_list = analysis_directory.rglob(f"*{sequence.run}*.history")
+        
+        if not options.test:
+            try:
+                next(history_files_list)
+            except StopIteration:
+                log.debug("No history files found.")
+                flag = False
+
         for history_file in history_files_list:
             # TODO: s.history should be SubRunObj attribute not RunObj
             # s.history only working for CALIBRATION sequence (run-wise), since it is
@@ -103,10 +118,11 @@ def are_all_jobs_correctly_finished(sequence_list):
             if out == 0:
                 log.debug(f"Job {sequence.seq} ({sequence.type}) correctly finished")
                 continue
-            if out == 1 and options.no_dl2:
+
+            if out == 2 and options.no_dl1ab:
                 log.debug(
                     f"Job {sequence.seq} ({sequence.type}) correctly "
-                    f"finished up to DL1ab, but --no-dl2 option selected"
+                    f"finished up to DL1A, but --no-dl1ab option selected"
                 )
                 continue
 
@@ -166,10 +182,9 @@ def historylevel(history_file: Path, data_type: str):
      - Sequence completed when reaching level 0
 
     Workflow for DATA sequences:
-     - R0->DL1, level 4->3
-     - DL1->DL1AB, level 3->2
-     - DATACHECK, level 2->1
-     - DL1->DL2, level 1->0
+     - R0->DL1, level 3->2
+     - DL1->DL1AB, level 2->1
+     - DATACHECK, level 1->0
      - Sequence completed when reaching level 0
 
     Parameters
@@ -188,7 +203,7 @@ def historylevel(history_file: Path, data_type: str):
     #  into account not only the last history line but also the others.
 
     if data_type == "DATA":
-        level = 4
+        level = 3
     elif data_type == "PEDCALIB":
         level = 2
     else:
@@ -197,6 +212,11 @@ def historylevel(history_file: Path, data_type: str):
     exit_status = 0
 
     if history_file.exists():
+        if data_type == "DATA":
+            match = re.search(r"sequence_LST1_(\d+)\.\d+", str(history_file))
+        elif data_type == "PEDCALIB":
+            match = re.search(r"sequence_LST1_(\d+)\.history", str(history_file))
+        run_id = int(match.group(1)) 
         for line in history_file.read_text().splitlines():
             words = line.split()
             try:
@@ -214,24 +234,18 @@ def historylevel(history_file: Path, data_type: str):
                     level = 0 if exit_status == 0 else 1
                 # Data sequence
                 elif program == cfg.get("lstchain", "r0_to_dl1"):
-                    level = 3 if exit_status == 0 else 4
+                    level = 2 if exit_status == 0 else 3
                 elif program == cfg.get("lstchain", "dl1ab"):
-                    if (exit_status == 0) and (prod_id == options.dl1_prod_id):
-                        log.debug(f"DL1ab prod ID: {options.dl1_prod_id} already produced")
-                        level = 2
+                    dl1_prod_id = get_dl1_prod_id_and_config(run_id)[0]
+                    if (exit_status == 0) and (prod_id == dl1_prod_id):
+                        log.debug(f"DL1ab prod ID: {dl1_prod_id} already produced")
+                        level = 1
                     else:
-                        level = 3
-                        log.debug(f"DL1ab prod ID: {options.dl1_prod_id} not produced yet")
+                        level = 2
+                        log.debug(f"DL1ab prod ID: {dl1_prod_id} not produced yet")
                         break
                 elif program == cfg.get("lstchain", "check_dl1"):
-                    level = 1 if exit_status == 0 else 2
-                elif program == cfg.get("lstchain", "dl1_to_dl2"):
-                    if (exit_status == 0) and (prod_id == options.dl2_prod_id):
-                        log.debug(f"DL2 prod ID: {options.dl2_prod_id} already produced")
-                        level = 0
-                    else:
-                        level = 1
-                        log.debug(f"DL2 prod ID: {options.dl2_prod_id} not produced yet")
+                    level = 0 if exit_status == 0 else 1
 
                 else:
                     log.warning(f"Program name not identified: {program}")
@@ -423,8 +437,8 @@ def data_sequence_job_template(sequence):
         commandargs.append("-s")
     if options.configfile:
         commandargs.extend(("--config", f"{Path(options.configfile).resolve()}"))
-    if sequence.type == "DATA" and options.no_dl2:
-        commandargs.append("--no-dl2")
+    if sequence.type == "DATA" and options.no_dl1ab:
+        commandargs.append("--no-dl1ab")
 
     commandargs.extend(
         (
@@ -439,6 +453,14 @@ def data_sequence_job_template(sequence):
         )
     )
 
+    if not options.no_dl1ab:
+        dl1_prod_id, dl1b_config = get_dl1_prod_id_and_config(sequence.run)
+        sequence.dl1_prod_id = dl1_prod_id
+        sequence.dl1b_config = dl1b_config
+
+        commandargs.append(f"--dl1b-config={sequence.dl1b_config}")
+        commandargs.append(f"--dl1-prod-id={sequence.dl1_prod_id}")
+
     content = job_header + "\n" + PYTHON_IMPORTS
 
     if not options.test:
@@ -727,6 +749,7 @@ def get_closer_sacct_output(sacct_output) -> pd.DataFrame:
         | (sacct_output["JobName"].str.contains("lstchain_longterm_dl1_check"))
         | (sacct_output["JobName"].str.contains("lstchain_cherenkov_transparency"))
         | (sacct_output["JobName"].str.contains("provproces"))
+        | (sacct_output["JobName"].str.contains("lstchain_dl1_to_dl2"))
     ]
 
     try:

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/nightsummary/extract.py

@@ -23,9 +23,9 @@ from osa.configs.datamodel import Sequence
 from osa.job import sequence_filenames
 from osa.nightsummary import database
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import sequence_calibration_files, get_run_date
+from osa.paths import sequence_calibration_files, get_run_date, get_dl1_prod_id_and_config, get_dl2_prod_id
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_iso, date_to_dir
+from osa.utils.utils import date_to_iso, date_to_dir, get_RF_model
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -263,6 +263,15 @@ def extract_sequences(date: datetime, run_obj_list: List[RunObj]) -> List[Sequen
                 f"Data sequence {sequence.seq} from run {run.run} whose parent is "
                 f"{sequence.parent} (DRS4 {required_drs4_run} & Ped-Cal {required_pedcal_run})"
             )
+            if not options.no_dl1ab and sequence.type=="DATA":
+                dl1_prod_id, dl1b_config = get_dl1_prod_id_and_config(sequence.run)
+                sequence.dl1_prod_id = dl1_prod_id
+                sequence.dl1b_config = dl1b_config
+
+            if not options.no_dl2 and not options.no_dl1ab and sequence.type=="DATA":
+                sequence.dl2_prod_id = get_dl2_prod_id(sequence.run)
+                sequence.rf_model = get_RF_model(sequence.run)
+
             sequence_list.append(sequence)
 
     # Add the calibration file names

{lstosa-0.10.17 → lstosa-0.10.19}/src/osa/nightsummary/tests/test_extract.py

@@ -11,6 +11,9 @@ def test_source_list(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Test that the list of name of sources is correct."""
     from osa.nightsummary.extract import get_source_list