lstosa 0.10.18__py3-none-any.whl → 0.11.0__py3-none-any.whl

{lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lstosa
-Version: 0.10.18
+Version: 0.11.0
 Summary: Onsite analysis pipeline for the CTA LST-1
 Author: María Láinez, José Enrique Ruiz, Lab Saha, Andrés Baquero, José Luis Contreras, Maximilian Linhoff
 Author-email: Daniel Morcuende <dmorcuen@ucm.es>
@@ -18,8 +18,8 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: astropy ~=5.0
-Requires-Dist: lstchain >=0.10.13
+Requires-Dist: astropy <8,>=6.1
+Requires-Dist: lstchain >=0.11.0
 Requires-Dist: matplotlib
 Requires-Dist: numpy
 Requires-Dist: pandas
@@ -31,9 +31,8 @@ Requires-Dist: psutil
 Requires-Dist: click
 Requires-Dist: toml
 Requires-Dist: pymongo
-Requires-Dist: gammapy ~=1.1
+Requires-Dist: gammapy ~=1.3
 Requires-Dist: tenacity
-Requires-Dist: lstcam-calib
 Provides-Extra: all
 Requires-Dist: lstosa[dev,doc,test] ; extra == 'all'
 Provides-Extra: dev

lstosa-0.11.0.dist-info/RECORD

@@ -0,0 +1,84 @@
+osa/__init__.py,sha256=crotf1NMTfNdZuCua_5T_jk3kvZrAAwVw4FPrfxv994,193
+osa/_version.py,sha256=h33CazvO6X2vAgSb_I-BIOLNm7W5MVFnfSYRgLiRpMA,513
+osa/conftest.py,sha256=AdSHdWqjeMTgIeZEF8fdw6eSliySouivMWCPIqUV2os,24667
+osa/job.py,sha256=gLf-e7QRIvNhjGRINDTR5-k2PyYQqbEvPnUcg0DaVdw,27791
+osa/osadb.py,sha256=pkCuYbEG-moHG0uQHxwB7giQAv2XTld4HJ5gdn1F1hA,2422
+osa/paths.py,sha256=_3-avNABr7I2koiOo4MMDymi98l7JdIj1zUVbTjHdYg,18011
+osa/raw.py,sha256=ZNIsuqfx5ljoz_hwhSuafdKf-wr8-cxRJmel-A2endg,1337
+osa/report.py,sha256=sL2V7n8Y_UUaSDbWJY2o4UxDb4FU5AaFIRR8R25DB8o,4634
+osa/version.py,sha256=9T2TtuGBQeOy5PJDxMCeGlqx5baxLaq47VmFTDc09z8,796
+osa/veto.py,sha256=BTirpHLXbmkppJajFq6fxMuXtEKtIMv2NH-vYPScRSU,3099
+osa/configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/configs/config.py,sha256=cX0Vr4sorBVQ2KRqPIRSEGENKH_uC5lrgVHx1hp6YTk,1148
+osa/configs/datamodel.py,sha256=L_WRM91PBlMrtuE30akh7YR-56P0g9D994qzKSfhNJc,1950
+osa/configs/options.py,sha256=WbsyKhOs1Ud4Yt21O8KcISUvJYmhipa8vl_pxD_TZf4,558
+osa/configs/sequencer.cfg,sha256=VALe3KDii6UPOkEjI2ZsjV4nhswR8h_t4UUu85jWU4s,5624
+osa/high_level/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/high_level/selection_cuts.toml,sha256=ReSmcKtOPZY5JsZ9ExnxYdz7OrJEB8gghCbzHmeOyFg,128
+osa/high_level/significance.py,sha256=mMeG_y2wDnt0O2lSosYkSjaGZQl0V4GnuFrqKwSKSbE,9066
+osa/high_level/tests/test_significance.py,sha256=nTHgwnL2zkQJduJuWXC4J4N8DlAghOidQki7njZmpSI,572
+osa/nightsummary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/nightsummary/database.py,sha256=6-1Y0Rw67QQtp0ekoynX9CVpImfECEUO64sA3fw24og,4276
+osa/nightsummary/extract.py,sha256=vI_A_GrP6eQH8f2mr8OreUfrs5LwLNcH9KQLx_WikhQ,11498
+osa/nightsummary/nightsummary.py,sha256=rPEN_J-rJSgsoCR_ONaW4PB9vjJzZvHgw0a7sYOA7wE,2666
+osa/nightsummary/set_source_coordinates.py,sha256=e2UT_I_Epm8vte22TasIp28A3KRdcl4dgV4NjlIxwak,1579
+osa/nightsummary/tests/test_database.py,sha256=w5Fts-H0eTi1KPV6Of0V2ZOD849Ie14KRe4wFhc9Hh0,242
+osa/nightsummary/tests/test_extract.py,sha256=SJf0ld33UlK1d2jKweCQWNpGY9ZrIU1vFUdX4slnJps,1877
+osa/nightsummary/tests/test_nightsummary.py,sha256=0bgGJtlXzwclO2ma9I3X315X0X1bDkjqCaYcp3fay1s,987
+osa/nightsummary/tests/test_source_coordinates.py,sha256=doyF2e93RnFScPbG79aUmtHLCdGg-5oW8D5qqNDzivw,942
+osa/provenance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/provenance/capture.py,sha256=WATpVqHU3dIvxD64piqGd8PLlyN8syXn512omVNZrQs,22368
+osa/provenance/io.py,sha256=8VAVKmE2S3YHcxwDC8PJ4hO6ogfz08EflAeprLGkp3Y,9471
+osa/provenance/utils.py,sha256=7coX3-dM5xsJsYI2xXUSwIczBD9XdHZSyt8dcSRDoMM,9312
+osa/provenance/config/definition.yaml,sha256=6USKM92kjLpF8cy0TFl2d1WRjowjOmtCclN4oPL6-sY,20705
+osa/provenance/config/environment.yaml,sha256=kPM6ucPyLZLDFzkwFWoY0C6vmAArG98U-P1UAl89bgE,246
+osa/provenance/config/logger.yaml,sha256=hy_lH3DfbRFh2VM_iawI-c-3wE0cjTRHy465C2eFfnQ,510
+osa/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/scripts/autocloser.py,sha256=_NGhk_ehJUu1BaC-rkGziTgCeb_B8Eb_fqLccOEmznw,14386
+osa/scripts/calibration_pipeline.py,sha256=oWX2L9WIofVHp5qYsX3YMfwoslfiUgnXtWA2EI8KuG4,6056
+osa/scripts/closer.py,sha256=vhoe9wkvPvMc5O_kXqAiBuWeqm_KrBpYzgb9zs9ZORw,21718
+osa/scripts/copy_datacheck.py,sha256=rAJu5-5oJxspynjYqVWLDKnXTF7Ib9xtzSf3_DiIHsk,3212
+osa/scripts/datasequence.py,sha256=VKeZnA2OTl3PgtiTpYdw4GiB6gpbfWiqLdZSaBCmNyw,8592
+osa/scripts/gain_selection.py,sha256=PUG1J2V2SIboVKXJ-u4-h_XJyOGzfurazPu37M7lmhA,23238
+osa/scripts/gainsel_webmaker.py,sha256=40_DX7RUmImX-31iqRTFPEUvdVXXPSKp0xMEJWuAu80,5006
+osa/scripts/provprocess.py,sha256=Zv8sHTOgGNuEzu1QPUF142VQbHyfOsrxO073-xA6KG8,19107
+osa/scripts/reprocess_longterm.py,sha256=wMfc3UVwickkGFiviIhOlB9ebMIqQPWoUrgg8hQ78Lg,2138
+osa/scripts/reprocessing.py,sha256=A97kqX7QtD7ewoUvFhmSaQVVIfplViGhzTtOenTx37w,3630
+osa/scripts/sequencer.py,sha256=2t-fcD1pTvYV54vFSxxKID-L7pkRg7geXo-E43cnL6M,11927
+osa/scripts/sequencer_catB_tailcuts.py,sha256=x9iigitwHYe4ngsl5kyWjPkbYXenlmcfbPKHSxEFYuU,9514
+osa/scripts/sequencer_webmaker.py,sha256=a9okC-l9UMJHELTLLM2Q933-JELFOhXEbbH1Kbp1E8g,4936
+osa/scripts/show_run_summary.py,sha256=ofznLdoFHKQB-TELD6g5zMqD_TE9M9BuJR12zHvNdXE,2518
+osa/scripts/simulate_processing.py,sha256=D3b0kjyUS6Lm7pkQNJ-tJzQMveTwoPabfBBW80RMqN0,6828
+osa/scripts/update_source_catalog.py,sha256=Po4KSBOQCAT2Do1DUu5wnKV48Dq_pONfvtD47hh8fYI,8277
+osa/scripts/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/scripts/tests/test_osa_scripts.py,sha256=gyijYBzLQ19J1Az1mOdLm9bMxMwCkZryd78SfxCNtwg,15088
+osa/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/tests/test_jobs.py,sha256=ozrMBjHujDhEMzEvJE7WynChqjbBTcORHsHmFXozeWk,16112
+osa/tests/test_osa.py,sha256=QCOsjUgPuNMHoef3Ym2sDXVjun2LaBrfKyroAIH-os8,415
+osa/tests/test_osadb.py,sha256=pJHV1dxxblGH2sjS-JPDPTkMn-ew1MzbioCFyg7wbB8,1599
+osa/tests/test_paths.py,sha256=jd3bc2T7bwaY25aCef7w5q-9bUhxKOR327Z4Ch4e7eY,3161
+osa/tests/test_raw.py,sha256=XHe9al85Wwhuyk2a_6xrL9y7P11nr9MPs-t6VN328PY,1062
+osa/tests/test_report.py,sha256=OY-EsrXytoS6esfjUeLnIAmCMIw9EzoGD-elySafyhE,1365
+osa/tests/test_veto.py,sha256=UIsooji_5Z8TtAhc0UlD2VqheVd9DBufuMxinJ3e0w8,1066
+osa/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/utils/cliopts.py,sha256=KdGsezB8iqKxPn6s3K5-P1gxO-PRa_iM4ALl9qQgXcc,14805
+osa/utils/iofile.py,sha256=kJ7KB1suynhS2cTf7EeHwhMXq3delC_ls2HFpCzvsZo,2021
+osa/utils/logging.py,sha256=1WcNPjjslo3y25jcEY_fe0yXOeJ6frZrGLAy1GJpu_k,1491
+osa/utils/mail.py,sha256=uQfqPQdiOVHTvEAXr9H15a7-g9DtYVNKjMEb9GnI0oY,554
+osa/utils/register.py,sha256=h6JmCyRdL-6WaTVexq2ibS2rFN3j-bisHb-qZWz6g9s,6373
+osa/utils/utils.py,sha256=a--RvA0P2JHVkuyG8uBy1BSFINLi33Ek-rNZ9jg3Fs8,13514
+osa/utils/tests/test_iofile.py,sha256=e35_EqJerp-dEOrOqwXEUZCc5P_9llf2QfveltagfIk,399
+osa/utils/tests/test_utils.py,sha256=ybngpeUyzHgr96Gcx8r2g5oHFTo3aDQIwkC5aE9Ztic,2357
+osa/webserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/webserver/utils.py,sha256=ymB2wTzsFkPH4ebUcz_zK_zyolpnBzEbWGYwCvbaHf0,2155
+osa/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+osa/workflow/dl3.py,sha256=ZMXPrdJA0WOxDiHEW9sdM2vmYix8R3eSMTGc6o42yxg,9860
+osa/workflow/stages.py,sha256=ILg91VyNXcy--si7CpDa1UFRiugqIH6nKl10Ru2zZVc,7422
+osa/workflow/tests/test_dl3.py,sha256=81Vt4lNxNjdKi_ehzymqfFWFjncw7GuQcix9S0Yigaw,384
+osa/workflow/tests/test_stages.py,sha256=aslfOIjf-rvf3r9N7NtimaOKkVB6K1M3bidgHOzRkMs,3985
+lstosa-0.11.0.dist-info/LICENSE,sha256=h6iWot11EtMvaDaS_AvCHKLTNByO5wEbMyNj1c90y1c,1519
+lstosa-0.11.0.dist-info/METADATA,sha256=exZrcLYd0g_zvsij_tiLzKpmtKmiJ0ZIrkfpbjTWGkI,7302
+lstosa-0.11.0.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
+lstosa-0.11.0.dist-info/entry_points.txt,sha256=CzDKpLjZZQm8jJBxOVpMR8Czpgg_Yh-k6IPETp30VZE,1048
+lstosa-0.11.0.dist-info/top_level.txt,sha256=_Tj8zVHdrOoWZuuWTHbDpNofxW0imUmKdlXhnxsXJek,4
+lstosa-0.11.0.dist-info/RECORD,,

{lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.3.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{lstosa-0.10.18.dist-info → lstosa-0.11.0.dist-info}/entry_points.txt

@@ -11,6 +11,7 @@ provprocess = osa.scripts.provprocess:main
 reprocess_longterm = osa.scripts.reprocess_longterm:main
 reprocessing = osa.scripts.reprocessing:main
 sequencer = osa.scripts.sequencer:main
+sequencer_catB_tailcuts = osa.scripts.sequencer_catB_tailcuts:main
 sequencer_webmaker = osa.scripts.sequencer_webmaker:main
 show_run_summary = osa.scripts.show_run_summary:main
 show_run_summary_tcu = osa.scripts.show_run_summary_tcu:main

osa/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.10.18'
-__version_tuple__ = version_tuple = (0, 10, 18)
+__version__ = version = '0.11.0'
+__version_tuple__ = version_tuple = (0, 11, 0)

osa/configs/options.py

@@ -15,6 +15,8 @@ verbose = None
 warning = None
 nocheck = None
 no_dl2 = None
+no_dl1ab = None
+no_gainsel = None
 prod_id = None
 dl1_prod_id = None
 dl2_prod_id = None

osa/configs/sequencer.cfg

@@ -9,27 +9,34 @@
 BASE: test_osa/test_files0
 # The directories below can be left untouched.
 MONITORING: %(BASE)s/monitoring
-R0_DIR: %(BASE)s/R0
+R0_DIR: %(BASE)s/R0G
+RAW_R0_DIR: %(BASE)s/R0
 DRIVE_DIR: %(MONITORING)s/DrivePositioning
 RUN_SUMMARY_DIR: %(MONITORING)s/RunSummary
 RUN_CATALOG: %(MONITORING)s/RunCatalog
 PEDESTAL_FINDER_DIR: %(BASE)s/auxiliary/PedestalFinder
+TAILCUTS_FINDER_DIR: %(BASE)s/auxiliary/TailCuts
 ANALYSIS_DIR: %(BASE)s/running_analysis
-CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration/Cat-A
-CALIB_DIR: %(CALIB_BASE_DIR)s/calibration
-PEDESTAL_DIR: %(CALIB_BASE_DIR)s/drs4_baseline
+CALIB_BASE_DIR: %(MONITORING)s/PixelCalibration
+CAT_A_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-A
+CAT_A_CALIB_DIR: %(CAT_A_CALIB_BASE)s/calibration
+CAT_A_PEDESTAL_DIR: %(CAT_A_CALIB_BASE)s/drs4_baseline
+CAT_B_CALIB_BASE: %(CALIB_BASE_DIR)s/Cat-B
 DL1_DIR: %(BASE)s/DL1
 DL1AB_DIR: %(BASE)s/DL1
 DL2_DIR: %(BASE)s/DL2
 DL3_DIR: %(BASE)s/DL3
+RF_MODELS: %(BASE)s/models/AllSky
 OSA_DIR: %(BASE)s/OSA
 CLOSER_DIR: %(OSA_DIR)s/Closer
 HIGH_LEVEL_DIR: %(OSA_DIR)s/HighLevel
 LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm
 MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv
+SOURCE_CATALOG: %(OSA_DIR)s/Catalog
 SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb
 GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel
 GAIN_SELECTION_WEB_DIR: %(OSA_DIR)s/GainSelWeb
+CALIB_ENV: /fefs/aswg/software/conda/envs/lstcam-env
 
 # To be set by the user. Using PROD-ID will overcome the automatic
 # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol).
@@ -43,8 +50,9 @@ DL2_PROD_ID: model2
 # Calibration steps in calibration pipeline script
 #drs4_baseline: onsite_create_drs4_pedestal_file
 #charge_calibration: onsite_create_calibration_file
-drs4_baseline: lstcam_calib_onsite_create_drs4_pedestal_file
-charge_calibration: lstcam_calib_onsite_create_calibration_file
+drs4_baseline: onsite_create_drs4_pedestal_file
+charge_calibration: onsite_create_calibration_file
+catB_calibration: onsite_create_cat_B_calibration_file
 use_ff_heuristic_id: False
 
 # Data processing steps in datasequence script
@@ -52,17 +60,23 @@ r0_to_dl1: lstchain_data_r0_to_dl1
 dl1ab: lstchain_dl1ab
 check_dl1: lstchain_check_dl1
 dl1_to_dl2: lstchain_dl1_to_dl2
+tailcuts_finder: lstchain_find_tailcuts
 
 # To be set by the user
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+apply_catB_calibration: False
+apply_standard_dl1b_config: False
 use_ff_heuristic_gain_selection: False
+create_longterm_symlink: True
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
-rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3
+mc_prod: 20240918_v0.10.12_allsky_
 dl3_config: /software/lstchain/data/dl3_std_config.json
 max_tries: 3
+use_lstcam_env_for_CatA_calib: False
+use_lstcam_env_for_CatB_calib: False
 
 [MC]
 IRF_file: /path/to/irf.fits

osa/conftest.py

@@ -26,13 +26,12 @@ from osa.nightsummary.nightsummary import run_summary_table
 from osa.scripts.tests.test_osa_scripts import run_program
 from osa.utils.utils import date_to_dir
 from datetime import datetime
-import lstchain
 
 date = datetime.fromisoformat("2020-01-17")
 nightdir = date_to_dir(date)
 prod_id = "v0.1.0"
-dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
-dl2_prod_id = cfg.get("LST1", "DL2_PROD_ID")
+dl1_prod_id = "tailcut84"
+dl2_prod_id = "tailcut84/nsb_tuning_0.14"
 
 
 @pytest.fixture(scope="session")
@@ -87,14 +86,14 @@ def drive_log(monitoring_dir):
 
 @pytest.fixture(scope="session")
 def calibration_dir(calibration_base_dir):
-    directory = calibration_base_dir / "calibration" / nightdir / f"v{lstchain.__version__}"
+    directory = calibration_base_dir / "calibration" / nightdir / "v0.1.1"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
 
 @pytest.fixture(scope="session")
 def drs4_baseline_dir(calibration_base_dir):
-    directory = calibration_base_dir / "drs4_baseline" / nightdir / f"v{lstchain.__version__}"
+    directory = calibration_base_dir / "drs4_baseline" / nightdir / "v0.1.1"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 
@@ -146,6 +145,13 @@ def r0_dir(base_test_dir):
     return r0_directory
 
 
+@pytest.fixture(scope="session")
+def r0g_dir(base_test_dir):
+    r0_directory = base_test_dir / "R0G" / nightdir
+    r0_directory.mkdir(parents=True, exist_ok=True)
+    return r0_directory
+
+
 @pytest.fixture(scope="session")
 def r0_data(r0_dir):
     r0_files = []
@@ -156,6 +162,16 @@ def r0_data(r0_dir):
     return r0_files
 
 
+@pytest.fixture(scope="session")
+def r0g_data(r0g_dir):
+    r0g_files = []
+    for i in range(4, 8):
+        r0g_file = r0g_dir / f"LST-1.1.Run0180{i}.0000.fits.fz"
+        r0g_file.touch()
+        r0g_files.append(r0g_file)
+    return r0g_files
+
+
 @pytest.fixture(scope="session")
 def dl1b_subdir(running_analysis_dir):
     dl1ab_directory = running_analysis_dir / dl1_prod_id
@@ -412,6 +428,9 @@ def sequence_list(
     r0_data,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     """Creates a sequence list from a run summary file."""
     options.directory = running_analysis_dir
@@ -574,10 +593,131 @@ def database(osa_dir):
 
 
 @pytest.fixture(scope="session")
-def gain_selection_flag_file(osa_dir):
+def rf_models_allsky_basedir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
 
+@pytest.fixture(scope="session")
+def rf_model_path(rf_models_allsky_basedir):
+    mc_prod = "20240131_allsky_v0.10.5_all_dec_base"
+    declination_str = "dec_2276"
+    rf_model_path = rf_models_allsky_basedir / mc_prod / declination_str
+    rf_model_path.mkdir(parents=True, exist_ok=True)
+    return rf_model_path
+
+  
+@pytest.fixture(scope="session")
+def gain_selection_flag_file(osa_dir):
     GainSel_dir = osa_dir / "GainSel" / "20200117"
     GainSel_dir.mkdir(parents=True, exist_ok=True)
     file = GainSel_dir / "GainSelFinished.txt"
     file.touch()
     return file
+
+
+@pytest.fixture(scope="session")
+def catB_closed_file(running_analysis_dir):
+
+    catB_closed_file = running_analysis_dir / "catB_00003.closed"
+    catB_closed_file.touch()
+    return catB_closed_file
+
+
+@pytest.fixture(scope="session")
+def catB_calib_base_dir(monitoring_dir):
+
+    catB_calib_base_dir = monitoring_dir / "PixelCalibration" / "Cat-B"
+    catB_calib_base_dir.mkdir(parents=True, exist_ok=True)
+    return catB_calib_base_dir
+
+
+@pytest.fixture(scope="session")
+def catB_calibration_file(catB_calib_dir):
+
+    catB_calib_dir = catB_calib_base_dir / "calibration" / nightdir / prod_id
+    catB_calib_file = catB_calib_dir / "cat_B_calibration_filters_52.Run00003.h5"
+    catB_calib_file.touch()
+    return catB_calib_file
+
+
+@pytest.fixture(scope="session")
+def tailcuts_finder_dir(base_test_dir):
+    tailcuts_finder_dir = base_test_dir / "auxiliary" / "TailCuts"
+    tailcuts_finder_dir.mkdir(parents=True, exist_ok=True)
+    return tailcuts_finder_dir
+
+
+@pytest.fixture(scope="session")
+def dl1b_config_files(tailcuts_finder_dir):
+    config_information = dedent(
+        """\
+            {
+            "tailcuts_clean_with_pedestal_threshold": {
+                "picture_thresh": 8,
+                "boundary_thresh": 4,
+                "sigma": 2.5,
+                "keep_isolated_pixels": false,
+                "min_number_picture_neighbors": 2,
+                "use_only_main_island": false,
+                "delta_time": 2
+            },
+            "dynamic_cleaning": {
+                "apply": true,
+                "threshold": 267,
+                "fraction_cleaning_intensity": 0.03
+            }
+        }"""
+    )
+    config_file1 = tailcuts_finder_dir / "dl1ab_Run01807.json"
+    config_file1.touch()
+    config_file1.write_text(config_information)
+    config_file2 = tailcuts_finder_dir / "dl1ab_Run01808.json"
+    config_file2.touch()
+    config_file2.write_text(config_information)
+    config_file3 = tailcuts_finder_dir / "dl1ab_Run04185.json"
+    config_file3.touch()
+    config_file3.write_text(config_information)
+    return config_file1, config_file2, config_file3
+
+
+@pytest.fixture(scope="session")
+def tailcuts_log_files(tailcuts_finder_dir):
+    log_information = dedent(
+        """\
+    Median of 95% quantile of pedestal charge: 5.416 p.e.
+
+    Additional NSB rate (over dark MC): 0.2221 p.e./ns
+    lstchain_find_tailcuts finished successfully!
+    """
+    )
+    log_file1 = tailcuts_finder_dir / "log_find_tailcuts_Run01807.log"
+    log_file1.touch()
+    log_file1.write_text(log_information)
+    log_file2 = tailcuts_finder_dir / "log_find_tailcuts_Run01808.log"
+    log_file2.touch()
+    log_file2.write_text(log_information)
+    log_file3 = tailcuts_finder_dir / "log_find_tailcuts_Run04185.log"
+    log_file3.touch()
+    log_file3.write_text(log_information)
+    return log_file1, log_file2, log_file3
+
+
+@pytest.fixture(scope="session")
+def rf_models_base_dir(base_test_dir):
+    directory = base_test_dir / "models/AllSky"
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def rf_models(rf_models_base_dir):
+    rf_models_prefix = cfg.get("lstchain", "mc_prod")
+    rf_models_path1 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.00"
+    rf_models_path1.mkdir(parents=True, exist_ok=True)
+    rf_models_path2 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_2276"
+    rf_models_path2.mkdir(parents=True, exist_ok=True)
+    rf_models_path3 = rf_models_base_dir / f"{rf_models_prefix}nsb_tuning_0.14/dec_4822"
+    rf_models_path3.mkdir(parents=True, exist_ok=True)
+    return rf_models_path1, rf_models_path2, rf_models_path3

osa/high_level/significance.py

@@ -26,7 +26,8 @@ from osa.configs import options
 from osa.configs.config import cfg
 from osa.nightsummary.extract import get_source_list
 from osa.paths import DEFAULT_CFG, destination_dir, analysis_path
-from osa.utils.cliopts import get_prod_id, get_dl2_prod_id
+from osa.utils.cliopts import get_prod_id
+from osa.paths import get_dl2_prod_id
 from osa.utils.logging import myLogger
 from osa.utils.utils import date_to_dir, YESTERDAY
 
@@ -205,11 +206,12 @@ def main(
     log.debug(f"Config: {config.resolve()}")
 
     # Initial setup of global parameters
+    if simulate:
+        options.simulate = True
     options.date = date
     flat_date = date_to_dir(date)
     options.tel_id = telescope
     options.prod_id = get_prod_id()
-    options.dl2_prod_id = get_dl2_prod_id()
     options.directory = analysis_path(options.tel_id)
     dl2_directory = Path(cfg.get("LST1", "DL2_DIR"))
     highlevel_directory = destination_dir("HIGH_LEVEL", create_dir=True)
@@ -232,7 +234,7 @@ def main(
                 dl2_directory
                 / flat_date
                 / options.prod_id
-                / options.dl2_prod_id
+                / get_dl2_prod_id(run)
                 / f"dl2_LST-1.Run{run:05d}.h5"
             )
             df = pd.concat([df, pd.read_hdf(input_file, key=dl2_params_lstcam_key)])

osa/high_level/tests/test_significance.py

@@ -10,6 +10,9 @@ def test_significance(
     systematic_correction_files,
     pedestal_ids_file,
     merged_run_summary,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     output = sp.run(
         ["theta2_significance", "-d", "2020-01-17", "-s", "LST1"],

osa/job.py

@@ -5,6 +5,7 @@ import logging
 import shutil
 import subprocess as sp
 import time
+import re
 from io import StringIO
 from pathlib import Path
 from textwrap import dedent
@@ -20,10 +21,16 @@ from osa.paths import (
     get_drive_file,
     get_summary_file,
     get_pedestal_ids_file,
+    get_dl1_prod_id_and_config,
 )
 from osa.utils.iofile import write_to_file
 from osa.utils.logging import myLogger
-from osa.utils.utils import date_to_dir, time_to_seconds, stringify, date_to_iso
+from osa.utils.utils import (
+    date_to_dir,
+    time_to_seconds,
+    stringify,
+    date_to_iso,
+)
 
 log = myLogger(logging.getLogger(__name__))
 
@@ -92,7 +99,15 @@ def are_all_jobs_correctly_finished(sequence_list):
     flag = True
     analysis_directory = Path(options.directory)
     for sequence in sequence_list:
-        history_files_list = analysis_directory.rglob(f"*{sequence.seq}*.history")
+        history_files_list = analysis_directory.rglob(f"*{sequence.run}*.history")
+        
+        if not options.test:
+            try:
+                next(history_files_list)
+            except StopIteration:
+                log.debug("No history files found.")
+                flag = False
+
         for history_file in history_files_list:
             # TODO: s.history should be SubRunObj attribute not RunObj
             # s.history only working for CALIBRATION sequence (run-wise), since it is
@@ -103,10 +118,11 @@ def are_all_jobs_correctly_finished(sequence_list):
             if out == 0:
                 log.debug(f"Job {sequence.seq} ({sequence.type}) correctly finished")
                 continue
-            if out == 1 and options.no_dl2:
+
+            if out == 2 and options.no_dl1ab:
                 log.debug(
                     f"Job {sequence.seq} ({sequence.type}) correctly "
-                    f"finished up to DL1ab, but --no-dl2 option selected"
+                    f"finished up to DL1A, but --no-dl1ab option selected"
                 )
                 continue
 
@@ -166,10 +182,9 @@ def historylevel(history_file: Path, data_type: str):
      - Sequence completed when reaching level 0
 
     Workflow for DATA sequences:
-     - R0->DL1, level 4->3
-     - DL1->DL1AB, level 3->2
-     - DATACHECK, level 2->1
-     - DL1->DL2, level 1->0
+     - R0->DL1, level 3->2
+     - DL1->DL1AB, level 2->1
+     - DATACHECK, level 1->0
      - Sequence completed when reaching level 0
 
     Parameters
@@ -188,7 +203,7 @@ def historylevel(history_file: Path, data_type: str):
     #  into account not only the last history line but also the others.
 
     if data_type == "DATA":
-        level = 4
+        level = 3
     elif data_type == "PEDCALIB":
         level = 2
     else:
@@ -197,6 +212,11 @@ def historylevel(history_file: Path, data_type: str):
     exit_status = 0
 
     if history_file.exists():
+        if data_type == "DATA":
+            match = re.search(r"sequence_LST1_(\d+)\.\d+", str(history_file))
+        elif data_type == "PEDCALIB":
+            match = re.search(r"sequence_LST1_(\d+)\.history", str(history_file))
+        run_id = int(match.group(1)) 
         for line in history_file.read_text().splitlines():
             words = line.split()
             try:
@@ -214,24 +234,18 @@ def historylevel(history_file: Path, data_type: str):
                     level = 0 if exit_status == 0 else 1
                 # Data sequence
                 elif program == cfg.get("lstchain", "r0_to_dl1"):
-                    level = 3 if exit_status == 0 else 4
+                    level = 2 if exit_status == 0 else 3
                 elif program == cfg.get("lstchain", "dl1ab"):
-                    if (exit_status == 0) and (prod_id == options.dl1_prod_id):
-                        log.debug(f"DL1ab prod ID: {options.dl1_prod_id} already produced")
-                        level = 2
+                    dl1_prod_id = get_dl1_prod_id_and_config(run_id)[0]
+                    if (exit_status == 0) and (prod_id == dl1_prod_id):
+                        log.debug(f"DL1ab prod ID: {dl1_prod_id} already produced")
+                        level = 1
                     else:
-                        level = 3
-                        log.debug(f"DL1ab prod ID: {options.dl1_prod_id} not produced yet")
+                        level = 2
+                        log.debug(f"DL1ab prod ID: {dl1_prod_id} not produced yet")
                         break
                 elif program == cfg.get("lstchain", "check_dl1"):
-                    level = 1 if exit_status == 0 else 2
-                elif program == cfg.get("lstchain", "dl1_to_dl2"):
-                    if (exit_status == 0) and (prod_id == options.dl2_prod_id):
-                        log.debug(f"DL2 prod ID: {options.dl2_prod_id} already produced")
-                        level = 0
-                    else:
-                        level = 1
-                        log.debug(f"DL2 prod ID: {options.dl2_prod_id} not produced yet")
+                    level = 0 if exit_status == 0 else 1
 
                 else:
                     log.warning(f"Program name not identified: {program}")
@@ -423,8 +437,8 @@ def data_sequence_job_template(sequence):
         commandargs.append("-s")
     if options.configfile:
         commandargs.extend(("--config", f"{Path(options.configfile).resolve()}"))
-    if sequence.type == "DATA" and options.no_dl2:
-        commandargs.append("--no-dl2")
+    if sequence.type == "DATA" and options.no_dl1ab:
+        commandargs.append("--no-dl1ab")
 
     commandargs.extend(
         (
@@ -439,6 +453,14 @@ def data_sequence_job_template(sequence):
         )
     )
 
+    if not options.no_dl1ab:
+        dl1_prod_id, dl1b_config = get_dl1_prod_id_and_config(sequence.run)
+        sequence.dl1_prod_id = dl1_prod_id
+        sequence.dl1b_config = dl1b_config
+
+        commandargs.append(f"--dl1b-config={sequence.dl1b_config}")
+        commandargs.append(f"--dl1-prod-id={sequence.dl1_prod_id}")
+
     content = job_header + "\n" + PYTHON_IMPORTS
 
     if not options.test:
@@ -493,7 +515,10 @@ def calibration_sequence_job_template(sequence):
     # Get the job header template.
     job_header = job_header_template(sequence)
 
-    commandargs = ["calibration_pipeline"]
+    if cfg.getboolean("lstchain", "use_lstcam_env_for_CatA_calib"):
+        commandargs = ["conda", "run", "-n", "lstcam-env", "calibration_pipeline"]
+    else:
+        commandargs = ["calibration_pipeline"]
 
     if options.verbose:
         commandargs.append("-v")
@@ -727,6 +752,7 @@ def get_closer_sacct_output(sacct_output) -> pd.DataFrame:
         | (sacct_output["JobName"].str.contains("lstchain_longterm_dl1_check"))
         | (sacct_output["JobName"].str.contains("lstchain_cherenkov_transparency"))
         | (sacct_output["JobName"].str.contains("provproces"))
+        | (sacct_output["JobName"].str.contains("lstchain_dl1_to_dl2"))
     ]
 
     try: