lstosa 0.10.18__py3-none-any.whl → 0.10.19__py3-none-any.whl

osa/scripts/tests/test_osa_scripts.py

@@ -30,7 +30,6 @@ ALL_SCRIPTS = [
 options.date = datetime.datetime.fromisoformat("2020-01-17")
 options.tel_id = "LST1"
 options.prod_id = "v0.1.0"
-options.dl1_prod_id = "tailcut84"
 options.directory = "test_osa/test_files0/running_analysis/20200117/v0.1.0/"
 
 
@@ -65,7 +64,11 @@ def test_simulate_processing(
     run_summary_file,
     r0_data,
     merged_run_summary,
-    drive_log
+    drive_log,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
+    tailcuts_finder_dir,
 ):
 
     for file in drs4_time_calibration_files:
@@ -80,13 +83,16 @@ def test_simulate_processing(
     assert run_summary_file.exists()
     assert merged_run_summary.exists()
     assert drive_log.exists()
+    assert rf_models[1].exists()
+    assert dl1b_config_files[0].exists()
+    assert tailcuts_log_files[0].exists()
 
     remove_provlog()
-    rc = run_program("simulate_processing", "-p", "--force")
+    rc = run_program("simulate_processing", "-p", "--force", "-d", "2020-01-17", "LST1")
     assert rc.returncode == 0
 
     prov_dl1_path = Path("./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/log")
-    prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/model2/log")
+    prov_dl2_path = Path("./test_osa/test_files0/DL2/20200117/v0.1.0/tailcut84/nsb_tuning_0.14/log")
     prov_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.log"
     prov_file_dl2 = prov_dl2_path / "calibration_to_dl2_01807_prov.log"
     json_file_dl1 = prov_dl1_path / "calibration_to_dl1_01807_prov.json"
@@ -101,23 +107,23 @@ def test_simulate_processing(
 
     with open(json_file_dl1) as file:
         dl1 = yaml.safe_load(file)
-    assert len(dl1["entity"]) == 19
+    assert len(dl1["entity"]) == 42
     assert len(dl1["activity"]) == 5
     assert len(dl1["used"]) == 15
     assert len(dl1["wasGeneratedBy"]) == 10
 
     with open(json_file_dl2) as file:
         dl2 = yaml.safe_load(file)
-    assert len(dl2["entity"]) == 25
-    assert len(dl2["activity"]) == 6
-    assert len(dl2["used"]) == 21
-    assert len(dl2["wasGeneratedBy"]) == 12
+    assert len(dl2["entity"]) == 42
+    assert len(dl2["activity"]) == 5
+    assert len(dl2["used"]) == 15
+    assert len(dl2["wasGeneratedBy"]) == 10
 
-    rc = run_program("simulate_processing", "-p")
+    rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
     assert rc.returncode == 0
 
     remove_provlog()
-    rc = run_program("simulate_processing", "-p")
+    rc = run_program("simulate_processing", "-p", "-d", "2020-01-17", "LST1")
     assert rc.returncode == 0
 
 
@@ -129,6 +135,10 @@ def test_simulated_sequencer(
     r0_data,
     merged_run_summary,
     gain_selection_flag_file,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
+    dl2_merged,
 ):
     assert run_summary_file.exists()
     assert run_catalog.exists()
@@ -143,7 +153,10 @@ def test_simulated_sequencer(
     for file in systematic_correction_files:
         assert file.exists()
 
-    rc = run_program("sequencer", "-d", "2020-01-17", "-s", "-t", "LST1")
+    for file in dl2_merged:
+        assert file.exists()
+
+    rc = run_program("sequencer", "-d", "2020-01-17", "--no-gainsel", "-s", "-t", "LST1")
 
     assert rc.returncode == 0
     now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M")
@@ -152,8 +165,8 @@ def test_simulated_sequencer(
         =================================== Starting sequencer.py at {now} UTC for LST, Telescope: LST1, Date: 2020-01-17 ===================================
         Tel   Seq  Parent  Type      Run   Subruns  Source        Action  Tries  JobID  State  CPU_time  Exit  DL1%  MUONS%  DL1AB%  DATACHECK%  DL2%  
         LST1    1  None    PEDCALIB  1809  5        None          None    None   None   None   None      None  None  None    None    None        None  
-        LST1    2       1  DATA      1807  11       Crab          None    None   None   None   None      None     0       0       0           0     0  
-        LST1    3       1  DATA      1808  9        MadeUpSource  None    None   None   None   None      None     0       0       0           0     0  
+        LST1    2       1  DATA      1807  11       Crab          None    None   None   None   None      None     0       0       0           0   100  
+        LST1    3       1  DATA      1808  9        MadeUpSource  None    None   None   None   None      None     0       0       0           0   100  
         """  # noqa: E501
     )
 
@@ -186,6 +199,9 @@ def test_closer(
     longterm_dir,
     longterm_link_latest_dir,
     daily_datacheck_dl1_files,
+    dl1b_config_files,
+    tailcuts_log_files,
+    rf_models,
 ):
     # First assure that the end of night flag is not set and remove it otherwise
     night_finished_flag = Path(
@@ -209,6 +225,7 @@ def test_closer(
     assert longterm_link_latest_dir.exists()
     for check_file in daily_datacheck_dl1_files:
         assert check_file.exists()
+    assert rf_models[2].exists()
 
     run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1")
     closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed"
@@ -227,9 +244,6 @@ def test_closer(
         "./test_osa/test_files0/DL1/20200117/v0.1.0/tailcut84/datacheck/"
         "datacheck_dl1_LST-1.Run01808.0011.h5"
     )
-    assert os.path.exists(
-        "./test_osa/test_files0/DL2/20200117/v0.1.0/model2/dl2_LST-1.Run01808.0011.h5"
-    )
     # Assert that the link to dl1 and muons files have been created
     assert os.path.islink(
         "./test_osa/test_files0/running_analysis/20200117/v0.1.0/muons_LST-1.Run01808.0011.fits"
@@ -242,7 +256,16 @@ def test_closer(
     assert closed_seq_file.exists()
 
 
-def test_datasequence(running_analysis_dir):
+def test_datasequence(
+    running_analysis_dir,
+    run_catalog,
+    run_catalog_dir,
+    rf_models_base_dir,
+    rf_models,
+    catB_closed_file,
+    dl1b_config_files,
+    tailcuts_log_files,
+):
     drs4_file = "drs4_pedestal.Run00001.0000.fits"
     calib_file = "calibration.Run00002.0000.hdf5"
     timecalib_file = "time_calibration.Run00002.0000.hdf5"
@@ -250,9 +273,16 @@ def test_datasequence(running_analysis_dir):
     drive_file = "DrivePosition_20200117.txt"
     runsummary_file = "RunSummary_20200117.ecsv"
     prod_id = "v0.1.0"
-    run_number = "00003.0000"
+    run_number = "01807.0000"
     options.directory = running_analysis_dir
 
+    assert run_catalog_dir.exists()
+    assert run_catalog.exists()
+    assert rf_models_base_dir.exists()
+    assert rf_models[1].exists()
+    assert catB_closed_file.exists()
+    assert dl1b_config_files[0].exists()
+
     output = run_program(
         "datasequence",
         "--date=2020-01-17",
@@ -264,6 +294,8 @@ def test_datasequence(running_analysis_dir):
         f"--systematic-correction-file={systematic_correction_file}",
         f"--drive-file={drive_file}",
         f"--run-summary={runsummary_file}",
+        f"--dl1b-config={dl1b_config_files[0]}",
+        "--dl1-prod-id=tailcut84",
         run_number,
         "LST1",
     )
@@ -288,7 +320,15 @@ def test_calibration_pipeline(running_analysis_dir):
     assert output.returncode == 0
 
 
-def test_is_sequencer_successful(run_summary, running_analysis_dir):
+def test_is_sequencer_successful(
+        run_summary,
+        running_analysis_dir,
+        dl1b_config_files,
+        tailcuts_log_files,
+        rf_models,
+        merged_run_summary,
+    ):
+    assert merged_run_summary.exists()
     options.directory = running_analysis_dir
     options.test = True
     seq_tuple = is_finished_check(run_summary)
@@ -300,12 +340,14 @@ def test_drs4_pedestal_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import drs4_pedestal_command
 
     cmd = drs4_pedestal_command(drs4_pedestal_run_id="01804")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         cfg.get("lstchain", "drs4_baseline"),
         "-r",
         "01804",
         "-b",
         base_test_dir,
+        f"--r0-dir={r0_dir}",
         "--no-progress",
     ]
     assert cmd == expected_command
@@ -315,6 +357,7 @@ def test_calibration_file_cmd(base_test_dir):
     from osa.scripts.calibration_pipeline import calibration_file_command
 
     cmd = calibration_file_command(drs4_pedestal_run_id="01804", pedcal_run_id="01809")
+    r0_dir = base_test_dir / "R0"
     expected_command = [
         cfg.get("lstchain", "charge_calibration"),
         "-p",
@@ -323,6 +366,7 @@ def test_calibration_file_cmd(base_test_dir):
         "01809",
         "-b",
         base_test_dir,
+        f"--r0-dir={r0_dir}",
     ]
     assert cmd == expected_command
 

osa/scripts/update_source_catalog.py

@@ -12,7 +12,7 @@ from astropy.time import Time
 from lstchain.io.io import dl1_params_lstcam_key
 from lstchain.reco.utils import add_delta_t_key, get_effective_time
 
-from osa.paths import get_major_version
+from osa.paths import get_major_version, get_dl1_prod_id
 from osa.utils.utils import get_lstchain_version
 
 pd.set_option("display.float_format", "{:.1f}".format)
@@ -24,6 +24,7 @@ log = logging.getLogger(__name__)
 BASE_DL1 = Path("/fefs/aswg/data/real/DL1")
 BASE_MONITORING = Path("/fefs/aswg/data/real/monitoring")
 CATALOG_DIR = Path("/fefs/aswg/data/real/OSA/Catalog")
+TAILCUTS_DIR = Path("/fefs/aswg/data/real/auxiliary/TailCuts")
 
 
 def add_table_to_html(html_table):
@@ -125,7 +126,9 @@ def add_start_and_elapsed(table: Table, datedir: str, version: str) -> None:
 
     for run in table["run_id"]:
         major_version = get_major_version(version)
-        file = BASE_DL1 / datedir / major_version / f"tailcut84/dl1_LST-1.Run{run:05d}.h5"
+        dl1b_config_file = TAILCUTS_DIR / f"dl1ab_Run{run:05d}.json"
+        dl1_prod_id = get_dl1_prod_id(dl1b_config_file)
+        file = BASE_DL1 / datedir / major_version / dl1_prod_id / f"dl1_LST-1.Run{run:05d}.h5"
         df = pd.read_hdf(file, key=dl1_params_lstcam_key)
 
         # Timestamp of the first event

osa/tests/test_jobs.py

@@ -9,18 +9,21 @@ from osa.configs.config import cfg, DEFAULT_CFG
 
 extra_files = Path(os.getenv("OSA_TEST_DATA", "extra"))
 datasequence_history_file = extra_files / "history_files/sequence_LST1_04185.0010.history"
+datasequence_history_file2 = extra_files / "history_files/sequence_LST1_04185.0001.history"
 calibration_history_file = extra_files / "history_files/sequence_LST1_04183.history"
 options.date = "2020-01-17"
 options.tel_id = "LST1"
 options.prod_id = "v0.1.0"
 
 
-def test_historylevel():
+def test_historylevel(
+        run_catalog,
+        dl1b_config_files,
+        tailcuts_log_files,
+        rf_models,
+):
     from osa.job import historylevel
 
-    options.dl1_prod_id = "tailcut84"
-    options.dl2_prod_id = "model1"
-
     level, rc = historylevel(datasequence_history_file, "DATA")
     assert level == 0
     assert rc == 0
@@ -29,11 +32,8 @@ def test_historylevel():
     assert level == 0
     assert rc == 0
 
-    options.dl1_prod_id = "tailcut84"
-    options.dl2_prod_id = "model2"
-
-    level, rc = historylevel(datasequence_history_file, "DATA")
-    assert level == 1
+    level, rc = historylevel(datasequence_history_file2, "DATA")
+    assert level == 2
     assert rc == 0
 
 
@@ -41,6 +41,7 @@ def test_preparejobs(running_analysis_dir, sequence_list):
     from osa.job import prepare_jobs
 
     options.simulate = False
+    options.test = True
     options.directory = running_analysis_dir
     prepare_jobs(sequence_list)
     expected_calib_script = os.path.join(running_analysis_dir, "sequence_LST1_01809.py")
@@ -94,6 +95,7 @@ def test_job_header_template(sequence_list, running_analysis_dir):
     from osa.job import job_header_template
 
     # Extract the first sequence
+    options.test = False
     first_sequence = sequence_list[0]
     header = job_header_template(first_sequence)
     output_string1 = dedent(
@@ -138,13 +140,17 @@ def test_create_job_template_scheduler(
     calibration_file,
     run_summary_file,
     pedestal_ids_file,
+    dl1b_config_files,
+    rf_models,
 ):
     from osa.job import data_sequence_job_template
 
     assert pedestal_ids_file.exists()
+    assert rf_models[1].exists()
 
     options.test = False
     options.simulate = False
+
     content1 = data_sequence_job_template(sequence_list[1])
     expected_content1 = dedent(
         f"""\
@@ -184,6 +190,8 @@ def test_create_job_template_scheduler(
             '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
             '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
             '--run-summary={run_summary_file}',
+            '--dl1b-config={dl1b_config_files[0]}',
+            '--dl1-prod-id=tailcut84',
             f'01807.{{subruns:04d}}',
             'LST1'
         ])
@@ -230,6 +238,8 @@ def test_create_job_template_scheduler(
                 '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
                 '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
                 '--run-summary={run_summary_file}',
+                '--dl1b-config={dl1b_config_files[1]}',
+                '--dl1-prod-id=tailcut84',
                 f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
                 f'01808.{{subruns:04d}}',
                 'LST1'
@@ -252,6 +262,8 @@ def test_create_job_template_local(
     run_summary_file,
     pedestal_ids_file,
     r0_data,
+    dl1b_config_files,
+    rf_models,
 ):
     """Check the job file in local mode (assuming no scheduler)."""
     from osa.job import data_sequence_job_template
@@ -266,6 +278,7 @@ def test_create_job_template_local(
         assert file.exists()
 
     assert pedestal_ids_file.exists()
+    assert rf_models[0].exists()
 
     options.test = True
     options.simulate = False
@@ -296,6 +309,8 @@ def test_create_job_template_local(
             '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
             '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
             '--run-summary={run_summary_file}',
+            '--dl1b-config={dl1b_config_files[0]}',
+            '--dl1-prod-id=tailcut84',
             f'01807.{{subruns:04d}}',
             'LST1'
         ])
@@ -329,6 +344,8 @@ def test_create_job_template_local(
                 '--systematic-correction-file={Path.cwd()}/test_osa/test_files0/monitoring/PixelCalibration/Cat-A/ffactor_systematics/20200725/pro/ffactor_systematics_20200725.h5',
                 '--drive-file={Path.cwd()}/test_osa/test_files0/monitoring/DrivePositioning/DrivePosition_log_20200117.txt',
                 '--run-summary={run_summary_file}',
+                '--dl1b-config={dl1b_config_files[1]}',
+                '--dl1-prod-id=tailcut84',
                 f'--pedestal-ids-file={Path.cwd()}/test_osa/test_files0/auxiliary/PedestalFinder/20200117/pedestal_ids_Run01808.{{subruns:04d}}.h5',
                 f'01808.{{subruns:04d}}',
                 'LST1'
@@ -337,11 +354,11 @@ def test_create_job_template_local(
         sys.exit(proc.returncode)"""
     )
 
-    options.simulate = True
-
     assert content1 == expected_content1
     assert content2 == expected_content2
 
+    options.simulate = True
+
 
 def test_create_job_scheduler_calibration(sequence_list):
     """Check the pilot job file for the calibration pipeline."""

osa/tests/test_paths.py

@@ -52,8 +52,8 @@ def test_destination_dir():
     from osa.paths import destination_dir
 
     datedir = date_to_dir(options.date)
-    options.dl1_prod_id = cfg.get("LST1", "DL1_PROD_ID")
-    options.dl2_prod_id = cfg.get("LST1", "DL2_PROD_ID")
+    dl1_prod_id = "tailcut84"
+    dl2_prod_id = "tailcut84/nsb_tuning_0.14"
     options.prod_id = cfg.get("LST1", "PROD_ID")
     base_directory = cfg.get("LST1", "BASE")
     base_path = Path(base_directory)
@@ -67,14 +67,14 @@ def test_destination_dir():
     }
 
     for concept, dst_dir in data_types.items():
-        directory = destination_dir(concept, create_dir=False)
+        directory = destination_dir(concept, create_dir=False, dl1_prod_id=dl1_prod_id, dl2_prod_id=dl2_prod_id)
         if concept == "DL1AB":
             expected_directory = (
-                base_path / dst_dir / datedir / options.prod_id / options.dl1_prod_id
+                base_path / dst_dir / datedir / options.prod_id / dl1_prod_id
             )
         elif concept == "DATACHECK":
             expected_directory = (
-                base_path / dst_dir / datedir / options.prod_id / options.dl1_prod_id / "datacheck"
+                base_path / dst_dir / datedir / options.prod_id / dl1_prod_id / "datacheck"
             )
         elif concept == "MUON":
             expected_directory = (
@@ -86,7 +86,7 @@ def test_destination_dir():
             )
         elif concept == "DL2":
             expected_directory = (
-                base_path / dst_dir / datedir / options.prod_id / options.dl2_prod_id
+                base_path / dst_dir / datedir / options.prod_id / dl2_prod_id
             )
 
         assert directory == expected_directory

osa/utils/cliopts.py

@@ -6,15 +6,11 @@ from argparse import ArgumentParser
 from pathlib import Path
 
 from osa.configs import options
-from osa.configs.config import cfg
 from osa.paths import analysis_path, DEFAULT_CFG
 from osa.utils.logging import myLogger
 from osa.utils.utils import (
-    get_dl1_prod_id,
-    get_dl2_prod_id,
     get_prod_id,
     is_defined,
-    set_prod_ids,
     YESTERDAY,
 )
 
@@ -32,8 +28,6 @@ __all__ = [
     "sequencer_webmaker_argparser",
     "valid_date",
     "get_prod_id",
-    "get_dl1_prod_id",
-    "get_dl2_prod_id",
     "calibration_pipeline_cliparsing",
     "calibration_pipeline_argparser",
     "autocloser_cli_parser",
@@ -130,7 +124,7 @@ def closercliparsing():
     # setting the default date and directory if needed
     options.date = set_default_date_if_needed()
     options.directory = analysis_path(options.tel_id)
-    set_prod_ids()
+    options.prod_id = get_prod_id()
 
 
 def calibration_pipeline_argparser():
@@ -184,10 +178,10 @@ def data_sequence_argparser():
         help="Set the prod ID to define data directories",
     )
     parser.add_argument(
-        "--no-dl2",
+        "--no-dl1ab",
         action="store_true",
         default=False,
-        help="Do not produce DL2 files (default False)",
+        help="Do not launch the script lstchain_dl1ab (default False)",
     )
     parser.add_argument("--pedcal-file", type=Path, help="Path of the calibration file")
     parser.add_argument("--drs4-pedestal-file", type=Path, help="Path of the DRS4 pedestal file")
@@ -210,6 +204,18 @@ def data_sequence_argparser():
         type=Path,
         help="Path to a file containing the ids of the interleaved pedestal events",
     )
+    parser.add_argument(
+        "--dl1b-config",
+        type=Path,
+        default=None,
+        help="Configuration file for the production of DL1b files"
+    )
+    parser.add_argument(
+        "--dl1-prod-id",
+        type=str,
+        default=None,
+        help="Production id of the DL1b files"
+    )
     parser.add_argument("run_number", help="Number of the run to be processed")
     parser.add_argument("tel_id", choices=["ST", "LST1", "LST2"])
     return parser
@@ -226,7 +232,7 @@ def data_sequence_cli_parsing():
     options.verbose = opts.verbose
     options.simulate = opts.simulate
     options.prod_id = opts.prod_id
-    options.no_dl2 = opts.no_dl2
+    options.no_dl1ab = opts.no_dl1ab
     options.tel_id = opts.tel_id
 
     log.debug(f"The options and arguments are {opts}")
@@ -234,8 +240,7 @@ def data_sequence_cli_parsing():
     # setting the default date and directory if needed
     options.date = set_default_date_if_needed()
     options.directory = analysis_path(options.tel_id)
-
-    set_prod_ids()
+    options.prod_id = get_prod_id()
 
     return (
         opts.pedcal_file,
@@ -246,6 +251,8 @@ def data_sequence_cli_parsing():
         opts.run_summary,
         opts.pedestal_ids_file,
         opts.run_number,
+        opts.dl1b_config,
+        opts.dl1_prod_id,
     )
 
 
@@ -269,10 +276,10 @@ def sequencer_argparser():
         "calibration products already produced (default False)",
     )
     parser.add_argument(
-        "--no-dl2",
+        "--no-dl1ab",
         action="store_true",
         default=False,
-        help="Do not produce DL2 files (default False)",
+        help="Do not launch the script lstchain_dl1ab (default False)",
     )
     parser.add_argument(
         "--no-gainsel",
@@ -304,13 +311,13 @@ def sequencer_cli_parsing():
     set_common_globals(opts)
     options.no_submit = opts.no_submit
     options.no_calib = opts.no_calib
-    options.no_dl2 = opts.no_dl2
+    options.no_dl1ab = opts.no_dl1ab
     options.no_gainsel = opts.no_gainsel
     options.force_submit = opts.force_submit
 
     log.debug(f"the options are {opts}")
 
-    set_prod_ids()
+    options.prod_id = get_prod_id()
 
     # setting the default date and directory if needed
     options.date = set_default_date_if_needed()
@@ -352,7 +359,7 @@ def provprocess_argparser():
     )
     parser.add_argument("pedcal_run_id", help="Number of the used pedcal used in the calibration")
     parser.add_argument("run", help="Number of the run whose provenance is to be extracted")
-    parser.add_argument("date", action="store", type=str, help="Observation starting date YYYYMMDD")
+    parser.add_argument("date", action="store", type=valid_date, help="Date (YYYY-MM-DD) of the start of the night")
     parser.add_argument("prod_id", action="store", type=str, help="Production ID")
 
     return parser
@@ -376,7 +383,8 @@ def provprocessparsing():
     options.filter = opts.filter
     options.quit = opts.quit
     options.no_dl2 = opts.no_dl2
-    set_prod_ids()
+    options.prod_id = get_prod_id()
+    options.tel_id = "LST1"
 
 
 def simproc_argparser():
@@ -402,15 +410,15 @@ def simproc_argparser():
         dest="append",
         help="append provenance capture to existing prov.log file",
     )
-    # parser.add_argument(
-    #     "-d",
-    #     "--date",
-    #     action="store",
-    #     type=str,
-    #     dest="date",
-    #     help="observation ending date YYYY-MM-DD [default today]",
-    # )
-    # parser.add_argument("tel_id", choices=["ST", "LST1", "LST2"])
+    parser.add_argument(
+         "-d",
+         "--date",
+         action="store",
+         type=valid_date,
+         dest="date",
+         help="observation ending date YYYY-MM-DD [default today]",
+    )
+    parser.add_argument("tel_id", choices=["ST", "LST1", "LST2"])
 
     return parser
 
@@ -424,6 +432,8 @@ def simprocparsing():
     options.provenance = opts.provenance
     options.force = opts.force
     options.append = opts.append
+    options.date = opts.date
+    options.tel_id = opts.tel_id
 
 
 def copy_datacheck_argparser():
@@ -445,11 +455,6 @@ def copy_datacheck_parsing():
     options.directory = analysis_path(options.tel_id)
     options.prod_id = get_prod_id()
 
-    if cfg.get("LST1", "DL1_PROD_ID") is not None:
-        options.dl1_prod_id = get_dl1_prod_id()
-    else:
-        options.dl1_prod_id = options.prod_id
-
 
 def sequencer_webmaker_argparser():
     parser = ArgumentParser(