ctao-bdms-clients 0.2.0rc1__py3-none-any.whl → 0.3.0__py3-none-any.whl

bdms/extract_fits_metadata.py

@@ -0,0 +1,134 @@
+"""Functions to extract metadata from input files."""
+
+import logging
+
+import numpy as np
+from protozfits import File
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+# COMMON HEADER
+start_time = "DataStream.DATE"
+
+# COMMON DATA
+origin = "DataStream.ORIGIN"
+sb_id = "DataStream.sb_id"
+obs_id = "DataStream.obs_id"
+
+# -- FOR TEL_TRIG
+tel_ids = "DataStream.tel_ids"
+
+# -- FOR TEL_SUB
+subarray_id = "DataStream.subarray_id"
+
+METADATA_TEL = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "Events.DATEEND",
+    },
+    "PAYLOAD": {
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+METADATA_SUB = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "SubarrayEvents.DATEEND",
+    },
+    "PAYLOAD": {
+        "subarray_id": subarray_id,
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+METADATA_TRIG = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "Triggers.DATEEND",
+    },
+    "PAYLOAD": {
+        "tel_ids": tel_ids,
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+#: Mapping from DataStream.PBFHEAD to the metadata items we want to collect
+METADATA_SCHEMAS = {
+    "DL0v1.Trigger.DataStream": METADATA_TRIG,
+    "DL0v1.Subarray.DataStream": METADATA_SUB,
+    "DL0v1.Telescope.DataStream": METADATA_TEL,
+}
+
+
+def extract_metadata_from_headers(hdul):
+    """Extract metadata from FITS headers of hdul."""
+    all_headers = {}
+    for hdu in hdul:
+        if hdu.is_image:
+            continue
+        all_headers[hdu.name] = dict(hdu.header)
+
+    try:
+        all_headers["DataStream"]
+    except KeyError:
+        logger.error("No DataStream HDU found in the FITS file.")
+        return {}
+
+    pbfhead = all_headers["DataStream"]["PBFHEAD"]
+    schema = METADATA_SCHEMAS.get(pbfhead)
+    if schema is None:
+        logger.error(
+            "The PBFHEAD %r does not correspond to any known FITS type.", pbfhead
+        )
+        return {}
+
+    logger.debug("Headers extracted: %s", all_headers.keys())
+
+    metadata = {}
+    for value_name, metadata_path in schema["HEADER"].items():
+        extname, header_key = metadata_path.split(".")
+        table = all_headers[extname][header_key]
+        metadata[value_name] = table
+
+    return metadata
+
+
+def extract_metadata_from_data(path):
+    """Extract metadata from zFITS payload in path."""
+    with File(path) as f:
+        if not hasattr(f, "DataStream"):
+            return {}
+
+        pbfhead = f.DataStream.header["PBFHEAD"]
+        schema = METADATA_SCHEMAS.get(pbfhead)
+        if schema is None:
+            logger.error(
+                "The PBFHEAD %r does not correspond to any known FITS type.", pbfhead
+            )
+            return {}
+
+        metadata = {}
+        for value_name, metadata_path in schema["PAYLOAD"].items():
+            hdu, column = metadata_path.split(".")
+            row = getattr(f, hdu)[0]
+            metadata[value_name] = getattr(row, column)
+
+            if isinstance(metadata[value_name], np.ndarray):
+                # Convert numpy array to a Python list
+                metadata[value_name] = metadata[value_name].tolist()
+
+            logger.debug(
+                "Value '%s' from '%s' extracted. (renamed as '%s')",
+                column,
+                hdu,
+                value_name,
+            )
+        return metadata

bdms/tests/conftest.py

@@ -1,14 +1,17 @@
+import json
 import logging
 import os
 import subprocess as sp
+import time
 from datetime import datetime
 from pathlib import Path
 from secrets import token_hex
 
 import pytest
+from filelock import FileLock
 from rucio.client.scopeclient import ScopeClient
 
-from bdms.tests.utils import download_test_file
+from bdms.tests.utils import download_test_file, reset_xrootd_permissions
 
 USER_CERT = os.getenv("RUCIO_CFG_CLIENT_CERT", "/opt/rucio/etc/usercert.pem")
 USER_KEY = os.getenv("RUCIO_CFG_CLIENT_KEY", "/opt/rucio/etc/userkey.pem")
@@ -41,18 +44,26 @@ def _auth_proxy(tmp_path_factory):
     # Key has to have 0o600 permissions, but due to the way we
     # we create and mount it, it does not. We copy to a tmp file
     # set correct permissions and then create the proxy
-    sp.run(
-        [
-            "voms-proxy-init",
-            "-valid",
-            "9999:00",
-            "-cert",
-            USER_CERT,
-            "-key",
-            USER_KEY,
-        ],
-        check=True,
-    )
+
+    try:
+        sp.run(
+            [
+                "voms-proxy-init",
+                "-valid",
+                "9999:00",
+                "-cert",
+                USER_CERT,
+                "-key",
+                USER_KEY,
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+
+    except sp.CalledProcessError as e:
+        error_msg = e.stderr.strip() if e.stderr else str(e)
+        raise pytest.fail(f"VOMS proxy failed: {error_msg}")
 
 
 @pytest.fixture(scope="session")
@@ -90,5 +101,137 @@ def tel_trigger_test_file():
 @pytest.fixture(scope="session")
 def tel_events_test_file():
     """Fixture to download a telescope events test file"""
-    path = "acada-small/DL0/LSTN-01/ctao-n-acada/acada-adh/events/2025/02/04/TEL001_SDH0000_20250204T213354_SBID0000000002000000066_OBSID0000000002000000200_CHUNK000.fits.fz"
+    path = "acada-small/DL0/LSTN-01/ctao-n-acada/acada-adh/events/2025/02/04/TEL001_SDH0000_20250204T213354_SBID0000000002000000066_OBSID0000000002000000200_CHUNK001.fits.fz"
     return download_test_file(path)
+
+
+@pytest.fixture
+def onsite_test_file(
+    storage_mount_path: Path, test_scope: str, test_vo: str
+) -> tuple[Path, str]:
+    """Create a dummy file in the shared storage for testing."""
+
+    unique_id = f"{datetime.now():%Y%m%d_%H%M%S}_{token_hex(8)}"
+    filename = f"testfile_{unique_id}.txt"
+
+    test_file_path = storage_mount_path / test_vo / test_scope / filename
+    test_file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write a small test content (simulating a .fits.fz file with minimal content for testing)
+    test_file_content = f"Test file with random content: {unique_id}"
+    test_file_path.write_text(test_file_content)
+
+    # need to change file permissions of created directories so that
+    # the xrootd still can read and write there
+    reset_xrootd_permissions(storage_mount_path)
+
+    return test_file_path, test_file_content
+
+
+def run_kubectl(args: list[str]) -> str:
+    """Run a kubectl command with the given arguments and return the output."""
+    result = sp.run(
+        ["./kubectl"] + args,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"kubectl command failed: {result.stderr}")
+
+    return result.stdout.strip()
+
+
+def wait_for_deployment_ready(deployment_name, replicas):
+    """Wait for a deployment to be ready with the specified number of replicas."""
+
+    timeout_stop_at = time.time() + 300
+    while True:
+        result = run_kubectl(["get", deployment_name, "-o", "json"])
+        ready_replicas = json.loads(result)["status"].get("readyReplicas", 0)
+
+        if ready_replicas >= replicas:
+            logging.info(
+                "%s deployment is ready with %s replicas.",
+                deployment_name,
+                ready_replicas,
+            )
+            break
+
+        if time.time() > timeout_stop_at:
+            raise TimeoutError(
+                f"Timeout while waiting for {deployment_name} deployment to be ready."
+            )
+
+        logging.info(
+            "Waiting for %s deployment to be ready. Current ready replicas: %s, expected: %s till timeout in %s s",
+            deployment_name,
+            ready_replicas,
+            replicas,
+            int(timeout_stop_at - time.time()),
+        )
+
+        time.sleep(1)
+
+
+def deployment_scale(daemon_name: str, replicas: int = 1) -> None:
+    """Scale a deployment to a specific number of replicas."""
+
+    deployment_name = "deployment/bdms-" + daemon_name
+
+    run_kubectl(
+        [
+            "scale",
+            deployment_name,
+            f"--replicas={replicas}",
+        ]
+    )
+
+    if replicas > 0:
+        wait_for_deployment_ready(deployment_name, replicas)
+
+    # there is a delay between demon writing lock file and the daemon starting to process trigger files
+    time.sleep(3)
+
+    # wait for any terminating pods to finish.
+    # they tend to linger around and while they do not count as replicas, they may still interfere with tests by modifying the trigger files.
+    while True:
+        result = run_kubectl(["get", "pods"])
+
+        if "Terminating" not in result:
+            break
+
+        logging.info("Waiting for any Terminating pods to disappear...")
+        time.sleep(5)
+
+
+@pytest.fixture
+def enable_repearer_daemon():
+    """Fixture to enable the repeater daemon during tests."""
+
+    deployment_scale("judge-repairer", 1)
+    yield
+    deployment_scale("judge-repairer", 0)
+
+
+@pytest.fixture
+def enable_ingestion_daemon():
+    """Fixture to enable the ingestion daemon during tests."""
+
+    deployment_scale("ingestion-daemon", 1)
+    yield
+    deployment_scale("ingestion-daemon", 0)
+
+
+@pytest.fixture
+def disable_ingestion_daemon():
+    """Fixture to suspend the ingestion daemon during tests."""
+    deployment_scale("ingestion-daemon", 0)
+
+
+@pytest.fixture
+def lock_for_ingestion_daemon():
+    """Fixture to prevent daemon tests from running simultaneously."""
+
+    with FileLock(STORAGE_MOUNT_PATH / "ingestion_daemon.lock"):
+        yield

bdms/tests/test_acada_ingest_cli.py

@@ -0,0 +1,279 @@
+import threading
+import time
+from pathlib import Path
+from shutil import copy2
+
+import numpy as np
+import pytest
+import yaml
+from astropy.io import fits
+from rucio.client.downloadclient import DownloadClient
+from rucio.client.replicaclient import ReplicaClient
+from rucio.common.utils import adler32
+
+from bdms.acada_ingest_cli import main as ingest_cli
+from bdms.acada_ingest_cli import parse_args_and_config
+from bdms.tests.utils import reset_xrootd_permissions
+
+ONSITE_RSE = "STORAGE-1"
+
+
+@pytest.mark.usefixtures("_auth_proxy", "lock_for_ingestion_daemon")
+@pytest.mark.parametrize("dry_run", [True, False], ids=["dry_run", "no_dry_run"])
+def test_cli_ingestion(
+    storage_mount_path, test_vo, test_scope, subarray_test_file, tmp_path, dry_run
+):
+    """
+    Test CLI ACADA ingestion.
+    """
+    filename = Path(subarray_test_file).name
+    acada_path = (
+        storage_mount_path / test_vo / test_scope / "test_cli_ingestion" / filename
+    )
+    acada_path.parent.mkdir(parents=True, exist_ok=True)
+    copy2(subarray_test_file, str(acada_path))
+    reset_xrootd_permissions(storage_mount_path)
+
+    expected_lfn = f"/{acada_path.relative_to(storage_mount_path)}"
+    lock_file = tmp_path / "cli_test.lock"
+
+    def run_daemon():
+        args = [
+            f"--data-path={storage_mount_path}",
+            f"--rse={ONSITE_RSE}",
+            f"--vo={test_vo}",
+            f"--scope={test_scope}",
+            "--workers=1",
+            f"--lock-file={lock_file}",
+            "--polling-interval=0.5",
+            "--disable-metrics",
+            f"--log-file={tmp_path / 'daemon.log'}",
+        ]
+
+        if dry_run:
+            args.append("--dry-run")
+
+        ingest_cli(args=args)
+
+    # Start daemon
+    daemon_thread = threading.Thread(target=run_daemon, daemon=True)
+    daemon_thread.start()
+    time.sleep(1.0)  # time for daemon to initialize
+
+    if not dry_run:
+        trigger_file = Path(str(acada_path) + ".trigger")
+        trigger_file.symlink_to(acada_path.relative_to(acada_path.parent))
+
+        # Wait for ingestion to complete
+        replica_client = ReplicaClient()
+        for _ in range(30):
+            try:
+                replicas = list(
+                    replica_client.list_replicas(
+                        dids=[{"scope": test_scope, "name": expected_lfn}]
+                    )
+                )
+                if replicas:
+                    break
+            except Exception:
+                pass
+            time.sleep(1.0)
+        else:
+            pytest.fail(f"No replica found for {expected_lfn}")
+
+        for _ in range(10):
+            if not trigger_file.exists():
+                break
+            time.sleep(1.0)
+
+        # lock file cleanup
+        if lock_file.exists():
+            lock_file.unlink()
+
+        # Clean up filelock file
+        filelock_file = Path(str(lock_file) + ".lock")
+        if filelock_file.exists():
+            filelock_file.unlink()
+
+        # verify download
+        download_spec = {
+            "did": f"{test_scope}:{expected_lfn}",
+            "base_dir": str(tmp_path),
+            "no_subdir": True,
+        }
+
+        download_client = DownloadClient()
+        download_client.download_dids([download_spec])
+
+        download_path = tmp_path / expected_lfn.lstrip("/")
+        assert download_path.is_file(), f"Download failed at {download_path}"
+        assert adler32(str(download_path)) == adler32(
+            str(subarray_test_file)
+        ), "Downloaded file content does not match the original."
+
+
+def parse_args_and_check_error(args, error_message):
+    """
+    Helper function to run the CLI and check for expected errors.
+    """
+    if error_message:
+        with pytest.raises(SystemExit) as e:
+            parse_args_and_config(args=args)
+        assert error_message in e.value.__context__.message
+    else:
+        # Run without exceptions
+        return parse_args_and_config(args=args)
+
+
+@pytest.mark.parametrize(
+    ("port", "error_message"),
+    [
+        (1234, None),
+        (80, "Metrics port must be between 1024"),
+        ("invalid_metrics", "Metrics port must be an integer"),
+    ],
+    ids=["valid_port", "low_port", "invalid_port"],
+)
+def test_cli_metrics_port_validation(port, error_message):
+    """
+    Test CLI ACADA ingestion exceptions.
+    """
+
+    parse_args_and_check_error(
+        [
+            f"--metrics-port={port}",
+        ],
+        error_message,
+    )
+
+
+@pytest.mark.parametrize(
+    ("polling_interval", "error_message"),
+    [
+        (1, None),
+        (0, "Polling interval must be positive"),
+        ("invalid", "Polling interval must be a number, got"),
+    ],
+    ids=["valid_offsite", "negative_offsite", "invalid_offsite"],
+)
+def test_cli_polling_interval(polling_interval, error_message):
+    """
+    Test CLI ACADA ingestion with offsite copies.
+    """
+    parse_args_and_check_error(
+        [
+            f"--polling-interval={polling_interval}",
+        ],
+        error_message,
+    )
+
+
+@pytest.mark.parametrize(
+    ("check_interval", "error_message"),
+    [
+        (1.0, None),
+        (0.0, "Check interval must be positive"),
+        ("invalid", "Check interval must be a number, got "),
+    ],
+    ids=["valid_check_interval", "zero_check_interval", "invalid_check_interval"],
+)
+def test_cli_check_interval_validation(check_interval, error_message):
+    """
+    Test CLI ACADA ingestion with check interval validation.
+    """
+
+    parse_args_and_check_error(
+        [
+            f"--check-interval={check_interval}",
+        ],
+        error_message,
+    )
+
+
+@pytest.mark.usefixtures("_auth_proxy", "lock_for_ingestion_daemon")
+def test_cli_ingestion_parallel(storage_mount_path, test_vo, test_scope, tmp_path):
+    """Test CLI with 7 files and 4 workers for parallel ingestion."""
+
+    test_dir = storage_mount_path / test_vo / test_scope
+    test_dir.mkdir(parents=True, exist_ok=True)
+
+    test_files = []
+    rng = np.random.default_rng()
+    for i in range(7):
+        test_file = test_dir / f"testfile_{i}_20250609.fits"
+        hdu = fits.PrimaryHDU(rng.random((50, 50)))
+        hdu.writeto(test_file, overwrite=True, checksum=True)
+        test_files.append(test_file)
+
+    reset_xrootd_permissions(storage_mount_path)
+
+    lock_file = tmp_path / "ingestion_queue_test.lock"
+
+    def run_daemon():
+        ingest_cli(
+            args=[
+                f"--data-path={storage_mount_path}",
+                f"--rse={ONSITE_RSE}",
+                f"--vo={test_vo}",
+                f"--scope={test_scope}",
+                "--workers=4",
+                f"--lock-file={lock_file}",
+                "--polling-interval=0.5",
+                "--disable-metrics",
+            ]
+        )
+
+    # Start daemon
+    daemon_thread = threading.Thread(target=run_daemon, daemon=True)
+    daemon_thread.start()
+    time.sleep(1.0)
+
+    for test_file in test_files:
+        trigger_file = Path(str(test_file) + ".trigger")
+        trigger_file.symlink_to(test_file.relative_to(test_file.parent))
+
+    # Wait for all files to be processed, ingestion done
+    replica_client = ReplicaClient()
+    for _ in range(30):
+        processed = 0
+        for test_file in test_files:
+            lfn = f"/{test_file.relative_to(storage_mount_path)}"
+            try:
+                replicas = list(
+                    replica_client.list_replicas(
+                        dids=[{"scope": test_scope, "name": lfn}]
+                    )
+                )
+                if replicas:
+                    processed += 1
+            except Exception:
+                pass
+
+        if processed == 7:
+            break
+        time.sleep(1.0)
+    else:
+        pytest.fail("Not all files were processed")
+
+    # Cleanup
+    for test_file in test_files:
+        test_file.unlink()
+
+    if lock_file.exists():
+        lock_file.unlink()
+
+    filelock_file = Path(str(lock_file) + ".lock")
+    if filelock_file.exists():
+        filelock_file.unlink()
+
+
+def test_parse_config(tmp_path):
+    config_path = tmp_path / "config.yaml"
+    with config_path.open("w") as f:
+        yaml.dump({"workers": 12, "polling_interval": 60.0}, f)
+
+    args = parse_args_and_config([f"--config={config_path}", "--polling-interval=30.0"])
+    # config is parsed
+    assert args.workers == 12
+    # but cli args override config
+    assert args.polling_interval == 30.0