ctao-bdms-clients 0.1.0rc3__py3-none-any.whl → 0.2.0__py3-none-any.whl

bdms/tests/test_extract_fits_metadata.py

@@ -0,0 +1,97 @@
+from astropy.io import fits
+
+from bdms.extract_fits_metadata import (
+    extract_metadata_from_data,
+    extract_metadata_from_headers,
+)
+
+
+def test_extraction_correct_value_subarray_file(subarray_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(subarray_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(subarray_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the SUBARRAY FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-02-04T21:34:05",
+        "end_time": "2025-02-04T21:43:12",
+        "subarray_id": 0,
+        "sb_id": 2000000066,
+        "obs_id": 2000000200,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extraction_correct_value_tel_trigger_file(tel_trigger_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(tel_trigger_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(tel_trigger_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the Telescope TRIGGER FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-02-04T21:34:05",
+        "end_time": "2025-02-04T21:43:11",
+        "tel_ids": [1],
+        "sb_id": 2000000066,
+        "obs_id": 2000000200,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extraction_correct_value_tel_events_file(tel_events_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(tel_events_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(tel_events_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the Telescope EVENTS FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-04-01T15:25:02",
+        "end_time": "2025-04-01T15:25:03",
+        "sb_id": 0,
+        "obs_id": 0,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extract_metadata_from_data_incorrect_header(tmp_path):
+    """Test the extraction of metadata from an empty FITS file header."""
+    fits_file_path = tmp_path / "empty_fits.fits.fz"
+    hdul = fits.HDUList([fits.PrimaryHDU()])
+    hdul.writeto(fits_file_path, checksum=True)
+
+    with fits.open(fits_file_path) as hdul:
+        metadata = extract_metadata_from_headers(hdul)
+
+    assert metadata == {}, "Expected empty metadata in the header"
+
+
+def test_extract_metadata_from_data_incorrect_data(tmp_path):
+    """Test the extraction of metadata from an empty FITS file data."""
+    fits_file_path = tmp_path / "empty_fits.fits.fz"
+    hdul = fits.HDUList([fits.PrimaryHDU()])
+    hdul.writeto(fits_file_path, checksum=True)
+
+    metadata = extract_metadata_from_data(fits_file_path)
+
+    assert metadata == {}, "Expected empty metadata in the payload"

bdms/tests/test_onsite_storage.py

@@ -0,0 +1,100 @@
+import subprocess as sp
+from pathlib import Path
+
+import pytest
+from rucio.client.rseclient import RSEClient
+
+from .conftest import STORAGE_HOSTNAME, STORAGE_PROTOCOL
+
+# Constants for RSEs and expected attributes
+RSE_CONFIG = {
+    "STORAGE-1": {"ONSITE": True, "OFFSITE": None},
+    "STORAGE-2": {"ONSITE": None, "OFFSITE": True},
+    "STORAGE-3": {"ONSITE": None, "OFFSITE": True},
+}
+
+
+def test_shared_storage(storage_mount_path: Path) -> Path:
+    """Ensure shared storage directory exists before any test runs"""
+    assert (
+        storage_mount_path.exists()
+    ), f"Shared storage {storage_mount_path} is not available on the client"
+
+
+def test_file_access_from_onsite_storage_using_gfal(
+    storage_mount_path: Path, onsite_test_file: tuple[Path, str]
+):
+    """Verify that the file is accessible from the onsite storage pod using gfal-ls"""
+    test_file_path, _ = onsite_test_file
+    test_file_lfn = f"/{test_file_path.relative_to(storage_mount_path)}"
+    test_file_name = test_file_path.name
+
+    gfal_url = f"{STORAGE_PROTOCOL}://{STORAGE_HOSTNAME}/rucio/{test_file_lfn}"
+    cmd = ["gfal-ls", gfal_url]
+
+    ret = sp.run(cmd, capture_output=True, text=True)
+    stdout = ret.stdout.strip()
+    stderr = ret.stderr.strip()
+    msg = f"gfal-ls failed for {gfal_url}:\nSTDERR: {stderr}\nSTDOUT: {stderr}"
+    assert ret.returncode == 0, msg
+
+    msg = f"File {test_file_name} not accessible; gfal-ls output: {stdout!r}"
+    assert any(test_file_name in line for line in stdout.splitlines()), msg
+
+
+@pytest.mark.usefixtures("_auth_proxy")
+def test_rse_attributes():
+    """Verify onsite and offsite RSE attributes set by setup_rucio.sh during the bootstrap job deployment
+
+    Ensures:
+    - STORAGE-1 has onsite=True and no offsite=True
+    - STORAGE-2 and STORAGE-3 have offsite=True and no onsite=True
+
+    Raises:
+        pytest.fail: If RSE details cannot be retrieved (in case of RSEs not found or Rucio server connectivity issues)
+        AssertionError: If attribute values don't match the expected ones
+    """
+
+    rse_client = RSEClient()
+
+    for rse_name, expected_attrs in RSE_CONFIG.items():
+        try:
+            # Verify RSE exists
+            rse_details = rse_client.get_rse(rse_name)
+            print(f"{rse_name} metadata: {rse_details}")
+
+            # Fetch attributes
+            attrs = rse_client.list_rse_attributes(rse_name)
+            print(f"{rse_name} attributes: {attrs}")
+
+            # Verify RSE onsite attribute
+            onsite_value = attrs.get("ONSITE")
+            expected_onsite = expected_attrs["ONSITE"]
+            assert onsite_value == expected_onsite, (
+                f"{rse_name} onsite attribute mismatch: "
+                f"expected {expected_onsite!r}, got {onsite_value!r}. "
+                f"Full attributes: {attrs}"
+            )
+
+            # Verify RSE offsite attribute
+            offsite_value = attrs.get("OFFSITE")
+            expected_offsite = expected_attrs["OFFSITE"]
+            if expected_offsite is None:
+                assert offsite_value is not True, (
+                    f"{rse_name} should not have offsite=True, "
+                    f"got {offsite_value!r}. Full attributes: {attrs}"
+                )
+            else:
+                assert offsite_value == expected_offsite, (
+                    f"{rse_name} offsite attribute mismatch: "
+                    f"expected {expected_offsite!r}, got {offsite_value!r}. "
+                    f"Full attributes: {attrs}"
+                )
+
+            print(f"{rse_name} passed attribute tests")
+
+        except Exception as e:
+            pytest.fail(
+                f"Failed to retrieve RSE details for {rse_name}: {str(e)}. "
+                "Check Rucio server connectivity or RSE existence"
+            )

bdms/tests/utils.py

@@ -0,0 +1,130 @@
+"""Utility functions for BDMS tests."""
+
+import logging
+import os
+import time
+from pathlib import Path
+
+from dotenv import load_dotenv
+from minio import Minio
+from rucio.client.ruleclient import RuleClient
+from rucio.common.exception import RucioException
+
+# Default timeout and polling interval (in seconds) for waiting for replication
+DEFAULT_TIMEOUT = 1000
+DEFAULT_POLL_INTERVAL = 30
+XROOTD_UID = int(os.getenv("XROOTD_UID", 994))
+XROOTD_GID = int(os.getenv("XROOTD_GID", 994))
+LOGGER = logging.getLogger(__name__)
+
+
+def reset_xrootd_permissions(path):
+    recursive_chown(path, uid=XROOTD_UID, gid=XROOTD_GID)
+
+
+def recursive_chown(path: Path, uid: int, gid: int):
+    """Equivalent of unix chmod -R <uid>:<gid> <path>."""
+    os.chown(path, uid, gid)
+
+    for root, dirs, files in os.walk(path):
+        root = Path(root)
+        for d in dirs:
+            os.chown(root / d, uid, gid)
+        for f in files:
+            os.chown(root / f, uid, gid)
+
+
+def wait_for_replication_status(
+    rule_client: RuleClient,
+    rule_id: str,
+    expected_status: str = "OK",
+    timeout: int = DEFAULT_TIMEOUT,
+    poll_interval: int = DEFAULT_POLL_INTERVAL,
+    logger: logging.Logger = None,
+) -> None:
+    if logger is None:
+        logger = LOGGER
+
+    start_time = time.perf_counter()
+    current_status = None
+    result = None
+    max_retries = 3
+
+    while (time.perf_counter() - start_time) < timeout:
+        retries = 0
+        while retries < max_retries:
+            try:
+                result = rule_client.get_replication_rule(rule_id)
+                current_status = result["state"]
+                break
+            except RucioException as e:
+                retries += 1
+                if retries == max_retries:
+                    raise RuntimeError(
+                        f"Failed to check replication rule status for rule {rule_id} after {max_retries} retries: {str(e)}"
+                    ) from e
+                logger.warning(
+                    "Failed to check rule %s status (attempt %s/%s): %s. Retrying...",
+                    rule_id,
+                    retries,
+                    max_retries,
+                    str(e),
+                )
+                time.sleep(1)
+
+        if current_status == expected_status:
+            logger.info(
+                "Replication rule %s reached status '%s'", rule_id, expected_status
+            )
+            return
+
+        logger.debug(
+            "Rule %s is in state %s, waiting for %s (elapsed: %.2f seconds)",
+            rule_id,
+            current_status,
+            expected_status,
+            time.perf_counter() - start_time,
+        )
+        time.sleep(poll_interval)
+
+    msg = (
+        f"Replication rule {rule_id} did not reach status '{expected_status}' within {timeout} seconds. "
+        f"Current status is '{current_status}'.\nFull output: {result}"
+    )
+    raise TimeoutError(msg)
+
+
+TEST_DATA_DIR = Path(os.getenv("BDMS_TEST_DATA_DIR", "test_data")).absolute()
+
+
+def download_test_file(path):
+    """Get a FITS file from the MinIO server"""
+
+    load_dotenv()
+
+    access_key = os.environ["MINIO_ACCESS_KEY"]
+    secret_key = os.environ["MINIO_SECRET_KEY"]
+
+    output_path = TEST_DATA_DIR / path
+
+    TEST_DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+    if not output_path.exists():
+        client = Minio(
+            "minio-cta.zeuthen.desy.de",
+            access_key,
+            secret_key,
+            secure=True,
+        )
+
+        LOGGER.info("Downloading %s", path)
+        client.fget_object(
+            "dpps-data-private",
+            path,
+            output_path,
+        )
+
+    else:
+        LOGGER.info("File %s already exists, skipping download", output_path)
+
+    return output_path

{ctao_bdms_clients-0.1.0rc3.dist-info → ctao_bdms_clients-0.2.0.dist-info}/METADATA

@@ -1,31 +1,38 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.1.0rc3
+Version: 0.2.0
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
 Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms
 Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: rucio-clients~=35.4.1
-Requires-Dist: ctao-bdms-rucio-policy==0.1.0
+Requires-Dist: astropy<8.0.0a0,>=6.0.1
+Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
+Requires-Dist: rucio-clients~=35.7.0
+Requires-Dist: protozfits>=2.7.2
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-requirements; extra == "test"
+Requires-Dist: python-dotenv; extra == "test"
+Requires-Dist: minio; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Provides-Extra: doc
 Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: numpydoc; extra == "doc"
 Requires-Dist: ctao-sphinx-theme; extra == "doc"
 Requires-Dist: myst-parser; extra == "doc"
 Requires-Dist: sphinx-changelog; extra == "doc"
+Requires-Dist: sphinx-automodapi; extra == "doc"
 Provides-Extra: dev
 Requires-Dist: setuptools_scm; extra == "dev"
 Requires-Dist: sphinx-autobuild; extra == "dev"
 Provides-Extra: all
 Requires-Dist: bdms[dev,doc,test]; extra == "all"
+Dynamic: license-file
 
 # Bulk Data Management System
 

ctao_bdms_clients-0.2.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
+bdms/_version.py,sha256=iB5DfB5V6YB5Wo4JmvS-txT42QtmGaWcWp3udRT7zCI,511
+bdms/acada_ingestion.py,sha256=L-LBdfd7dbSbW0poseXsZ8CbgWch8j57yaQncIemnOs,17671
+bdms/extract_fits_metadata.py,sha256=ZGJQCFJCXkWg8N3CAb17GB-wwPj-wTvNg0JOS-MemZ0,3431
+bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
+bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bdms/tests/conftest.py,sha256=TqMBSExgO4omUHVqoXcI1AOhS3F8de-03275IlYEW6k,3896
+bdms/tests/test_acada_ingestion.py,sha256=u27Srhm8kXKtFoPr5gQdZeGDjrlECMr9ysWwGfs2w_Q,18277
+bdms/tests/test_basic_rucio_functionality.py,sha256=9GIX8IO6wBJm40LKFEH2StS-fMKvC07sxFHPVR7dftU,3583
+bdms/tests/test_dpps_rel_0_0.py,sha256=MnbuBoS_kUUiMcHE3-jqOzekQNUa-wcsjCJqJQ2J9S4,2957
+bdms/tests/test_extract_fits_metadata.py,sha256=A935WD2TF3lBcaeDmzGSlH2IXUF1v8qslrsW30lnEAA,3490
+bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
+bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
+bdms/tests/test_onsite_storage.py,sha256=waK7t9kBquzJbuLLYcpeNU9YuA70XTRS88RMxBWxawI,3765
+bdms/tests/utils.py,sha256=4g7__ms-xnTyyBKMlmV4hpC505V6uVaXJDi9XQ8UC_4,3717
+ctao_bdms_clients-0.2.0.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
+ctao_bdms_clients-0.2.0.dist-info/METADATA,sha256=Rstsh4x_9neUhLc8mFSOA8iY4YR0Lvrn7IRVWcROqXI,2383
+ctao_bdms_clients-0.2.0.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+ctao_bdms_clients-0.2.0.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
+ctao_bdms_clients-0.2.0.dist-info/RECORD,,

{ctao_bdms_clients-0.1.0rc3.dist-info → ctao_bdms_clients-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

bdms/_dev_version/__init__.py

@@ -1,9 +0,0 @@
-# Try to use setuptools_scm to get the current version; this is only used
-# in development installations from the git repository.
-# see ../version.py for details
-try:
-    from setuptools_scm import get_version
-
-    version = get_version(root="../../..", relative_to=__file__)
-except Exception as e:
-    raise ImportError(f"setuptools_scm broken or not installed: {e}")

ctao_bdms_clients-0.1.0rc3.dist-info/RECORD

@@ -1,15 +0,0 @@
-bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
-bdms/_version.py,sha256=T50yPh_uNdS9vwClzyZNHuV3NjwtB-VEu54RR1k2Pr8,414
-bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
-bdms/_dev_version/__init__.py,sha256=3qlzT1l_MfLxHuRphBwNwkb2WRttg3hGooj5n3BBZi4,369
-bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bdms/tests/conftest.py,sha256=vN96UXhO2503svkg3V2jxfq5SSUtlRCWZt5MQgblu7Y,1374
-bdms/tests/test_basic_rucio_functionality.py,sha256=Sm6Kif8g5xPaw6tIIJ1TavDxVje2YaYYC9PnhVbVQFI,3999
-bdms/tests/test_dpps_rel_0_0.py,sha256=MnbuBoS_kUUiMcHE3-jqOzekQNUa-wcsjCJqJQ2J9S4,2957
-bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
-bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
-ctao_bdms_clients-0.1.0rc3.dist-info/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
-ctao_bdms_clients-0.1.0rc3.dist-info/METADATA,sha256=XtiN3evP0TQGDRZ_M0_b9DXz1lWWjPXU2li95xj8RZs,2103
-ctao_bdms_clients-0.1.0rc3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-ctao_bdms_clients-0.1.0rc3.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
-ctao_bdms_clients-0.1.0rc3.dist-info/RECORD,,