ctao-bdms-clients 0.2.0rc1__py3-none-any.whl → 0.2.1__py3-none-any.whl

bdms/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0rc1'
-__version_tuple__ = version_tuple = (0, 2, 0, 'rc1')
+__version__ = version = '0.2.1'
+__version_tuple__ = version_tuple = (0, 2, 1)

bdms/acada_ingestion.py

@@ -4,16 +4,16 @@ This module provides the IngestionClient class to manage the ingestion of ACADA
 It includes functionality for constructing FITS file paths, converting ACADA paths to Logical File Names (LFNs),
 and registering replicas in Rucio.
 """
-# this is the latest working file (as on 6:40 pm) with judge repairer repairing the STUCK rule.
 
 import logging
 import os
+from contextlib import ExitStack
 from pathlib import Path
 from typing import Optional, Union
 
 from astropy.io import fits
 from rucio.client.accountclient import AccountClient
-from rucio.client.client import Client
+from rucio.client.client import Client, DIDClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.client.rseclient import RSEClient
 from rucio.client.ruleclient import RuleClient
@@ -21,6 +21,11 @@ from rucio.client.scopeclient import ScopeClient
 from rucio.common.exception import Duplicate, RucioException
 from rucio.common.utils import adler32
 
+from bdms.extract_fits_metadata import (
+    extract_metadata_from_data,
+    extract_metadata_from_headers,
+)
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -91,6 +96,7 @@ class IngestionClient:
             self.account_client = AccountClient()
             self.rse_client = RSEClient()
             self.rule_client = RuleClient()
+            self.did_client = DIDClient()
         except RucioException as e:
             self.logger.error("Failed to initialize Rucio clients: %s", str(e))
             raise
@@ -261,9 +267,10 @@ class IngestionClient:
             return lfn
 
         # Proceed with registering the replica if check_replica_exists returns False
+        valid, metadata = verify_and_extract_metadata(acada_path)
+        metadata["valid_fits_checksum"] = valid
 
-        # Compute file metadata
-        # TODO: use functions to identify file type, extract metadata, validate integrity, when this functionality is ready https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/work_items/46
+        # Compute rucio file metadata
         file_size = acada_path.stat().st_size
         checksum = adler32(acada_path)
 
@@ -286,6 +293,10 @@ class IngestionClient:
             )
         self.logger.info("Successfully registered the replica for lfn '%s'", lfn)
 
+        if len(metadata) > 0:
+            self.did_client.set_metadata_bulk(scope=self.scope, name=lfn, meta=metadata)
+            self.logger.info("Set metadata of %r to %r", lfn, metadata)
+
         return lfn
 
     def add_offsite_replication_rules(
@@ -433,3 +444,36 @@ def verify_fits_checksum(hdul: fits.HDUList):
             raise FITSVerificationError(msg)
         elif checksum_result == 2 and pos != 0:  # ignore primary for warning
             LOGGER.warning("No CHECKSUM in HDU %d with name %r", pos, name)
+
+
+def verify_and_extract_metadata(fits_path):
+    """Verify checksums and extract metadata from FITS files.
+
+    This wrapper transforms exceptions into log errors and minimizes
+    the number of times the FITS file has to be opened.
+    """
+    # this context manager allows elegant handling
+    # of conditionally present context managers
+    # which allows better handling of exceptions below
+    context = ExitStack()
+    metadata = {}
+    with context:
+        try:
+            hdul = context.enter_context(fits.open(fits_path))
+        except Exception as e:
+            LOGGER.error("Failed to open FITS file %r: %s", fits_path, e)
+            return False, metadata
+
+        try:
+            verify_fits_checksum(hdul)
+        except FITSVerificationError as e:
+            LOGGER.error("File %r failed FITS checksum verification: %s", fits_path, e)
+            return False, metadata
+
+        try:
+            metadata = extract_metadata_from_headers(hdul)
+            metadata.update(extract_metadata_from_data(fits_path))
+            return True, metadata
+        except Exception as e:
+            LOGGER.error("Failed to extract metadata from %r: %s", fits_path, e)
+            return False, metadata

bdms/extract_fits_metadata.py

@@ -0,0 +1,134 @@
+"""Functions to extract metadata from input files."""
+
+import logging
+
+import numpy as np
+from protozfits import File
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+# COMMON HEADER
+start_time = "DataStream.DATE"
+
+# COMMON DATA
+origin = "DataStream.ORIGIN"
+sb_id = "DataStream.sb_id"
+obs_id = "DataStream.obs_id"
+
+# -- FOR TEL_TRIG
+tel_ids = "DataStream.tel_ids"
+
+# -- FOR TEL_SUB
+subarray_id = "DataStream.subarray_id"
+
+METADATA_TEL = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "Events.DATEEND",
+    },
+    "PAYLOAD": {
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+METADATA_SUB = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "SubarrayEvents.DATEEND",
+    },
+    "PAYLOAD": {
+        "subarray_id": subarray_id,
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+METADATA_TRIG = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "Triggers.DATEEND",
+    },
+    "PAYLOAD": {
+        "tel_ids": tel_ids,
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+#: Mapping from DataStream.PBFHEAD to the metadata items we want to collect
+METADATA_SCHEMAS = {
+    "DL0v1.Trigger.DataStream": METADATA_TRIG,
+    "DL0v1.Subarray.DataStream": METADATA_SUB,
+    "DL0v1.Telescope.DataStream": METADATA_TEL,
+}
+
+
+def extract_metadata_from_headers(hdul):
+    """Extract metadata from FITS headers of hdul."""
+    all_headers = {}
+    for hdu in hdul:
+        if hdu.is_image:
+            continue
+        all_headers[hdu.name] = dict(hdu.header)
+
+    try:
+        all_headers["DataStream"]
+    except KeyError:
+        logger.error("No DataStream HDU found in the FITS file.")
+        return {}
+
+    pbfhead = all_headers["DataStream"]["PBFHEAD"]
+    schema = METADATA_SCHEMAS.get(pbfhead)
+    if schema is None:
+        logger.error(
+            "The PBFHEAD %r does not correspond to any known FITS type.", pbfhead
+        )
+        return {}
+
+    logger.debug("Headers extracted: %s", all_headers.keys())
+
+    metadata = {}
+    for value_name, metadata_path in schema["HEADER"].items():
+        extname, header_key = metadata_path.split(".")
+        table = all_headers[extname][header_key]
+        metadata[value_name] = table
+
+    return metadata
+
+
+def extract_metadata_from_data(path):
+    """Extract metadata from zFITS payload in path."""
+    with File(path) as f:
+        if not hasattr(f, "DataStream"):
+            return {}
+
+        pbfhead = f.DataStream.header["PBFHEAD"]
+        schema = METADATA_SCHEMAS.get(pbfhead)
+        if schema is None:
+            logger.error(
+                "The PBFHEAD %r does not correspond to any known FITS type.", pbfhead
+            )
+            return {}
+
+        metadata = {}
+        for value_name, metadata_path in schema["PAYLOAD"].items():
+            hdu, column = metadata_path.split(".")
+            row = getattr(f, hdu)[0]
+            metadata[value_name] = getattr(row, column)
+
+            if isinstance(metadata[value_name], np.ndarray):
+                # Convert numpy array to a Python list
+                metadata[value_name] = metadata[value_name].tolist()
+
+            logger.debug(
+                "Value '%s' from '%s' extracted. (renamed as '%s')",
+                column,
+                hdu,
+                value_name,
+            )
+        return metadata

bdms/tests/conftest.py

@@ -8,7 +8,7 @@ from secrets import token_hex
 import pytest
 from rucio.client.scopeclient import ScopeClient
 
-from bdms.tests.utils import download_test_file
+from bdms.tests.utils import download_test_file, reset_xrootd_permissions
 
 USER_CERT = os.getenv("RUCIO_CFG_CLIENT_CERT", "/opt/rucio/etc/usercert.pem")
 USER_KEY = os.getenv("RUCIO_CFG_CLIENT_KEY", "/opt/rucio/etc/userkey.pem")
@@ -90,5 +90,28 @@ def tel_trigger_test_file():
 @pytest.fixture(scope="session")
 def tel_events_test_file():
     """Fixture to download a telescope events test file"""
-    path = "acada-small/DL0/LSTN-01/ctao-n-acada/acada-adh/events/2025/02/04/TEL001_SDH0000_20250204T213354_SBID0000000002000000066_OBSID0000000002000000200_CHUNK000.fits.fz"
+    path = "acada-small/DL0/LSTN-01/ctao-n-acada/acada-adh/events/2025/02/04/TEL001_SDH0000_20250204T213354_SBID0000000002000000066_OBSID0000000002000000200_CHUNK001.fits.fz"
     return download_test_file(path)
+
+
+@pytest.fixture
+def onsite_test_file(
+    storage_mount_path: Path, test_scope: str, test_vo: str
+) -> tuple[Path, str]:
+    """Create a dummy file in the shared storage for testing."""
+
+    unique_id = f"{datetime.now():%Y%m%d_%H%M%S}_{token_hex(8)}"
+    filename = f"testfile_{unique_id}.txt"
+
+    test_file_path = storage_mount_path / test_vo / test_scope / filename
+    test_file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write a small test content (simulating a .fits.fz file with minimal content for testing)
+    test_file_content = f"Test file with random content: {unique_id}"
+    test_file_path.write_text(test_file_content)
+
+    # need to change file permissions of created directories so that
+    # the xrootd still can read and write there
+    reset_xrootd_permissions(storage_mount_path)
+
+    return test_file_path, test_file_content

bdms/tests/test_acada_ingestion.py

@@ -5,11 +5,9 @@ and the replication of data between Rucio storage elements (RSEs).
 """
 
 import logging
-import os
 import subprocess
-from datetime import datetime
 from pathlib import Path
-from secrets import token_hex
+from shutil import copy2
 
 import pytest
 from astropy.io import fits
@@ -22,12 +20,10 @@ from rucio.common.exception import RucioException
 from rucio.common.utils import adler32
 
 from bdms.acada_ingestion import IngestionClient
-from bdms.tests.utils import wait_for_replication_status
+from bdms.tests.utils import reset_xrootd_permissions, wait_for_replication_status
 
 LOGGER = logging.getLogger(__name__)
 
-XROOTD_UID = 994
-XROOTD_GID = 994
 ONSITE_RSE = "STORAGE-1"
 OFFSITE_RSE_1 = "STORAGE-2"
 OFFSITE_RSE_2 = "STORAGE-3"
@@ -41,16 +37,6 @@ def test_shared_storage(storage_mount_path: Path):
     ), f"Shared storage {storage_mount_path} is not available on the client"
 
 
-def recursive_chown(path: Path, uid: int, gid: int):
-    """Equivalent of unix chmod -R <uid>:<gid> <path>."""
-    for root, dirs, files in os.walk(path):
-        root = Path(root)
-        for d in dirs:
-            os.chown(root / d, uid, gid)
-        for f in files:
-            os.chown(root / f, uid, gid)
-
-
 def trigger_judge_repairer() -> None:
     """Trigger the rucio-judge-repairer daemon to run once and fix any STUCK rules."""
 
@@ -80,30 +66,6 @@ def trigger_judge_repairer() -> None:
         raise
 
 
-@pytest.fixture
-def test_file(
-    storage_mount_path: Path, test_scope: str, test_vo: str
-) -> tuple[Path, str]:
-    """Create a dummy .fits.fz file in the shared storage for testing."""
-
-    unique_id = f"{datetime.now():%Y%m%d_%H%M%S}_{token_hex(8)}"
-    filename = f"testfile_{unique_id}.fits.fz"
-
-    test_file_path = storage_mount_path / test_vo / test_scope / filename
-    test_file_path.parent.mkdir(parents=True, exist_ok=True)
-
-    # need to change file permissions of created directories so that
-    # the xrootd still can read and write there
-    recursive_chown(storage_mount_path / test_vo, XROOTD_UID, XROOTD_GID)
-
-    # Write a small test content (simulating a .fits.fz file with minimal content for testing)
-    test_file_content = f"FITS-like content for {unique_id}"
-    test_file_path.write_text(test_file_content)
-    os.chown(test_file_path, XROOTD_UID, XROOTD_GID)
-
-    return test_file_path, test_file_content
-
-
 def test_acada_to_lfn(storage_mount_path: Path, test_vo: str):
     """Test the acada_to_lfn method of IngestionClient with valid and invalid inputs."""
 
@@ -151,7 +113,10 @@ def test_acada_to_lfn(storage_mount_path: Path, test_vo: str):
 
 @pytest.mark.usefixtures("_auth_proxy")
 def test_check_replica_exists(
-    storage_mount_path: Path, test_scope: str, test_file: tuple[Path, str], test_vo: str
+    storage_mount_path: Path,
+    test_scope: str,
+    onsite_test_file: tuple[Path, str],
+    test_vo: str,
 ):
     """Test the check_replica_exists method of IngestionClient."""
 
@@ -159,7 +124,7 @@ def test_check_replica_exists(
         storage_mount_path, ONSITE_RSE, scope=test_scope, vo=test_vo
     )
 
-    acada_path, _ = test_file
+    acada_path, _ = onsite_test_file
 
     # Generate the LFN
     lfn = ingestion_client.acada_to_lfn(acada_path)
@@ -181,10 +146,54 @@ def test_check_replica_exists(
     assert not ingestion_client.check_replica_exists(nonexistent_lfn), msg
 
 
+@pytest.fixture
+def file_location(request):
+    return request.getfixturevalue(request.param)
+
+
+@pytest.mark.parametrize(
+    ("file_location", "metadata_dict"),
+    [
+        (
+            "subarray_test_file",
+            {
+                "observatory": "CTA",
+                "start_time": "2025-02-04T21:34:05",
+                "end_time": "2025-02-04T21:43:12",
+                "subarray_id": 0,
+                "sb_id": 2000000066,
+                "obs_id": 2000000200,
+            },
+        ),
+        (
+            "tel_trigger_test_file",
+            {
+                "observatory": "CTA",
+                "start_time": "2025-02-04T21:34:05",
+                "end_time": "2025-02-04T21:43:11",
+                "tel_ids": [1],
+                "sb_id": 2000000066,
+                "obs_id": 2000000200,
+            },
+        ),
+        (
+            "tel_events_test_file",
+            {
+                "observatory": "CTA",
+                "start_time": "2025-04-01T15:25:02",
+                "end_time": "2025-04-01T15:25:03",
+                "sb_id": 0,
+                "obs_id": 0,
+            },
+        ),
+    ],
+    indirect=["file_location"],
+)
 @pytest.mark.usefixtures("_auth_proxy")
 @pytest.mark.verifies_usecase("UC-110-1.1.1")
-def test_add_onsite_replica_with_dummy_file(
-    test_file: tuple[Path, str],
+def test_add_onsite_replica_with_minio_fits_file(
+    file_location: str,
+    metadata_dict: dict,
     test_scope: str,
     tmp_path: Path,
     storage_mount_path,
@@ -193,11 +202,16 @@ def test_add_onsite_replica_with_dummy_file(
 ):
     """Test the add_onsite_replica method of IngestionClient using a dummy file."""
 
+    filename = str(file_location).split("/")[-1]
+    acada_path = storage_mount_path / test_vo / test_scope / filename
+    acada_path.parent.mkdir(parents=True, exist_ok=True)
+    copy2(file_location, str(acada_path))
+    reset_xrootd_permissions(storage_mount_path)
+
     ingestion_client = IngestionClient(
         storage_mount_path, ONSITE_RSE, scope=test_scope, vo=test_vo
     )
 
-    acada_path, test_file_content = test_file
     # Use add_onsite_replica to register the replica
     lfn = ingestion_client.add_onsite_replica(acada_path=acada_path)
 
@@ -218,11 +232,9 @@ def test_add_onsite_replica_with_dummy_file(
     download_path = tmp_path / lfn.lstrip("/")
     assert download_path.is_file(), f"Download failed at {download_path}"
 
-    downloaded_content = download_path.read_text()
-    assert downloaded_content == test_file_content, (
-        f"Downloaded file content does not match the original. "
-        f"Expected: {test_file_content}, Got: {downloaded_content}"
-    )
+    assert adler32(download_path) == adler32(
+        file_location
+    ), "Downloaded file content does not match the original. "
 
     # Check for don't ingest again if its already registered
     caplog.clear()
@@ -231,6 +243,19 @@ def test_add_onsite_replica_with_dummy_file(
         r.message for r in caplog.records
     ]
 
+    # Retrieve metadata using the DIDClient
+    did_client = Client()
+    retrieved_metadata = did_client.get_metadata(
+        scope=ingestion_client.scope, name=lfn, plugin="JSON"
+    )
+
+    # Verify the metadata matches the expected metadata
+    for key, value in metadata_dict.items():
+        assert retrieved_metadata.get(key) == value, (
+            f"Metadata mismatch for key '{key}'. "
+            f"Expected: {value}, Got: {retrieved_metadata.get(key)}"
+        )
+
 
 def test_rses():
     """Test that the expected RSEs are configured."""
@@ -245,14 +270,14 @@ def test_rses():
 
 @pytest.fixture
 def pre_existing_lfn(
-    test_file: tuple[Path, str],
+    onsite_test_file: tuple[Path, str],
     test_scope: str,
     test_vo: str,
 ) -> str:
     """Fixture to provide an LFN for a replica pre-registered in Rucio without using IngestionClient."""
 
     # Construct the LFN manually based on the test file and scope
-    acada_path, _ = test_file
+    acada_path, _ = onsite_test_file
     relative_path = str(acada_path).split(f"{test_vo}/{test_scope}/", 1)[-1]
     lfn = f"/{test_vo}/{test_scope}/{relative_path}"
     checksum = adler32(acada_path)
@@ -296,7 +321,7 @@ def test_add_offsite_replication_rules(
     test_vo: str,
     storage_mount_path: Path,
     tmp_path: Path,
-    test_file: tuple[Path, str],
+    onsite_test_file: tuple[Path, str],
     caplog,
 ):
     """Test the add_offsite_replication_rules method of IngestionClient."""
@@ -309,7 +334,7 @@ def test_add_offsite_replication_rules(
     lfn = pre_existing_lfn
     did = {"scope": test_scope, "name": lfn}
 
-    _, test_file_content = test_file  # Get the test file content
+    _, test_file_content = onsite_test_file  # Get the test file content
 
     offsite_rse_expression = "OFFSITE"
     copies = 2
@@ -389,7 +414,7 @@ def test_add_offsite_replication_rules_single_copy(
     test_vo: str,
     storage_mount_path: Path,
     tmp_path: Path,
-    test_file: tuple[Path, str],
+    onsite_test_file: tuple[Path, str],
     caplog,
 ):
     """Test the add_offsite_replication_rules method of IngestionClient with a single copy (copies=1)."""
@@ -402,7 +427,7 @@ def test_add_offsite_replication_rules_single_copy(
     lfn = pre_existing_lfn
     did = {"scope": test_scope, "name": lfn}
 
-    _, test_file_content = test_file
+    _, test_file_content = onsite_test_file
 
     offsite_rse_expression = "OFFSITE"
     copies = 1

bdms/tests/test_basic_rucio_functionality.py

@@ -124,7 +124,6 @@ def test_replication(test_vo, test_scope, tmp_path):
         rule_client,
         rule_id=rule,
         expected_status="OK",
-        timeout=600,
         poll_interval=5,
     )
     replicas = next(replica_client.list_replicas(dids))

bdms/tests/test_extract_fits_metadata.py

@@ -0,0 +1,97 @@
+from astropy.io import fits
+
+from bdms.extract_fits_metadata import (
+    extract_metadata_from_data,
+    extract_metadata_from_headers,
+)
+
+
+def test_extraction_correct_value_subarray_file(subarray_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(subarray_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(subarray_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the SUBARRAY FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-02-04T21:34:05",
+        "end_time": "2025-02-04T21:43:12",
+        "subarray_id": 0,
+        "sb_id": 2000000066,
+        "obs_id": 2000000200,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extraction_correct_value_tel_trigger_file(tel_trigger_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(tel_trigger_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(tel_trigger_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the Telescope TRIGGER FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-02-04T21:34:05",
+        "end_time": "2025-02-04T21:43:11",
+        "tel_ids": [1],
+        "sb_id": 2000000066,
+        "obs_id": 2000000200,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extraction_correct_value_tel_events_file(tel_events_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(tel_events_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(tel_events_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the Telescope EVENTS FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-04-01T15:25:02",
+        "end_time": "2025-04-01T15:25:03",
+        "sb_id": 0,
+        "obs_id": 0,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extract_metadata_from_data_incorrect_header(tmp_path):
+    """Test the extraction of metadata from an empty FITS file header."""
+    fits_file_path = tmp_path / "empty_fits.fits.fz"
+    hdul = fits.HDUList([fits.PrimaryHDU()])
+    hdul.writeto(fits_file_path, checksum=True)
+
+    with fits.open(fits_file_path) as hdul:
+        metadata = extract_metadata_from_headers(hdul)
+
+    assert metadata == {}, "Expected empty metadata in the header"
+
+
+def test_extract_metadata_from_data_incorrect_data(tmp_path):
+    """Test the extraction of metadata from an empty FITS file data."""
+    fits_file_path = tmp_path / "empty_fits.fits.fz"
+    hdul = fits.HDUList([fits.PrimaryHDU()])
+    hdul.writeto(fits_file_path, checksum=True)
+
+    metadata = extract_metadata_from_data(fits_file_path)
+
+    assert metadata == {}, "Expected empty metadata in the payload"

bdms/tests/test_onsite_storage.py

@@ -1,8 +1,5 @@
-import os
 import subprocess as sp
-from datetime import datetime
 from pathlib import Path
-from secrets import token_hex
 
 import pytest
 from rucio.client.rseclient import RSEClient
@@ -24,41 +21,25 @@ def test_shared_storage(storage_mount_path: Path) -> Path:
     ), f"Shared storage {storage_mount_path} is not available on the client"
 
 
-@pytest.fixture(scope="session")
-def test_file(storage_mount_path, test_scope) -> tuple[Path, str]:
-    """Create a test file in the shared storage and return its path and content"""
-    unique_id = f"{datetime.now():%Y%m%d_%H%M%S}_{token_hex(8)}"
-    test_file_name = f"/ctao.dpps.test/{test_scope}/testfile_{unique_id}.txt"
-    test_file_path = storage_mount_path / test_file_name.lstrip("/")
-    test_file_content = f"This is a test file {unique_id}"
-    test_file_path.parent.mkdir(parents=True, exist_ok=True)
-    test_file_path.write_text(test_file_content)
-    assert test_file_path.exists(), f"Test file {test_file_path} was not created successfully at {storage_mount_path}"
-
-    return test_file_name, test_file_content
-
-
-def test_file_access_from_onsite_storage_using_gfal(test_file: tuple[Path, str]):
+def test_file_access_from_onsite_storage_using_gfal(
+    storage_mount_path: Path, onsite_test_file: tuple[Path, str]
+):
     """Verify that the file is accessible from the onsite storage pod using gfal-ls"""
-    test_file_lfn, _ = test_file
-    test_file_name = os.path.basename(test_file_lfn)
+    test_file_path, _ = onsite_test_file
+    test_file_lfn = f"/{test_file_path.relative_to(storage_mount_path)}"
+    test_file_name = test_file_path.name
 
-    gfal_url = f"{STORAGE_PROTOCOL}://{STORAGE_HOSTNAME}/rucio{test_file_lfn}"
+    gfal_url = f"{STORAGE_PROTOCOL}://{STORAGE_HOSTNAME}/rucio/{test_file_lfn}"
     cmd = ["gfal-ls", gfal_url]
-    try:
-        output = sp.run(cmd, capture_output=True, text=True, check=True)
-        debug = True  # Adjust as needed
-        if debug:
-            print(f"GFAL Output: {output.stdout.strip()}")
-        stdout = output.stdout.strip()
-    except sp.CalledProcessError as e:
-        pytest.fail(
-            f"gfal-ls failed for {gfal_url}:\nSTDERR: {e.stderr.strip()}\nSTDOUT: {e.stdout.strip()}"
-        )
-
-    assert any(
-        test_file_name in line for line in stdout.splitlines()
-    ), f"File {test_file_name} not accessible; gfal-ls output: {stdout!r}"
+
+    ret = sp.run(cmd, capture_output=True, text=True)
+    stdout = ret.stdout.strip()
+    stderr = ret.stderr.strip()
+    msg = f"gfal-ls failed for {gfal_url}:\nSTDERR: {stderr}\nSTDOUT: {stderr}"
+    assert ret.returncode == 0, msg
+
+    msg = f"File {test_file_name} not accessible; gfal-ls output: {stdout!r}"
+    assert any(test_file_name in line for line in stdout.splitlines()), msg
 
 
 @pytest.mark.usefixtures("_auth_proxy")

bdms/tests/utils.py

@@ -13,9 +13,27 @@ from rucio.common.exception import RucioException
 # Default timeout and polling interval (in seconds) for waiting for replication
 DEFAULT_TIMEOUT = 1000
 DEFAULT_POLL_INTERVAL = 30
+XROOTD_UID = int(os.getenv("XROOTD_UID", 994))
+XROOTD_GID = int(os.getenv("XROOTD_GID", 994))
 LOGGER = logging.getLogger(__name__)
 
 
+def reset_xrootd_permissions(path):
+    recursive_chown(path, uid=XROOTD_UID, gid=XROOTD_GID)
+
+
+def recursive_chown(path: Path, uid: int, gid: int):
+    """Equivalent of unix chmod -R <uid>:<gid> <path>."""
+    os.chown(path, uid, gid)
+
+    for root, dirs, files in os.walk(path):
+        root = Path(root)
+        for d in dirs:
+            os.chown(root / d, uid, gid)
+        for f in files:
+            os.chown(root / f, uid, gid)
+
+
 def wait_for_replication_status(
     rule_client: RuleClient,
     rule_id: str,

{ctao_bdms_clients-0.2.0rc1.dist-info → ctao_bdms_clients-0.2.1.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.2.0rc1
+Version: 0.2.1
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
 Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms
 Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms
 Requires-Python: >=3.9
@@ -12,12 +12,14 @@ License-File: LICENSE
 Requires-Dist: astropy<8.0.0a0,>=6.0.1
 Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
 Requires-Dist: rucio-clients~=35.7.0
+Requires-Dist: protozfits>=2.7.2
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-requirements; extra == "test"
 Requires-Dist: python-dotenv; extra == "test"
 Requires-Dist: minio; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Provides-Extra: doc
 Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: numpydoc; extra == "doc"

ctao_bdms_clients-0.2.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
+bdms/_version.py,sha256=UoNvMtd4wCG76RwoSpNCUtaFyTwakGcZolfjXzNVSMY,511
+bdms/acada_ingestion.py,sha256=L-LBdfd7dbSbW0poseXsZ8CbgWch8j57yaQncIemnOs,17671
+bdms/extract_fits_metadata.py,sha256=ZGJQCFJCXkWg8N3CAb17GB-wwPj-wTvNg0JOS-MemZ0,3431
+bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
+bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bdms/tests/conftest.py,sha256=TqMBSExgO4omUHVqoXcI1AOhS3F8de-03275IlYEW6k,3896
+bdms/tests/test_acada_ingestion.py,sha256=u27Srhm8kXKtFoPr5gQdZeGDjrlECMr9ysWwGfs2w_Q,18277
+bdms/tests/test_basic_rucio_functionality.py,sha256=9GIX8IO6wBJm40LKFEH2StS-fMKvC07sxFHPVR7dftU,3583
+bdms/tests/test_dpps_rel_0_0.py,sha256=MnbuBoS_kUUiMcHE3-jqOzekQNUa-wcsjCJqJQ2J9S4,2957
+bdms/tests/test_extract_fits_metadata.py,sha256=A935WD2TF3lBcaeDmzGSlH2IXUF1v8qslrsW30lnEAA,3490
+bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
+bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
+bdms/tests/test_onsite_storage.py,sha256=waK7t9kBquzJbuLLYcpeNU9YuA70XTRS88RMxBWxawI,3765
+bdms/tests/utils.py,sha256=4g7__ms-xnTyyBKMlmV4hpC505V6uVaXJDi9XQ8UC_4,3717
+ctao_bdms_clients-0.2.1.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
+ctao_bdms_clients-0.2.1.dist-info/METADATA,sha256=1LZfXzTMTtQUqn1Npr11vfQnZ2m_GUbkdnGXWRF-e3Y,2383
+ctao_bdms_clients-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ctao_bdms_clients-0.2.1.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
+ctao_bdms_clients-0.2.1.dist-info/RECORD,,

{ctao_bdms_clients-0.2.0rc1.dist-info → ctao_bdms_clients-0.2.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ctao_bdms_clients-0.2.0rc1.dist-info/RECORD

@@ -1,18 +0,0 @@
-bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
-bdms/_version.py,sha256=6j6NVXRMR-dX2osPsF0-SkvP1-ofWxEz6ew_4VL2kCY,521
-bdms/acada_ingestion.py,sha256=bKnXbAYvtYHYQk6ir5Sw1YIjCXGZTyk3IpZz-XGkkPo,16248
-bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
-bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bdms/tests/conftest.py,sha256=lArkd8Kn7Ef19_BhqXq77taei9LKggWUu3FDUhrt9M4,3009
-bdms/tests/test_acada_ingestion.py,sha256=A0G9-ssHN3dx0Jz_eIN72dQp21gfZqdQnyAgLY3BDF4,17738
-bdms/tests/test_basic_rucio_functionality.py,sha256=GFUCq2QlM0M_5k5Qz9iPXPftE6nGuGYbW_IVS76T978,3604
-bdms/tests/test_dpps_rel_0_0.py,sha256=MnbuBoS_kUUiMcHE3-jqOzekQNUa-wcsjCJqJQ2J9S4,2957
-bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
-bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
-bdms/tests/test_onsite_storage.py,sha256=xBwVbr2q0KHnesIrF0I8ova_hfDXDs3CBya2Sxi6VWM,4633
-bdms/tests/utils.py,sha256=fh23X6iN2-lsoRBU3WSeWkweiHZlOtIUK5xzHbWyP6c,3185
-ctao_bdms_clients-0.2.0rc1.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
-ctao_bdms_clients-0.2.0rc1.dist-info/METADATA,sha256=88TkbmaMgsbU1dwCRzPHKWK-yYb323BT1HqFgsQboEg,2297
-ctao_bdms_clients-0.2.0rc1.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-ctao_bdms_clients-0.2.0rc1.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
-ctao_bdms_clients-0.2.0rc1.dist-info/RECORD,,