ctao-bdms-clients 0.2.0rc1__py3-none-any.whl → 0.3.0rc1__py3-none-any.whl

bdms/tests/test_basic_rucio_functionality.py

@@ -124,7 +124,6 @@ def test_replication(test_vo, test_scope, tmp_path):
         rule_client,
         rule_id=rule,
         expected_status="OK",
-        timeout=600,
         poll_interval=5,
     )
     replicas = next(replica_client.list_replicas(dids))

bdms/tests/test_dpps_rel_0_0.py

@@ -88,3 +88,9 @@ def test_retrieve_by_lfn(test_scope, generic_data_product, tmp_path):
     expected_path = tmp_path / test_scope / test_lfn.lstrip("/")
     assert expected_path.is_file(), "File not downloaded to expected location"
     assert expected_path.read_text() == expected_content
+
+
+@pytest.mark.verifies_usecase("DPPS-UC-110-1.3.1")
+@pytest.mark.xfail
+def test_retrieve_by_lfn_with_iam(test_scope, generic_data_product, tmp_path):
+    raise NotImplementedError

bdms/tests/test_extract_fits_metadata.py

@@ -0,0 +1,97 @@
+from astropy.io import fits
+
+from bdms.extract_fits_metadata import (
+    extract_metadata_from_data,
+    extract_metadata_from_headers,
+)
+
+
+def test_extraction_correct_value_subarray_file(subarray_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(subarray_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(subarray_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the SUBARRAY FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-02-04T21:34:05",
+        "end_time": "2025-02-04T21:43:12",
+        "subarray_id": 0,
+        "sb_id": 2000000066,
+        "obs_id": 2000000200,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extraction_correct_value_tel_trigger_file(tel_trigger_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(tel_trigger_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(tel_trigger_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the Telescope TRIGGER FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-02-04T21:34:05",
+        "end_time": "2025-02-04T21:43:11",
+        "tel_ids": [1],
+        "sb_id": 2000000066,
+        "obs_id": 2000000200,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extraction_correct_value_tel_events_file(tel_events_test_file):
+    """Test the extraction of metadata from a FITS file."""
+    with fits.open(tel_events_test_file) as hdul:
+        metadata_header = extract_metadata_from_headers(hdul)
+
+    metadata_payload = extract_metadata_from_data(tel_events_test_file)
+    metadata_fits = {**metadata_header, **metadata_payload}
+
+    assert len(metadata_fits) > 0, "No metadata found in the Telescope EVENTS FITS"
+
+    expected_keys_in_fits_file = {
+        "observatory": "CTA",
+        "start_time": "2025-04-01T15:25:02",
+        "end_time": "2025-04-01T15:25:03",
+        "sb_id": 0,
+        "obs_id": 0,
+    }
+
+    for key, value in expected_keys_in_fits_file.items():
+        assert metadata_fits[key] == value, f"Expected key '{key}' not found."
+
+
+def test_extract_metadata_from_data_incorrect_header(tmp_path):
+    """Test the extraction of metadata from an empty FITS file header."""
+    fits_file_path = tmp_path / "empty_fits.fits.fz"
+    hdul = fits.HDUList([fits.PrimaryHDU()])
+    hdul.writeto(fits_file_path, checksum=True)
+
+    with fits.open(fits_file_path) as hdul:
+        metadata = extract_metadata_from_headers(hdul)
+
+    assert metadata == {}, "Expected empty metadata in the header"
+
+
+def test_extract_metadata_from_data_incorrect_data(tmp_path):
+    """Test the extraction of metadata from an empty FITS file data."""
+    fits_file_path = tmp_path / "empty_fits.fits.fz"
+    hdul = fits.HDUList([fits.PrimaryHDU()])
+    hdul.writeto(fits_file_path, checksum=True)
+
+    metadata = extract_metadata_from_data(fits_file_path)
+
+    assert metadata == {}, "Expected empty metadata in the payload"

bdms/tests/test_onsite_storage.py

@@ -1,8 +1,5 @@
-import os
 import subprocess as sp
-from datetime import datetime
 from pathlib import Path
-from secrets import token_hex
 
 import pytest
 from rucio.client.rseclient import RSEClient
@@ -24,41 +21,25 @@ def test_shared_storage(storage_mount_path: Path) -> Path:
     ), f"Shared storage {storage_mount_path} is not available on the client"
 
 
-@pytest.fixture(scope="session")
-def test_file(storage_mount_path, test_scope) -> tuple[Path, str]:
-    """Create a test file in the shared storage and return its path and content"""
-    unique_id = f"{datetime.now():%Y%m%d_%H%M%S}_{token_hex(8)}"
-    test_file_name = f"/ctao.dpps.test/{test_scope}/testfile_{unique_id}.txt"
-    test_file_path = storage_mount_path / test_file_name.lstrip("/")
-    test_file_content = f"This is a test file {unique_id}"
-    test_file_path.parent.mkdir(parents=True, exist_ok=True)
-    test_file_path.write_text(test_file_content)
-    assert test_file_path.exists(), f"Test file {test_file_path} was not created successfully at {storage_mount_path}"
-
-    return test_file_name, test_file_content
-
-
-def test_file_access_from_onsite_storage_using_gfal(test_file: tuple[Path, str]):
+def test_file_access_from_onsite_storage_using_gfal(
+    storage_mount_path: Path, onsite_test_file: tuple[Path, str]
+):
     """Verify that the file is accessible from the onsite storage pod using gfal-ls"""
-    test_file_lfn, _ = test_file
-    test_file_name = os.path.basename(test_file_lfn)
+    test_file_path, _ = onsite_test_file
+    test_file_lfn = f"/{test_file_path.relative_to(storage_mount_path)}"
+    test_file_name = test_file_path.name
 
-    gfal_url = f"{STORAGE_PROTOCOL}://{STORAGE_HOSTNAME}/rucio{test_file_lfn}"
+    gfal_url = f"{STORAGE_PROTOCOL}://{STORAGE_HOSTNAME}/rucio/{test_file_lfn}"
     cmd = ["gfal-ls", gfal_url]
-    try:
-        output = sp.run(cmd, capture_output=True, text=True, check=True)
-        debug = True  # Adjust as needed
-        if debug:
-            print(f"GFAL Output: {output.stdout.strip()}")
-        stdout = output.stdout.strip()
-    except sp.CalledProcessError as e:
-        pytest.fail(
-            f"gfal-ls failed for {gfal_url}:\nSTDERR: {e.stderr.strip()}\nSTDOUT: {e.stdout.strip()}"
-        )
-
-    assert any(
-        test_file_name in line for line in stdout.splitlines()
-    ), f"File {test_file_name} not accessible; gfal-ls output: {stdout!r}"
+
+    ret = sp.run(cmd, capture_output=True, text=True)
+    stdout = ret.stdout.strip()
+    stderr = ret.stderr.strip()
+    msg = f"gfal-ls failed for {gfal_url}:\nSTDERR: {stderr}\nSTDOUT: {stderr}"
+    assert ret.returncode == 0, msg
+
+    msg = f"File {test_file_name} not accessible; gfal-ls output: {stdout!r}"
+    assert any(test_file_name in line for line in stdout.splitlines()), msg
 
 
 @pytest.mark.usefixtures("_auth_proxy")

bdms/tests/utils.py

@@ -13,9 +13,37 @@ from rucio.common.exception import RucioException
 # Default timeout and polling interval (in seconds) for waiting for replication
 DEFAULT_TIMEOUT = 1000
 DEFAULT_POLL_INTERVAL = 30
+XROOTD_UID = int(os.getenv("XROOTD_UID", 994))
+XROOTD_GID = int(os.getenv("XROOTD_GID", 994))
 LOGGER = logging.getLogger(__name__)
 
 
+def reset_xrootd_permissions(path):
+    recursive_chown(path, uid=XROOTD_UID, gid=XROOTD_GID)
+
+
+def recursive_chown(path: Path, uid: int, gid: int):
+    """Equivalent of unix chmod -R <uid>:<gid> <path>."""
+    os.chown(path, uid, gid)
+
+    for root, dirs, files in os.walk(path):
+        root = Path(root)
+        for d in dirs:
+            os.chown(root / d, uid, gid)
+
+        for f in files:
+            # skip temporary files created by rucio
+            # they should already have correct ownership and might go away
+            # between finding them and executing chown
+            if f.endswith(".rucio.upload"):
+                continue
+
+            try:
+                os.chown(root / f, uid, gid)
+            except Exception as e:
+                LOGGER.warning("Failed to chown file %s: %s", root / f, e)
+
+
 def wait_for_replication_status(
     rule_client: RuleClient,
     rule_id: str,

{ctao_bdms_clients-0.2.0rc1.dist-info → ctao_bdms_clients-0.3.0rc1.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.2.0rc1
+Version: 0.3.0rc1
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
 Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms
 Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms
 Requires-Python: >=3.9
@@ -12,12 +12,18 @@ License-File: LICENSE
 Requires-Dist: astropy<8.0.0a0,>=6.0.1
 Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
 Requires-Dist: rucio-clients~=35.7.0
+Requires-Dist: protozfits>=2.7.2
+Requires-Dist: watchdog>=6.0.0
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: prometheus-client>=0.22.1
+Requires-Dist: ruamel.yaml
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-requirements; extra == "test"
 Requires-Dist: python-dotenv; extra == "test"
 Requires-Dist: minio; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Provides-Extra: doc
 Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: numpydoc; extra == "doc"

ctao_bdms_clients-0.3.0rc1.dist-info/RECORD

@@ -0,0 +1,23 @@
+bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
+bdms/_version.py,sha256=ymwdyKB404aMzKXrx7y01ltePvHS_nIJjfh_zAIIN44,521
+bdms/acada_ingest_cli.py,sha256=xkf9nT5Lk7SjcbxVeBpKJWuJ-8Luze5-MSq4yki-7_k,12866
+bdms/acada_ingestion.py,sha256=mB5ilvzJbPblFp94Jcca-IzYvrMuQlroDZxuujpFB_I,36373
+bdms/extract_fits_metadata.py,sha256=ZGJQCFJCXkWg8N3CAb17GB-wwPj-wTvNg0JOS-MemZ0,3431
+bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
+bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bdms/tests/conftest.py,sha256=n7KN9foojCXDxFuZinI0MvhnSvLk5Mn7aFmjQKmO8eI,7364
+bdms/tests/test_acada_ingest_cli.py,sha256=SYVt1xlEDsrbPX0C5Isf0thjUcaxr7cjflyZSwpPBaw,8314
+bdms/tests/test_acada_ingestion.py,sha256=xQN07Qbx00IW_w0vCcR5r5H3qvvl_JNYmCUuWJX9xrc,59485
+bdms/tests/test_basic_rucio_functionality.py,sha256=9GIX8IO6wBJm40LKFEH2StS-fMKvC07sxFHPVR7dftU,3583
+bdms/tests/test_dpps_rel_0_0.py,sha256=2NhxpdhXQg_8lmK-tRrPQ_FcijsIEfv07x-kVlT8Zik,3138
+bdms/tests/test_extract_fits_metadata.py,sha256=A935WD2TF3lBcaeDmzGSlH2IXUF1v8qslrsW30lnEAA,3490
+bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
+bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
+bdms/tests/test_onsite_storage.py,sha256=waK7t9kBquzJbuLLYcpeNU9YuA70XTRS88RMxBWxawI,3765
+bdms/tests/utils.py,sha256=PUayWe60JGVDs5mkWmHVjFV_yqg5XUQlxoAvhT1P0OM,4101
+ctao_bdms_clients-0.3.0rc1.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
+ctao_bdms_clients-0.3.0rc1.dist-info/METADATA,sha256=NRkliF-xYd9V8Vfkdgj2MEFQI9ee67wmUVKxhFD9tMo,2517
+ctao_bdms_clients-0.3.0rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ctao_bdms_clients-0.3.0rc1.dist-info/entry_points.txt,sha256=YZCIOePi_xXaJunA6lAQxAKh1tn3wOd4pmqymFRvah4,60
+ctao_bdms_clients-0.3.0rc1.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
+ctao_bdms_clients-0.3.0rc1.dist-info/RECORD,,

{ctao_bdms_clients-0.2.0rc1.dist-info → ctao_bdms_clients-0.3.0rc1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ctao_bdms_clients-0.3.0rc1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+acada-ingest = bdms.acada_ingest_cli:main

ctao_bdms_clients-0.2.0rc1.dist-info/RECORD

@@ -1,18 +0,0 @@
-bdms/__init__.py,sha256=7btE6tNhFqXSv2eUhZ-0m1J3nTTs4Xo6HWcQI4eh5Do,142
-bdms/_version.py,sha256=6j6NVXRMR-dX2osPsF0-SkvP1-ofWxEz6ew_4VL2kCY,521
-bdms/acada_ingestion.py,sha256=bKnXbAYvtYHYQk6ir5Sw1YIjCXGZTyk3IpZz-XGkkPo,16248
-bdms/version.py,sha256=mTfi1WzbIs991NyImM6mcMg1R39a6U1W2pKnk-Tt5Vw,765
-bdms/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bdms/tests/conftest.py,sha256=lArkd8Kn7Ef19_BhqXq77taei9LKggWUu3FDUhrt9M4,3009
-bdms/tests/test_acada_ingestion.py,sha256=A0G9-ssHN3dx0Jz_eIN72dQp21gfZqdQnyAgLY3BDF4,17738
-bdms/tests/test_basic_rucio_functionality.py,sha256=GFUCq2QlM0M_5k5Qz9iPXPftE6nGuGYbW_IVS76T978,3604
-bdms/tests/test_dpps_rel_0_0.py,sha256=MnbuBoS_kUUiMcHE3-jqOzekQNUa-wcsjCJqJQ2J9S4,2957
-bdms/tests/test_file_replicas.py,sha256=NqutrSJa5ME50JpmyATNPSLqq1AOq1ruv84XSY3PKLI,2635
-bdms/tests/test_metadata.py,sha256=f0tSqNGlYe-ydoSDJw0k1De2kHoPl6g-GYBj_jP6kCY,3728
-bdms/tests/test_onsite_storage.py,sha256=xBwVbr2q0KHnesIrF0I8ova_hfDXDs3CBya2Sxi6VWM,4633
-bdms/tests/utils.py,sha256=fh23X6iN2-lsoRBU3WSeWkweiHZlOtIUK5xzHbWyP6c,3185
-ctao_bdms_clients-0.2.0rc1.dist-info/licenses/LICENSE,sha256=Py9riZY_f0CmXbrZ5JreE3WgglyWkRnwUfqydvX6jxE,1556
-ctao_bdms_clients-0.2.0rc1.dist-info/METADATA,sha256=88TkbmaMgsbU1dwCRzPHKWK-yYb323BT1HqFgsQboEg,2297
-ctao_bdms_clients-0.2.0rc1.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-ctao_bdms_clients-0.2.0rc1.dist-info/top_level.txt,sha256=ao0U8aA33KRHpcqmr7yrK8y2AQ6ahSu514tfaN4hDV8,5
-ctao_bdms_clients-0.2.0rc1.dist-info/RECORD,,