ctao-bdms-clients 0.2.0rc1__tar.gz → 0.2.1__tar.gz

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/.gitignore

@@ -145,3 +145,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Test folder for Fits files
+test_data/

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/.gitlab-ci.yml

@@ -1,6 +1,6 @@
 include:
   - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
-    ref: 23eafda64340b8036cbed5ce196784a03666cad8
+    ref: 7ebf44c2b02bfd72f596e7ea27ea0e9a1f925903
     file: 'ci-functions.yml'
   - "aiv-config.yml"
 
@@ -9,7 +9,6 @@ variables:
   CHART_LOCATION: chart
   CHART_NAME: bdms
   CHART_EXTRA_VALUES: "--set dev.client_image_tag=${DOCKER_TAG}"
-  DPPS_AIV_TOOLKIT_DIR: dpps-aiv-toolkit
   DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
   RUCIO_VERSION: "35.4.1"
   RUCIO_TAG: "release-${RUCIO_VERSION}"
@@ -23,6 +22,7 @@ stages:
   - sonarqube
   - publish
   - report
+  - changelog
 
 k8s-integration-tests:
   # override from toolkit

ctao_bdms_clients-0.2.1/CHANGES.rst

@@ -0,0 +1,80 @@
+BDMS v0.2.1 (2025-06-03)
+------------------------
+
+Bugfix release to allow (pre-)production deployment without test storages.
+
+Bug Fixes
+~~~~~~~~~
+
+- Allow to further configure first test RSE: set storageclass and make optional. [`!112 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/112>`__]
+
+
+BDMS 0.2.0 (2025-05-07)
+-----------------------
+
+
+API Changes
+~~~~~~~~~~~
+
+- Add basic ingestion of ACADA files (UC-110-1.1.1) [`!76 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/76>`__]
+
+- Replicate Data Products (UC-110-1.6) with test case for copies=2 [`!83 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/83>`__]
+
+- Extraction of metadata from ACADA-LST DL0 FITS files and adding metadata to ingested file (UC-110-1.1.1) [`!85 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/85>`__]
+
+- Add test case for copies=1 for the replication (UC-110-1.6) [`!87 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/87>`__]
+
+
+Bug Fixes
+~~~~~~~~~
+
+- Add missing permissions to fix onsite storage permissions [`!103 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/103>`__]
+
+
+New Features
+~~~~~~~~~~~~
+
+- Adapt one RSE as onsite and exposing its diskspace to client test pod [`!69 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/69>`__]
+
+- Add dcache and its dependencies in the BDMS helm chart (UC-170-1.6) [`!80 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/80>`__]
+
+- Add FITS checksum verification [`!94 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/94>`__]
+
+- Make storage configurable by providing additional config for RSE and also make xrootd image configurable [`!96 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/96>`__]
+
+
+Maintenance
+~~~~~~~~~~~
+
+- Fix rucio server not using correct lfn2pfn algorithm [`!72 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/72>`__]
+
+- Use test_vo fixture for ingestion tests [`!81 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/81>`__]
+
+- Update dpps release for test report and signature matrix [`!82 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/82>`__]
+
+- Update BDMS docs for Rel 0.1 [`!89 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/89>`__]
+
+- Update fts image and toolkit [`!90 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/90>`__]
+
+- Added download_test_file in utils.py to download a FITS file from MinIO server [`!93 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/93>`__]
+
+- Update rucio to 35.7.0 [`!97 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/97>`__]
+
+- Update FTS subchart version [`!99 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/99>`__]
+
+- Resolve Test container build fails due to missing -y in autoremove [`!101 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/101>`__]
+
+
+Refactoring and Optimization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Update FTS chart to 0.3 to reduce integration test time [`!100 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/100>`__]
+
+BDMS v0.1.0 (2025-02-21)
+---------------------------
+
+First release of the Bulk Data Management System (BDMS).
+
+* Deployment of Rucio 35.4 using helm.
+* Client package pinning the correct rucio and rucio policy package versions.
+* Integration tests for DPPS release 0.0 use cases.

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/Dockerfile

@@ -22,7 +22,7 @@ USER root
 COPY --from=builder /tmp/dist/ /tmp/dist/
 RUN  dnf install -y --setopt=install_weak_deps=False git \
   && python3 -m pip install --no-cache-dir /tmp/dist/ctao_bdms* \
-  && dnf autoremove \
+  && dnf autoremove -y \
   && dnf clean all
 
 USER user

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.2.0rc1
+Version: 0.2.1
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
 Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms
 Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms
 Requires-Python: >=3.9
@@ -12,12 +12,14 @@ License-File: LICENSE
 Requires-Dist: astropy<8.0.0a0,>=6.0.1
 Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
 Requires-Dist: rucio-clients~=35.7.0
+Requires-Dist: protozfits>=2.7.2
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-requirements; extra == "test"
 Requires-Dist: python-dotenv; extra == "test"
 Requires-Dist: minio; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Provides-Extra: doc
 Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: numpydoc; extra == "doc"

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/aiv-config.yml

@@ -15,3 +15,5 @@ variables:
   APPLICATION_AUTHOR_ORGANIZATION: CTAO
 
   EXTRA_CONFIG_FILES: aiv-config-dependencies.yml
+
+  DEFAULT_HELM_TEST: name=bdms-pytest

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/Chart.yaml

@@ -12,7 +12,7 @@ keywords:
 
 maintainers:
 - name: The BDMS Authors
-  email: ""
+  email: ''
 
 dependencies:
 - name: postgresql
@@ -30,7 +30,7 @@ dependencies:
 
 - name: cert-generator-grid
   condition: cert-generator-grid.enabled
-  version: v0.2.2
+  version: v2.0.0
   repository: oci://harbor.cta-observatory.org/dpps
 
 - name: fts

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/README.md

@@ -14,7 +14,7 @@ A Helm chart for the bdms project
 
 | Repository | Name | Version |
 |------------|------|---------|
-| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v0.2.2 |
+| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v2.0.0 |
 | oci://harbor.cta-observatory.org/dpps | fts | v0.3.0 |
 | oci://harbor.cta-observatory.org/dpps | rucio-daemons | 35.0.0 |
 | oci://harbor.cta-observatory.org/dpps | rucio-server | 35.0.0 |
@@ -43,6 +43,7 @@ A Helm chart for the bdms project
 | database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The Rucio database connection URI |
 | dev.client_image_tag | string | `nil` |  |
 | dev.mount_repo | bool | `true` |  |
+| dev.n_test_jobs | int | `4` | number of jobs to use for pytest |
 | dev.run_tests | bool | `true` |  |
 | dev.sleep | bool | `false` | sleep after test to allow interactive development |
 | fts.enabled | bool | `true` | Specifies the configuration for FTS test step (FTS server, FTS database, and ActiveMQ broker containers). Enables or disables the deployment of a FTS instance for testing. This is set to 'False' if an external FTS is used |
@@ -145,7 +146,9 @@ A Helm chart for the bdms project
 | server.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
 | server.rucioHost | string | `"rucio-server-manual-tc.local"` | The hostname of the Rucio server. It is used by clients and services to communicate with Rucio |
 | suffix_namespace | string | `"default"` | Specifies the Namespace suffix used for managing deployments in kubernetes |
-| test_storages | object | `{"xrootd":{"image":{"repository":"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd","tag":"37.1.0"},"instances":["rucio-storage-1","rucio-storage-2","rucio-storage-3"]}}` | - A list of test storages, deployed in the test setup |
+| test_storages | object | `{"enabled":true,"xrootd":{"image":{"repository":"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd","tag":"37.1.0"},"instances":["rucio-storage-1","rucio-storage-2","rucio-storage-3"],"rucio_storage_1_storage_class":"standard"}}` | - A list of test storages, deployed in the test setup |
+| test_storages.enabled | bool | `true` | If true, deploys test storages for testing purposes. This is set to 'False' if an external storage is used as in the production setup |
 | test_storages.xrootd.image.repository | string | `"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd"` | The container image repository for the XRootD storage deployment |
 | test_storages.xrootd.image.tag | string | `"37.1.0"` | Defines the specific version of the XRootD image to use |
+| test_storages.xrootd.rucio_storage_1_storage_class | string | `"standard"` | The storage class name for the PVC used by rucio-storage-1 |
 

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/scripts/certificates/install_ca.sh

@@ -1,7 +1,6 @@
 hash=$(openssl x509 -noout -hash -in /etc/grid-security/ca.pem)
 
 mkdir -pv /etc/grid-security/certificates
-update-ca-trust force-enable
 
 cp -fv /etc/grid-security/ca.pem  /etc/pki/ca-trust/source/anchors/rucio-ca.pem
 update-ca-trust extract

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/templates/bootstrap_jobs.yaml

@@ -47,7 +47,7 @@ spec:
         - /bin/sh
         - -c
         - |
-          set -x
+          set -ex
 
           {{ .Files.Get "scripts/certificates/install_ca.sh" | indent 10 }}
           {{ .Files.Get "scripts/bootstrap_rucio/wait_for_rucio.sh" | indent 10 }}
@@ -56,6 +56,12 @@ spec:
 
           python3 /usr/local/rucio/tools/reset_database.py
 
+          curl -LO https://dl.k8s.io/release/v1.33.0/bin/linux/amd64/kubectl -o ./kubectl
+          chmod +x ./kubectl
+
+          # could be also in post-upgrade hook but only if bootstrap was performed
+          ./kubectl rollout restart deployment {{ .Release.Name }}-rucio-server
+
         volumeMounts:
         {{ include "volume_mounts_rucio_config" . | indent 8 }}
         {{ include "volume_mounts_cert" . | indent 8 }}

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/templates/test_storages.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.test_storages.enabled }}
 {{- $suffix_namespace := .Values.suffix_namespace -}}
 {{ $values := .Values -}}
 {{- range $name := .Values.test_storages.xrootd.instances }}
@@ -44,7 +45,7 @@ spec:
       {{- if eq $name "rucio-storage-1" }}
       initContainers: #Initialize the PersistentVolumeClaim volume
         - name: bdms-storage-1-init #Name
-          image: busybox:latest #You can use another image, like alpine/BusyBox, but must provide sh and chmod
+          image: harbor.cta-observatory.org/proxy_cache/busybox:latest #You can use another image, like alpine/BusyBox, but must provide sh and chmod
           command: ["/bin/sh", "-c", "mkdir -p /rucio/data && chmod -R 777 /rucio/data"] #This make the folder and sets all permissions correct
           volumeMounts:
             - name: bdms-storage-1-data
@@ -156,4 +157,5 @@ spec:
   resources:
     requests:
       storage: 2Gi  # Adjust size as needed
-  storageClassName: standard
+  storageClassName: {{ .Values.test_storages.xrootd.rucio_storage_1_storage_class }}
+{{ end }}

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/templates/tests/test_jobs.yaml

@@ -76,7 +76,8 @@ spec:
         --junitxml=/tmp/report.xml \
         --color=yes \
         --log-cli-level=INFO \
-        --durations=10 --durations-min=30
+        --durations=10 --durations-min=30 \
+        -n {{ .Values.dev.n_test_jobs }}
 
       curl -T /tmp/coverage.xml http://testkit:8000/coverage.xml
       curl -T /tmp/report.xml http://testkit:8000/report.xml

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/chart/values.yaml

@@ -176,6 +176,8 @@ configure:
 
 # --- A list of test storages, deployed in the test setup
 test_storages:
+  # -- If true, deploys test storages for testing purposes. This is set to 'False' if an external storage is used as in the production setup
+  enabled: true
   xrootd:
     image:
       # -- The container image repository for the XRootD storage deployment
@@ -187,6 +189,8 @@ test_storages:
     - "rucio-storage-1"
     - "rucio-storage-2"
     - "rucio-storage-3"
+    # -- The storage class name for the PVC used by rucio-storage-1
+    rucio_storage_1_storage_class: standard
 
 rucio_db:
   # -- The database connection URI for Rucio. It is of the format: `postgresql://<user>:<password>@<host>:<port>/<database>`, this field in use only if 'existingSecret.enabled' is set to 'false', otherwise ignored
@@ -452,6 +456,8 @@ dev:
   run_tests: true
   mount_repo: true
   client_image_tag:
+  # -- number of jobs to use for pytest
+  n_test_jobs: 4
 
 cert-generator-grid:
   enabled: true

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/docs/changelog.rst

@@ -3,4 +3,5 @@ Change Log
 
 .. changelog::
    :towncrier: ../
+   :towncrier-skip-if-empty:
    :changelog_file: ../CHANGES.rst

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "ctao-bdms-clients"
 description = "Client module for the CTAO DPPS Bulk Data Management System"
 readme = "README.md"
-license = {text = "BSD-3-Clause"}
+license = "BSD-3-Clause"
 authors = [
   {name = "Georgios Zacharis", email = "georgios.zacharis@inaf.it"},
   {name = "Stefano Gallozzi", email = "Stefano.gallozzi@inaf.it"},
@@ -21,6 +21,7 @@ dependencies = [
   "astropy>=6.0.1,<8.0.0a0",
   "ctao-bdms-rucio-policy~=0.1.0",
   "rucio-clients~=35.7.0",
+  "protozfits>=2.7.2",
 ]
 
 # needed for setuptools_scm, we don"t define a static version
@@ -41,6 +42,7 @@ test = [
   "pytest-requirements",
   "python-dotenv",
   "minio",
+  "pytest-xdist",
 ]
 doc = [
   "sphinx",

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/sonar-project.properties

@@ -8,5 +8,5 @@ sonar.language=python
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.python.version=3
 # ignore submodule, is external
-sonar.exclusions=dpps-aiv-toolkit/**,test-report/**,dpps-release-plan/**,requirements-traceability/**,chart/charts/rucio-daemons/patches/request_patched.py
-sonar.coverage.exclusions=**/tests/**,**/conftest.py,**/test_*.py
+sonar.exclusions=dpps-aiv-toolkit/**,test-report/**,dpps-release-plan/**,requirements-traceability/**,chart/charts/rucio-daemons/patches/request_patched.py,src/bdms/_version.py,src/bdms/_dev_version/__init__.py
+sonar.coverage.exclusions=**/tests/**,**/conftest.py,**/test_*.py,docs/conf.py

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/src/bdms/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0rc1'
-__version_tuple__ = version_tuple = (0, 2, 0, 'rc1')
+__version__ = version = '0.2.1'
+__version_tuple__ = version_tuple = (0, 2, 1)

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/src/bdms/acada_ingestion.py

@@ -4,16 +4,16 @@ This module provides the IngestionClient class to manage the ingestion of ACADA
 It includes functionality for constructing FITS file paths, converting ACADA paths to Logical File Names (LFNs),
 and registering replicas in Rucio.
 """
-# this is the latest working file (as on 6:40 pm) with judge repairer repairing the STUCK rule.
 
 import logging
 import os
+from contextlib import ExitStack
 from pathlib import Path
 from typing import Optional, Union
 
 from astropy.io import fits
 from rucio.client.accountclient import AccountClient
-from rucio.client.client import Client
+from rucio.client.client import Client, DIDClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.client.rseclient import RSEClient
 from rucio.client.ruleclient import RuleClient
@@ -21,6 +21,11 @@ from rucio.client.scopeclient import ScopeClient
 from rucio.common.exception import Duplicate, RucioException
 from rucio.common.utils import adler32
 
+from bdms.extract_fits_metadata import (
+    extract_metadata_from_data,
+    extract_metadata_from_headers,
+)
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -91,6 +96,7 @@ class IngestionClient:
             self.account_client = AccountClient()
             self.rse_client = RSEClient()
             self.rule_client = RuleClient()
+            self.did_client = DIDClient()
         except RucioException as e:
             self.logger.error("Failed to initialize Rucio clients: %s", str(e))
             raise
@@ -261,9 +267,10 @@ class IngestionClient:
             return lfn
 
         # Proceed with registering the replica if check_replica_exists returns False
+        valid, metadata = verify_and_extract_metadata(acada_path)
+        metadata["valid_fits_checksum"] = valid
 
-        # Compute file metadata
-        # TODO: use functions to identify file type, extract metadata, validate integrity, when this functionality is ready https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/work_items/46
+        # Compute rucio file metadata
         file_size = acada_path.stat().st_size
         checksum = adler32(acada_path)
 
@@ -286,6 +293,10 @@ class IngestionClient:
             )
         self.logger.info("Successfully registered the replica for lfn '%s'", lfn)
 
+        if len(metadata) > 0:
+            self.did_client.set_metadata_bulk(scope=self.scope, name=lfn, meta=metadata)
+            self.logger.info("Set metadata of %r to %r", lfn, metadata)
+
         return lfn
 
     def add_offsite_replication_rules(
@@ -433,3 +444,36 @@ def verify_fits_checksum(hdul: fits.HDUList):
             raise FITSVerificationError(msg)
         elif checksum_result == 2 and pos != 0:  # ignore primary for warning
             LOGGER.warning("No CHECKSUM in HDU %d with name %r", pos, name)
+
+
+def verify_and_extract_metadata(fits_path):
+    """Verify checksums and extract metadata from FITS files.
+
+    This wrapper transforms exceptions into log errors and minimizes
+    the number of times the FITS file has to be opened.
+    """
+    # this context manager allows elegant handling
+    # of conditionally present context managers
+    # which allows better handling of exceptions below
+    context = ExitStack()
+    metadata = {}
+    with context:
+        try:
+            hdul = context.enter_context(fits.open(fits_path))
+        except Exception as e:
+            LOGGER.error("Failed to open FITS file %r: %s", fits_path, e)
+            return False, metadata
+
+        try:
+            verify_fits_checksum(hdul)
+        except FITSVerificationError as e:
+            LOGGER.error("File %r failed FITS checksum verification: %s", fits_path, e)
+            return False, metadata
+
+        try:
+            metadata = extract_metadata_from_headers(hdul)
+            metadata.update(extract_metadata_from_data(fits_path))
+            return True, metadata
+        except Exception as e:
+            LOGGER.error("Failed to extract metadata from %r: %s", fits_path, e)
+            return False, metadata

ctao_bdms_clients-0.2.1/src/bdms/extract_fits_metadata.py

@@ -0,0 +1,134 @@
+"""Functions to extract metadata from input files."""
+
+import logging
+
+import numpy as np
+from protozfits import File
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+# COMMON HEADER
+start_time = "DataStream.DATE"
+
+# COMMON DATA
+origin = "DataStream.ORIGIN"
+sb_id = "DataStream.sb_id"
+obs_id = "DataStream.obs_id"
+
+# -- FOR TEL_TRIG
+tel_ids = "DataStream.tel_ids"
+
+# -- FOR TEL_SUB
+subarray_id = "DataStream.subarray_id"
+
+METADATA_TEL = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "Events.DATEEND",
+    },
+    "PAYLOAD": {
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+METADATA_SUB = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "SubarrayEvents.DATEEND",
+    },
+    "PAYLOAD": {
+        "subarray_id": subarray_id,
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+METADATA_TRIG = {
+    "HEADER": {
+        "observatory": origin,
+        "start_time": start_time,
+        "end_time": "Triggers.DATEEND",
+    },
+    "PAYLOAD": {
+        "tel_ids": tel_ids,
+        "sb_id": sb_id,
+        "obs_id": obs_id,
+    },
+}
+
+#: Mapping from DataStream.PBFHEAD to the metadata items we want to collect
+METADATA_SCHEMAS = {
+    "DL0v1.Trigger.DataStream": METADATA_TRIG,
+    "DL0v1.Subarray.DataStream": METADATA_SUB,
+    "DL0v1.Telescope.DataStream": METADATA_TEL,
+}
+
+
+def extract_metadata_from_headers(hdul):
+    """Extract metadata from FITS headers of hdul."""
+    all_headers = {}
+    for hdu in hdul:
+        if hdu.is_image:
+            continue
+        all_headers[hdu.name] = dict(hdu.header)
+
+    try:
+        all_headers["DataStream"]
+    except KeyError:
+        logger.error("No DataStream HDU found in the FITS file.")
+        return {}
+
+    pbfhead = all_headers["DataStream"]["PBFHEAD"]
+    schema = METADATA_SCHEMAS.get(pbfhead)
+    if schema is None:
+        logger.error(
+            "The PBFHEAD %r does not correspond to any known FITS type.", pbfhead
+        )
+        return {}
+
+    logger.debug("Headers extracted: %s", all_headers.keys())
+
+    metadata = {}
+    for value_name, metadata_path in schema["HEADER"].items():
+        extname, header_key = metadata_path.split(".")
+        table = all_headers[extname][header_key]
+        metadata[value_name] = table
+
+    return metadata
+
+
+def extract_metadata_from_data(path):
+    """Extract metadata from zFITS payload in path."""
+    with File(path) as f:
+        if not hasattr(f, "DataStream"):
+            return {}
+
+        pbfhead = f.DataStream.header["PBFHEAD"]
+        schema = METADATA_SCHEMAS.get(pbfhead)
+        if schema is None:
+            logger.error(
+                "The PBFHEAD %r does not correspond to any known FITS type.", pbfhead
+            )
+            return {}
+
+        metadata = {}
+        for value_name, metadata_path in schema["PAYLOAD"].items():
+            hdu, column = metadata_path.split(".")
+            row = getattr(f, hdu)[0]
+            metadata[value_name] = getattr(row, column)
+
+            if isinstance(metadata[value_name], np.ndarray):
+                # Convert numpy array to a Python list
+                metadata[value_name] = metadata[value_name].tolist()
+
+            logger.debug(
+                "Value '%s' from '%s' extracted. (renamed as '%s')",
+                column,
+                hdu,
+                value_name,
+            )
+        return metadata

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.2.1}/src/bdms/tests/conftest.py

@@ -8,7 +8,7 @@ from secrets import token_hex
 import pytest
 from rucio.client.scopeclient import ScopeClient
 
-from bdms.tests.utils import download_test_file
+from bdms.tests.utils import download_test_file, reset_xrootd_permissions
 
 USER_CERT = os.getenv("RUCIO_CFG_CLIENT_CERT", "/opt/rucio/etc/usercert.pem")
 USER_KEY = os.getenv("RUCIO_CFG_CLIENT_KEY", "/opt/rucio/etc/userkey.pem")
@@ -90,5 +90,28 @@ def tel_trigger_test_file():
 @pytest.fixture(scope="session")
 def tel_events_test_file():
     """Fixture to download a telescope events test file"""
-    path = "acada-small/DL0/LSTN-01/ctao-n-acada/acada-adh/events/2025/02/04/TEL001_SDH0000_20250204T213354_SBID0000000002000000066_OBSID0000000002000000200_CHUNK000.fits.fz"
+    path = "acada-small/DL0/LSTN-01/ctao-n-acada/acada-adh/events/2025/02/04/TEL001_SDH0000_20250204T213354_SBID0000000002000000066_OBSID0000000002000000200_CHUNK001.fits.fz"
     return download_test_file(path)
+
+
+@pytest.fixture
+def onsite_test_file(
+    storage_mount_path: Path, test_scope: str, test_vo: str
+) -> tuple[Path, str]:
+    """Create a dummy file in the shared storage for testing."""
+
+    unique_id = f"{datetime.now():%Y%m%d_%H%M%S}_{token_hex(8)}"
+    filename = f"testfile_{unique_id}.txt"
+
+    test_file_path = storage_mount_path / test_vo / test_scope / filename
+    test_file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write a small test content (simulating a .fits.fz file with minimal content for testing)
+    test_file_content = f"Test file with random content: {unique_id}"
+    test_file_path.write_text(test_file_content)
+
+    # need to change file permissions of created directories so that
+    # the xrootd still can read and write there
+    reset_xrootd_permissions(storage_mount_path)
+
+    return test_file_path, test_file_content