ctao-bdms-clients 0.1.0rc3__tar.gz → 0.2.0__tar.gz

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/.gitignore

@@ -1,4 +1,8 @@
-# Files downloaded by the dpps-aiv-toolkit
+# Downloaded test data
+test_data/
+
+# Files created by the dpps-aiv-toolkit
+logger.pid
 Chart.lock
 helm
 helm.tar.gz
@@ -141,3 +145,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Test folder for Fits files
+test_data/

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/.gitlab-ci.yml

@@ -1,6 +1,6 @@
 include:
   - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
-    ref: a79203bb9b79e2dc997c1572a209ad24c5c59ae3
+    ref: dd87c1aaf379ece916a8e11c069e79d07b521b56
     file: 'ci-functions.yml'
   - "aiv-config.yml"
 
@@ -8,13 +8,13 @@ include:
 variables:
   CHART_LOCATION: chart
   CHART_NAME: bdms
-  CHART_EXTRA_VALUES: "--set client_image_tag=${DOCKER_TAG}"
-  DPPS_AIV_TOOLKIT_DIR: dpps-aiv-toolkit
+  CHART_EXTRA_VALUES: "--set dev.client_image_tag=${DOCKER_TAG}"
   DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
   RUCIO_VERSION: "35.4.1"
   RUCIO_TAG: "release-${RUCIO_VERSION}"
 
 stages:
+  - prepare
   - lint
   - build
   - sign
@@ -23,6 +23,14 @@ stages:
   - publish
   - report
 
+k8s-integration-tests:
+  # override from toolkit
+  before_script:
+    - apk add --no-cache make wget || true # might fail depending on the image
+    - echo fs.inotify.max_user_watches=524288 | tee -a /etc/sysctl.conf && echo fs.inotify.max_user_instances=8192 | tee -a /etc/sysctl.conf && sysctl -p || true
+    # create .env file with CI secrets
+    - echo -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY\nMINIO_SECRET_KEY=$MINIO_SECRET_KEY\n" > .env
+
 build:
   variables:
     CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-client:${DOCKER_TAG}'

ctao_bdms_clients-0.2.0/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "dpps-aiv-toolkit"]
+	path = dpps-aiv-toolkit
+	url = ../../aiv/dpps-aiv-toolkit.git

ctao_bdms_clients-0.2.0/CHANGES.rst

@@ -0,0 +1,69 @@
+BDMS 0.2.0 (2025-05-07)
+-----------------------
+
+
+API Changes
+~~~~~~~~~~~
+
+- Add basic ingestion of ACADA files (UC-110-1.1.1) [`!76 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/76>`__]
+
+- Replicate Data Products (UC-110-1.6) with test case for copies=2 [`!83 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/83>`__]
+
+- Extraction of metadata from ACADA-LST DL0 FITS files and adding metadata to ingested file (UC-110-1.1.1) [`!85 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/85>`__]
+
+- Add test case for copies=1 for the replication (UC-110-1.6) [`!87 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/87>`__]
+
+
+Bug Fixes
+~~~~~~~~~
+
+- Add missing permissions to fix onsite storage permissions [`!103 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/103>`__]
+
+
+New Features
+~~~~~~~~~~~~
+
+- Adapt one RSE as onsite and exposing its diskspace to client test pod [`!69 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/69>`__]
+
+- Add dcache and its dependencies in the BDMS helm chart (UC-170-1.6) [`!80 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/80>`__]
+
+- Add FITS checksum verification [`!94 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/94>`__]
+
+- Make storage configurable by providing additional config for RSE and also make xrootd image configurable [`!96 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/96>`__]
+
+
+Maintenance
+~~~~~~~~~~~
+
+- Fix rucio server not using correct lfn2pfn algorithm [`!72 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/72>`__]
+
+- Use test_vo fixture for ingestion tests [`!81 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/81>`__]
+
+- Update dpps release for test report and signature matrix [`!82 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/82>`__]
+
+- Update BDMS docs for Rel 0.1 [`!89 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/89>`__]
+
+- Update fts image and toolkit [`!90 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/90>`__]
+
+- Added download_test_file in utils.py to download a FITS file from MinIO server [`!93 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/93>`__]
+
+- Update rucio to 35.7.0 [`!97 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/97>`__]
+
+- Update FTS subchart version [`!99 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/99>`__]
+
+- Resolve Test container build fails due to missing -y in autoremove [`!101 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/101>`__]
+
+
+Refactoring and Optimization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Update FTS chart to 0.3 to reduce integration test time [`!100 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/100>`__]
+
+BDMS v0.1.0 (2025-02-21)
+---------------------------
+
+First release of the Bulk Data Management System (BDMS).
+
+* Deployment of Rucio 35.4 using helm.
+* Client package pinning the correct rucio and rucio policy package versions.
+* Integration tests for DPPS release 0.0 use cases.

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/Dockerfile

@@ -1,7 +1,7 @@
  # rucio version used as base for the final image
-ARG RUCIO_TAG=release-35.4.1
+ARG RUCIO_TAG=release-35.7.0
 
-FROM python:3.9 AS builder
+FROM harbor.cta-observatory.org/proxy_cache/python:3.9 AS builder
 
 COPY pyproject.toml MANIFEST.in /tmp/bdms/
 COPY .git /tmp/bdms/.git/
@@ -13,7 +13,7 @@ RUN python -m pip install build \
 # second stage, copy and install wheel
 # We are using the official python 3.11 image
 # as base image in the slim variant to reduce image size.
-FROM rucio/rucio-clients:${RUCIO_TAG}
+FROM harbor.cta-observatory.org/proxy_cache/rucio/rucio-clients:${RUCIO_TAG}
 
 ARG RUCIO_TAG
 
@@ -22,7 +22,7 @@ USER root
 COPY --from=builder /tmp/dist/ /tmp/dist/
 RUN  dnf install -y --setopt=install_weak_deps=False git \
   && python3 -m pip install --no-cache-dir /tmp/dist/ctao_bdms* \
-  && dnf autoremove \
+  && dnf autoremove -y \
   && dnf clean all
 
 USER user

ctao_bdms_clients-0.2.0/MANIFEST.in

@@ -0,0 +1,2 @@
+prune src/bdms/_dev_version
+prune .github

{ctao_bdms_clients-0.1.0rc3/src/ctao_bdms_clients.egg-info → ctao_bdms_clients-0.2.0}/PKG-INFO

@@ -1,31 +1,38 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.1.0rc3
+Version: 0.2.0
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
 Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms
 Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: rucio-clients~=35.4.1
-Requires-Dist: ctao-bdms-rucio-policy==0.1.0
+Requires-Dist: astropy<8.0.0a0,>=6.0.1
+Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
+Requires-Dist: rucio-clients~=35.7.0
+Requires-Dist: protozfits>=2.7.2
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-requirements; extra == "test"
+Requires-Dist: python-dotenv; extra == "test"
+Requires-Dist: minio; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Provides-Extra: doc
 Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: numpydoc; extra == "doc"
 Requires-Dist: ctao-sphinx-theme; extra == "doc"
 Requires-Dist: myst-parser; extra == "doc"
 Requires-Dist: sphinx-changelog; extra == "doc"
+Requires-Dist: sphinx-automodapi; extra == "doc"
 Provides-Extra: dev
 Requires-Dist: setuptools_scm; extra == "dev"
 Requires-Dist: sphinx-autobuild; extra == "dev"
 Provides-Extra: all
 Requires-Dist: bdms[dev,doc,test]; extra == "all"
+Dynamic: license-file
 
 # Bulk Data Management System
 

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/aiv-config.yml

@@ -3,16 +3,15 @@
 variables:
   # Comma-separated list of "UC Groups" to select UCs and Requirements, e.g. "BDMS,AIV"
   DPPS_UC_GROUPS: BDMS
-  DPPS_RELEASE: v0.0
+  DPPS_RELEASE: v0.1
 
   # these paths are relative to TEST_ARTIFACTS_PATH
   REPORT_XML: report.xml
-  RELEASE_PLAN_FN: dpps-release-plan/release_development_document/dpps-release-plan.tex
-  TRACEABILITY_CSV_FN: requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
+  RELEASE_PLAN_FN: dpps-aiv-toolkit/dpps-release-plan/release_development_document/dpps-release-plan.tex
+  TRACEABILITY_CSV_FN: dpps-aiv-toolkit/requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
 
   APPLICATION_NAME: BDMS
   APPLICATION_AUTHOR: BDMS Team
   APPLICATION_AUTHOR_ORGANIZATION: CTAO
-  APPLICATION_VERSION: 0.0.0
 
   EXTRA_CONFIG_FILES: aiv-config-dependencies.yml

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/chart/Chart.yaml

@@ -12,7 +12,7 @@ keywords:
 
 maintainers:
 - name: The BDMS Authors
-  email:
+  email: ''
 
 dependencies:
 - name: postgresql
@@ -30,10 +30,10 @@ dependencies:
 
 - name: cert-generator-grid
   condition: cert-generator-grid.enabled
-  version: 0.0.0-75a4994-75a4994c
+  version: v1.0.0
   repository: oci://harbor.cta-observatory.org/dpps
 
 - name: fts
   condition: fts.enabled
-  version: v0.1.0
+  version: v0.3.0
   repository: oci://harbor.cta-observatory.org/dpps

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/chart/README.md

@@ -14,8 +14,8 @@ A Helm chart for the bdms project
 
 | Repository | Name | Version |
 |------------|------|---------|
-| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | 0.0.0-75a4994-75a4994c |
-| oci://harbor.cta-observatory.org/dpps | fts | v0.1.0 |
+| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v1.0.0 |
+| oci://harbor.cta-observatory.org/dpps | fts | v0.3.0 |
 | oci://harbor.cta-observatory.org/dpps | rucio-daemons | 35.0.0 |
 | oci://harbor.cta-observatory.org/dpps | rucio-server | 35.0.0 |
 | oci://registry-1.docker.io/bitnamicharts | postgresql | 15.5.10 |
@@ -32,22 +32,23 @@ A Helm chart for the bdms project
 | auth.certificate.letsencrypt.email | string | `""` | Email address for Let's encrypt registration and renewal reminders |
 | auth.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
 | bootstrap.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image for bootstrapping Rucio (initialization, configuration) with the CTAO Rucio policy package installed |
-| bootstrap.image.tag | string | `"35.4.1-v0.1.0"` | The specific image tag to use for the bootstrap container |
+| bootstrap.image.tag | string | `"35.7.0-v0.2.0"` | The specific image tag to use for the bootstrap container |
 | cert-generator-grid.enabled | bool | `true` |  |
 | cert-generator-grid.generatePreHooks | bool | `true` |  |
-| client_image_tag | string | `"6cbd744c"` |  |
+| configure | object | `{"extra_script":"# add a scope\nrucio-admin scope add --account root --scope root\nrucio add-container /ctao.dpps.test\n","identities":[{"account":"root","email":"dpps-test@cta-observatory.org","id":"CN=DPPS User","type":"X509"}],"rse_distances":[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]],"rses":{"STORAGE-1":{"attributes":{"ANY":true,"ONSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-1","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"rse_type":"DISK"},"STORAGE-2":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-2","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true},"STORAGE-3":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-3","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true}}}` | a list of Rucio Storage Elements (RSE) TODO: make more clear mechanism to handle different upgrade scenarios If there is a conflict between existing configuration, the configuration will fail. In this case, likely the configuration should be deleted and re-created. |
+| configure.extra_script | string | `"# add a scope\nrucio-admin scope add --account root --scope root\nrucio add-container /ctao.dpps.test\n"` | This script is executed after the Rucio server is deployed and configured. It can be used to perform additional configuration or setup tasks if they currently cannot be done with the chart values. |
+| configure.rse_distances | list | `[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]]` | A list of RSE distance specifications, each a list of 4 values: source RSE, destination RSE, distance (integer), and ranking (integer) |
 | configure_test_setup | bool | `true` | This will configure the rucio server with the storages |
 | database | object | `{"default":"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"}` | Databases Credentials used by Rucio to access the database. If postgresql subchart is deployed, these credentials should match those in postgresql.global.postgresql.auth. If postgresql subchart is not deployed, an external database must be provided |
 | database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The Rucio database connection URI |
+| dev.client_image_tag | string | `nil` |  |
 | dev.mount_repo | bool | `true` |  |
+| dev.n_test_jobs | int | `4` | number of jobs to use for pytest |
 | dev.run_tests | bool | `true` |  |
 | dev.sleep | bool | `false` | sleep after test to allow interactive development |
 | fts.enabled | bool | `true` | Specifies the configuration for FTS test step (FTS server, FTS database, and ActiveMQ broker containers). Enables or disables the deployment of a FTS instance for testing. This is set to 'False' if an external FTS is used |
 | fts.ftsdb_password | string | `"SDP2RQkbJE2f+ohUb2nUu6Ae10BpQH0VD70CsIQcDtM"` | Defines the password for the FTS database user |
 | fts.ftsdb_root_password | string | `"iB7dMiIybdoaozWZMkvRo0eg9HbQzG9+5up50zUDjE4"` | Defines the root password for the FTS database |
-| fts.image.pullPolicy | string | `"Always"` |  |
-| fts.image.repository | string | `"harbor.cta-observatory.org/proxy_cache/rucio/fts"` | The container image repository for the FTS deployment |
-| fts.image.tag | string | `"35.4.1"` | Defines the specific version of the FTS image to use |
 | fts.messaging.broker | string | `"localhost:61613"` |  |
 | fts.messaging.password | string | `"topsecret"` |  |
 | fts.messaging.use_broker_credentials | string | `"true"` |  |
@@ -60,6 +61,7 @@ A Helm chart for the bdms project
 | rethinkdb.enabled | bool | `false` |  |
 | rethinkdb.storageClassName | string | `nil` |  |
 | rethinkdb.storageSize | string | `"1Gi"` |  |
+| rucio-daemons.config.common.extract_scope | string | `"ctao_bdms"` |  |
 | rucio-daemons.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | Specifies the connection URI for the Rucio database, these settings will be written to 'rucio.cfg' |
 | rucio-daemons.config.messaging-fts3.brokers | string | `"fts-activemq"` | Specifies the message broker used for FTS messaging |
 | rucio-daemons.config.messaging-fts3.destination | string | `"/topic/transfer.fts_monitoring_complete"` | Specifies the message broker queue path where FTS sends transfer status updates. This is the place where Rucio listens for completed transfer notifications |
@@ -68,7 +70,11 @@ A Helm chart for the bdms project
 | rucio-daemons.config.messaging-fts3.port | int | `61613` | Defines the port used for the broker |
 | rucio-daemons.config.messaging-fts3.use_ssl | bool | `false` | Determines whether to use SSL for message broker connections. If true, valid certificates are required for securing the connection |
 | rucio-daemons.config.messaging-fts3.username | string | `"fts"` | Specifies the authentication credential (username) for connecting to the message broker |
-| rucio-daemons.config.policy.permission | string | `"ctao"` | Defines the policy permission model for Rucio for determining how authorization and access controls are applied, its value should be taken from the installed Rucio policy package |
+| rucio-daemons.config.messaging-fts3.voname | string | `"ctao"` |  |
+| rucio-daemons.config.policy.lfn2pfn_algorithm_default | string | `"ctao_bdms"` |  |
+| rucio-daemons.config.policy.package | string | `"bdms_rucio_policy"` | Defines the policy permission model for Rucio for determining how authorization and access controls are applied, its value should be taken from the installed Rucio policy package |
+| rucio-daemons.config.policy.permission | string | `"ctao"` |  |
+| rucio-daemons.config.policy.schema | string | `"ctao_bdms"` |  |
 | rucio-daemons.conveyorFinisher.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
 | rucio-daemons.conveyorFinisher.resources.limits.cpu | string | `"3000m"` |  |
 | rucio-daemons.conveyorFinisher.resources.limits.memory | string | `"4Gi"` |  |
@@ -95,21 +101,25 @@ A Helm chart for the bdms project
 | rucio-daemons.conveyorTransferSubmitterCount | int | `1` | Number of container instances to deploy for each Rucio daemon, this daemon submits new transfer requests to the FTS |
 | rucio-daemons.image.pullPolicy | string | `"Always"` | It defines when kubernetes should pull the container image, the options available are: Always, IfNotPresent, and Never |
 | rucio-daemons.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-daemons"` | Specifies the container image repository for Rucio daemons |
-| rucio-daemons.image.tag | string | `"35.4.1-v0.1.0"` | Specific image tag to use for deployment |
+| rucio-daemons.image.tag | string | `"35.7.0-v0.2.0"` | Specific image tag to use for deployment |
+| rucio-daemons.judgeEvaluator.resources.limits.cpu | string | `"3000m"` |  |
+| rucio-daemons.judgeEvaluator.resources.limits.memory | string | `"4Gi"` |  |
+| rucio-daemons.judgeEvaluator.resources.requests.cpu | string | `"700m"` |  |
+| rucio-daemons.judgeEvaluator.resources.requests.memory | string | `"1Gi"` |  |
 | rucio-daemons.judgeEvaluatorCount | int | `1` | Evaluates Rucio replication rules and triggers transfers |
 | rucio-daemons.useDeprecatedImplicitSecrets | bool | `true` | Enables the use of deprecated implicit secrets for authentication |
 | rucio-server.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. |
 | rucio-server.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The database connection URI for Rucio |
-| rucio-server.config.persistence.accessMode | string | `"ReadWriteOnce"` | Defines how storage can be accessed, the options available are: ReadWriteOnce, ReadOnlyMany, ReadWriteMany |
-| rucio-server.config.persistence.enabled | bool | `true` | Enables persistent storage for Rucio server, setting it to false means using temporary storage ('ephemeral' in the context of kubernetes) that gets wiped out and lost when the container is stopped or restarted |
-| rucio-server.config.persistence.size | string | `"1Gi"` | Defines the size of persistent volume claim (PVC) or the amount of the storage capacity provisioned to the Rucio server |
-| rucio-server.config.persistence.storageClass | string | `""` | Specifies the kubernetes storage class for persistent volume, default storage class is used when its value is left empty |
+| rucio-server.config.policy.lfn2pfn_algorithm_default | string | `"ctao_bdms"` |  |
+| rucio-server.config.policy.package | string | `"bdms_rucio_policy"` | Defines the policy permission model for Rucio for determining how authorization and access controls are applied, its value should be taken from the installed Rucio policy package |
+| rucio-server.config.policy.permission | string | `"ctao"` |  |
+| rucio-server.config.policy.schema | string | `"ctao_bdms"` |  |
 | rucio-server.ftsRenewal.enabled | bool | `false` | Enables automatic renewal of FTS credentials using X.509 certificates and proxy |
 | rucio-server.httpd_config.encoded_slashes | string | `"True"` | Allows for custom LFNs with slashes in request URLs so that Rucio server (Apache) can decode and handle such requests properly |
 | rucio-server.httpd_config.grid_site_enabled | string | `"True"` | Enables Rucio server to support and interact with grid middleware (storages) for X509 authentication with proxies |
 | rucio-server.image.pullPolicy | string | `"Always"` | It defines when kubernetes should pull the container image, the options available are: Always, IfNotPresent, and Never |
 | rucio-server.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image repository for Rucio server with the CTAO Rucio policy package installed |
-| rucio-server.image.tag | string | `"35.4.1-v0.1.0"` | The specific image tag to deploy |
+| rucio-server.image.tag | string | `"35.7.0-v0.2.0"` | The specific image tag to deploy |
 | rucio-server.ingress.enabled | bool | `true` | Enables an ingress resource (controller) for exposing the Rucio server externally to allow clients connect to the Rucio server. It needs one of the ingress controllers (NGINX, Traefik) to be installed |
 | rucio-server.ingress.hosts | list | `["rucio-server-manual-tc.local"]` | Defines the hostname to be used to access the Rucio server. It should match DNS configuration and TLS certificates |
 | rucio-server.replicaCount | int | `1` | Number of replicas of the Rucio server to deploy. We can increase it to meet higher availability goals |
@@ -121,7 +131,7 @@ A Helm chart for the bdms project
 | rucio-server.useSSL | bool | `true` | Enables the Rucio server to use SSL/TLS for secure communication, requiring valid certificates to be configured |
 | rucio.password | string | `"secret"` |  |
 | rucio.username | string | `"dpps"` | Specifies the username for Rucio operations as part of Rucio configuration |
-| rucio.version | string | `"35.4.1"` | The version of Rucio being deployed |
+| rucio.version | string | `"35.7.0"` | The version of Rucio being deployed |
 | rucio_db.connection | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The database connection URI for Rucio. It is of the format: `postgresql://<user>:<password>@<host>:<port>/<database>`, this field in use only if 'existingSecret.enabled' is set to 'false', otherwise ignored |
 | rucio_db.deploy | bool | `true` | If true, deploys a postgresql instance for the Rucio database, otherwise use an external database |
 | rucio_db.existingSecret.enabled | bool | `false` | If true, the database connection URI is obtained from a kubernetes secret in |
@@ -135,6 +145,8 @@ A Helm chart for the bdms project
 | server.certificate.letsencrypt.email | string | `""` |  |
 | server.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
 | server.rucioHost | string | `"rucio-server-manual-tc.local"` | The hostname of the Rucio server. It is used by clients and services to communicate with Rucio |
-| storages | list | `["rucio-storage-1","rucio-storage-2","rucio-storage-3"]` | a list of storage element (RSE) hostnames names, for each RSE, one deployment and service are configured, two configmaps xrdconfig and xrd-entrypoint for those three storages |
 | suffix_namespace | string | `"default"` | Specifies the Namespace suffix used for managing deployments in kubernetes |
+| test_storages | object | `{"xrootd":{"image":{"repository":"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd","tag":"37.1.0"},"instances":["rucio-storage-1","rucio-storage-2","rucio-storage-3"]}}` | - A list of test storages, deployed in the test setup |
+| test_storages.xrootd.image.repository | string | `"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd"` | The container image repository for the XRootD storage deployment |
+| test_storages.xrootd.image.tag | string | `"37.1.0"` | Defines the specific version of the XRootD image to use |
 

ctao_bdms_clients-0.2.0/chart/scripts/bootstrap_rucio/wait_for_rucio.sh

@@ -0,0 +1,20 @@
+while true; do
+    openssl s_client -connect ${HELM_RELEASE_NAME:?}-rucio-server:443 && break
+    sleep 3
+done
+
+echo "Rucio server is responding on port 443"
+
+if [ -z "$WAIT_RUCIO_PING" ]; then
+    echo "Skipping rucio ping check"
+else
+    while true; do
+        rucio ping && break
+        sleep 3
+    done
+
+    echo "Rucio server is responding to ping"
+fi
+
+ls -l /opt/rucio/etc/
+cat /opt/rucio/etc/rucio.cfg

{ctao_bdms_clients-0.1.0rc3 → ctao_bdms_clients-0.2.0}/chart/scripts/certificates/install_ca.sh

@@ -9,10 +9,3 @@ ls -lotr /etc/pki/tls/certs/ca-bundle.crt
 
 # TODO: avoid attempt copying the certificate if it already exists
 cp -fv /etc/grid-security/ca.pem /etc/grid-security/certificates/$hash.0 || echo "Certificate already exists"
-
-# openssl x509 -in /etc/grid-security/ca.pem -noout -text
-
-while true; do
-    openssl s_client -connect ${HELM_RELEASE_NAME:?}-rucio-server:443 && break
-    sleep 3
-done

ctao_bdms_clients-0.2.0/chart/templates/_helpers_cert.tpl

@@ -0,0 +1,40 @@
+{{- define "volume_mounts_rucio_config" }}
+- name: rucio-config
+  mountPath: /opt/rucio/etc/rucio.cfg
+  subPath: rucio.cfg
+- name: rucio-config
+  mountPath: /opt/rucio/etc/alembic.ini
+  subPath: alembic.ini
+{{- end }}
+{{- define "volume_mounts_cert" }}
+- name: cafile
+  subPath: ca.pem
+  mountPath: /etc/grid-security/ca.pem
+- name: dppsuser-certkey-400
+  mountPath: /opt/rucio/etc/userkey.pem
+  subPath: dppsuser.key.pem
+- name: dppsuser-certkey-600
+  mountPath: /opt/rucio/etc/usercert.pem
+  subPath: dppsuser.pem
+{{- end }}
+{{- define "volumes_cert" }}
+- name: rucio-config
+  configMap:
+    name: {{ template "bdms.fullname" . }}-rucio-config
+- name: cafile
+  secret:
+    defaultMode: 420
+    secretName: {{ template "certprefix" . }}-server-cafile
+- name: dppsuser-certkey-600
+  secret:
+    defaultMode: 0600
+    secretName: {{ template "certprefix" . }}-dppsuser-certkey
+- name: dppsuser-certkey-400
+  secret:
+    defaultMode: 0400
+    secretName: {{ template "certprefix" . }}-dppsuser-certkey
+{{- end }}
+{{- define "env_helm_release" }}
+- name: HELM_RELEASE_NAME
+  value: {{ .Release.Name }}
+{{- end }}