ctao-bdms-clients 0.2.0rc1__tar.gz → 0.3.0rc1__tar.gz

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/.gitignore

@@ -7,9 +7,13 @@ Chart.lock
 helm
 helm.tar.gz
 kind
+yq
 kubectl
 linux-amd64
+pull-images-statistics.json
 # generated file
+helm-debug.txt
+chart-image-list.txt
 chart/**/*.tgz
 report.xml
 # setuptools_scm generated version file
@@ -145,3 +149,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Test folder for Fits files
+test_data/

ctao_bdms_clients-0.3.0rc1/.gitlab-ci.yml

@@ -0,0 +1,60 @@
+include:
+  - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
+    ref: 1e9b640800ac73c215ce59df01330085d05b335e
+    file: 'ci-functions.yml'
+  - "aiv-config.yml"
+
+
+variables:
+  CHART_LOCATION: chart
+  CHART_NAME: bdms
+  CHART_EXTRA_VALUES: "--set dev.client_image_tag=${DOCKER_TAG} --set acada_ingest.image.tag=${DOCKER_TAG}"
+  DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
+  RUCIO_VERSION: "35.4.1"
+  RUCIO_TAG: "release-${RUCIO_VERSION}"
+
+stages:
+  - prepare
+  - lint
+  - build
+  - sign
+  - tests
+  - sonarqube
+  - publish
+  - report
+  - changelog
+
+k8s-integration-tests:
+  # override from toolkit to add .env file with CI secrets
+  script:
+    - echo -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY\nMINIO_SECRET_KEY=$MINIO_SECRET_KEY\n" > .env
+    - ${MAKE} test-chart 2>&1 | tee test-output.log
+
+k8s-integration-tests-with-upgrade:
+  extends: k8s-integration-tests
+  script:
+    - echo -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY\nMINIO_SECRET_KEY=$MINIO_SECRET_KEY\n" > .env
+    - ${MAKE} test-chart 2>&1 | tee test-output.log
+    - find -name Chart.lock -delete
+    # seconds test, upgrades current cluster
+    - ${MAKE} test-chart 2>&1 | tee test-output.log
+
+
+build:
+  variables:
+    CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-client:${DOCKER_TAG}'
+    KANIKO_EXTRA_ARGS: --build-arg RUCIO_TAG=${RUCIO_TAG}
+
+build-ingestion-daemon:
+  extends: build
+  variables:
+    CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-ingestion-daemon:${DOCKER_TAG}'
+    KANIKO_EXTRA_ARGS: --build-arg RUCIO_TAG=${RUCIO_TAG}
+
+hadolint:
+  rules:
+    - when: never
+
+sign:
+  rules:
+    - when: never

ctao_bdms_clients-0.3.0rc1/CHANGES.rst

@@ -0,0 +1,80 @@
+BDMS v0.2.1 (2025-06-03)
+------------------------
+
+Bugfix release to allow (pre-)production deployment without test storages.
+
+Bug Fixes
+~~~~~~~~~
+
+- Allow to further configure first test RSE: set storageclass and make optional. [`!112 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/112>`__]
+
+
+BDMS 0.2.0 (2025-05-07)
+-----------------------
+
+
+API Changes
+~~~~~~~~~~~
+
+- Add basic ingestion of ACADA files (UC-110-1.1.1) [`!76 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/76>`__]
+
+- Replicate Data Products (UC-110-1.6) with test case for copies=2 [`!83 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/83>`__]
+
+- Extraction of metadata from ACADA-LST DL0 FITS files and adding metadata to ingested file (UC-110-1.1.1) [`!85 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/85>`__]
+
+- Add test case for copies=1 for the replication (UC-110-1.6) [`!87 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/87>`__]
+
+
+Bug Fixes
+~~~~~~~~~
+
+- Add missing permissions to fix onsite storage permissions [`!103 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/103>`__]
+
+
+New Features
+~~~~~~~~~~~~
+
+- Adapt one RSE as onsite and exposing its diskspace to client test pod [`!69 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/69>`__]
+
+- Add dcache and its dependencies in the BDMS helm chart (UC-170-1.6) [`!80 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/80>`__]
+
+- Add FITS checksum verification [`!94 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/94>`__]
+
+- Make storage configurable by providing additional config for RSE and also make xrootd image configurable [`!96 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/96>`__]
+
+
+Maintenance
+~~~~~~~~~~~
+
+- Fix rucio server not using correct lfn2pfn algorithm [`!72 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/72>`__]
+
+- Use test_vo fixture for ingestion tests [`!81 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/81>`__]
+
+- Update dpps release for test report and signature matrix [`!82 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/82>`__]
+
+- Update BDMS docs for Rel 0.1 [`!89 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/89>`__]
+
+- Update fts image and toolkit [`!90 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/90>`__]
+
+- Added download_test_file in utils.py to download a FITS file from MinIO server [`!93 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/93>`__]
+
+- Update rucio to 35.7.0 [`!97 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/97>`__]
+
+- Update FTS subchart version [`!99 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/99>`__]
+
+- Resolve Test container build fails due to missing -y in autoremove [`!101 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/101>`__]
+
+
+Refactoring and Optimization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Update FTS chart to 0.3 to reduce integration test time [`!100 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/100>`__]
+
+BDMS v0.1.0 (2025-02-21)
+---------------------------
+
+First release of the Bulk Data Management System (BDMS).
+
+* Deployment of Rucio 35.4 using helm.
+* Client package pinning the correct rucio and rucio policy package versions.
+* Integration tests for DPPS release 0.0 use cases.

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/Dockerfile

@@ -20,9 +20,9 @@ ARG RUCIO_TAG
 # server and daemons use root, clients use "user", switch to root, install, then back
 USER root
 COPY --from=builder /tmp/dist/ /tmp/dist/
-RUN  dnf install -y --setopt=install_weak_deps=False git \
+RUN  dnf install -y --setopt=install_weak_deps=False git procps \
   && python3 -m pip install --no-cache-dir /tmp/dist/ctao_bdms* \
-  && dnf autoremove \
+  && dnf autoremove -y \
   && dnf clean all
 
 USER user

ctao_bdms_clients-0.3.0rc1/Makefile

@@ -0,0 +1,20 @@
+###
+# TODO: duplicate in config?
+export CHART_NAME=bdms
+export CHART_LOCATION=chart
+include dpps-aiv-toolkit/Makefile
+
+
+# TODO: move this to kit
+export TEST_ARTIFACTS_PATH ?= $(PWD)
+export TEST_REPORT_CONFIG ?= $(PWD)/aiv-config.yml
+export TEX_CONTENT_PATH ?= $(PWD)/report
+
+build-dev-server-images:
+	# TODO: should it be a separate image? it should have a different name
+	docker build . -t harbor.cta-observatory.org/dpps/bdms-ingestion-daemon:dev
+	./kind -n $(KUBECLUSTER) load docker-image \
+		harbor.cta-observatory.org/dpps/bdms-ingestion-daemon:dev
+
+
+dev: build-dev-server-images

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.2.0rc1
+Version: 0.3.0rc1
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
-License: BSD-3-Clause
+License-Expression: BSD-3-Clause
 Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms
 Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms
 Requires-Python: >=3.9
@@ -12,12 +12,18 @@ License-File: LICENSE
 Requires-Dist: astropy<8.0.0a0,>=6.0.1
 Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
 Requires-Dist: rucio-clients~=35.7.0
+Requires-Dist: protozfits>=2.7.2
+Requires-Dist: watchdog>=6.0.0
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: prometheus-client>=0.22.1
+Requires-Dist: ruamel.yaml
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: pytest-requirements; extra == "test"
 Requires-Dist: python-dotenv; extra == "test"
 Requires-Dist: minio; extra == "test"
+Requires-Dist: pytest-xdist; extra == "test"
 Provides-Extra: doc
 Requires-Dist: sphinx; extra == "doc"
 Requires-Dist: numpydoc; extra == "doc"

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/aiv-config.yml

@@ -3,7 +3,7 @@
 variables:
   # Comma-separated list of "UC Groups" to select UCs and Requirements, e.g. "BDMS,AIV"
   DPPS_UC_GROUPS: BDMS
-  DPPS_RELEASE: v0.1
+  DPPS_RELEASE: v0.2.0
 
   # these paths are relative to TEST_ARTIFACTS_PATH
   REPORT_XML: report.xml
@@ -15,3 +15,5 @@ variables:
   APPLICATION_AUTHOR_ORGANIZATION: CTAO
 
   EXTRA_CONFIG_FILES: aiv-config-dependencies.yml
+
+  DEFAULT_HELM_TEST: name=bdms-pytest

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/chart/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: bdms
 version: 0.1.0
-appVersion: 0.1.0
+appVersion: dev
 description: A Helm chart for the bdms project
 type: application
 keywords:
@@ -12,13 +12,13 @@ keywords:
 
 maintainers:
 - name: The BDMS Authors
-  email: ""
+  email: ''
 
 dependencies:
 - name: postgresql
   condition: postgresql.enabled
   version: 15.5.10
-  repository: oci://registry-1.docker.io/bitnamicharts
+  repository: oci://harbor.cta-observatory.org/proxy_cache/bitnamicharts
 
 - name: rucio-server
   version: 35.0.0
@@ -30,10 +30,10 @@ dependencies:
 
 - name: cert-generator-grid
   condition: cert-generator-grid.enabled
-  version: v0.2.2
+  version: v2.1.0
   repository: oci://harbor.cta-observatory.org/dpps
 
 - name: fts
   condition: fts.enabled
-  version: v0.3.0
+  version: v0.3.1
   repository: oci://harbor.cta-observatory.org/dpps

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/chart/Makefile

@@ -11,5 +11,5 @@ publish:
 reset:
 	helm delete bdms || true
 	kubectl delete secrets bdms-rucio-server bdms-rucio-server-tls bdms-server-cafile bdms-server-hostcert bdms-server-hostkey dppsuser-certkey || true
-	kubectl delete job  generate-certificates configure-test-rucio test-rucio || true
+	kubectl delete job  generate-certificates configure-rucio test-rucio || true
 	kubectl delete pvc data-bdms-postgresql-0 || true

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/chart/README.md

@@ -1,6 +1,6 @@
 # bdms
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square)
 
 A Helm chart for the bdms project
 
@@ -14,16 +14,41 @@ A Helm chart for the bdms project
 
 | Repository | Name | Version |
 |------------|------|---------|
-| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v0.2.2 |
-| oci://harbor.cta-observatory.org/dpps | fts | v0.3.0 |
+| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v2.1.0 |
+| oci://harbor.cta-observatory.org/dpps | fts | v0.3.1 |
 | oci://harbor.cta-observatory.org/dpps | rucio-daemons | 35.0.0 |
 | oci://harbor.cta-observatory.org/dpps | rucio-server | 35.0.0 |
-| oci://registry-1.docker.io/bitnamicharts | postgresql | 15.5.10 |
+| oci://harbor.cta-observatory.org/proxy_cache/bitnamicharts | postgresql | 15.5.10 |
 
 ## Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| acada_ingest.daemon.config.check_interval | float | `1` |  |
+| acada_ingest.daemon.config.data_path | string | `"/storage-1/"` |  |
+| acada_ingest.daemon.config.disable_metrics | bool | `false` |  |
+| acada_ingest.daemon.config.lock_file | string | `"/storage-1/bdms_ingest.lock"` |  |
+| acada_ingest.daemon.config.log_file | string | `nil` | The path to the log file, if not specified, logs to stdout |
+| acada_ingest.daemon.config.log_level | string | `"DEBUG"` | The logging level for the ingestion daemon |
+| acada_ingest.daemon.config.metrics_port | int | `8000` | The port for the Prometheus metrics server |
+| acada_ingest.daemon.config.offsite_copies | int | `2` |  |
+| acada_ingest.daemon.config.polling_interval | float | `1` |  |
+| acada_ingest.daemon.config.rse | string | `"STORAGE-1"` |  |
+| acada_ingest.daemon.config.scope | string | `"test_scope_persistent"` |  |
+| acada_ingest.daemon.config.vo | string | `"ctao.dpps.test"` |  |
+| acada_ingest.daemon.config.workers | int | `4` |  |
+| acada_ingest.daemon.replicas | int | `0` | The number of replicas of the ingestion daemon to run, set to 0 to disable the daemon |
+| acada_ingest.daemon.service.enabled | bool | `true` |  |
+| acada_ingest.daemon.service.type | string | `"ClusterIP"` |  |
+| acada_ingest.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-ingestion-daemon"` | The container image repository for the ingestion daemon |
+| acada_ingest.securityContext.fsGroup | int | `0` |  |
+| acada_ingest.securityContext.runAsGroup | int | `0` |  |
+| acada_ingest.securityContext.runAsUser | int | `0` | The security context for the ingestion daemon, it defines the user and group IDs under which the container runs |
+| acada_ingest.securityContext.supplementalGroups | list | `[]` |  |
+| acada_ingest.volumeMounts[0].mountPath | string | `"/storage-1/"` |  |
+| acada_ingest.volumeMounts[0].name | string | `"storage-1-data"` |  |
+| acada_ingest.volumes[0].name | string | `"storage-1-data"` |  |
+| acada_ingest.volumes[0].persistentVolumeClaim.claimName | string | `"storage-1-pvc"` |  |
 | auth.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. It is used by clients and services to authenticate with Rucio |
 | auth.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
 | auth.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
@@ -33,17 +58,20 @@ A Helm chart for the bdms project
 | auth.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
 | bootstrap.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image for bootstrapping Rucio (initialization, configuration) with the CTAO Rucio policy package installed |
 | bootstrap.image.tag | string | `"35.7.0-v0.2.0"` | The specific image tag to use for the bootstrap container |
+| bootstrap.pg_image.repository | string | `"harbor.cta-observatory.org/proxy_cache/postgres"` | Postgres client image used to wait for db readines during bootstrap |
+| bootstrap.pg_image.tag | string | `"16.3-bookworm"` | Postgres client image tag used to wait for db readines during bootstrap |
 | cert-generator-grid.enabled | bool | `true` |  |
 | cert-generator-grid.generatePreHooks | bool | `true` |  |
-| configure | object | `{"extra_script":"# add a scope\nrucio-admin scope add --account root --scope root\nrucio add-container /ctao.dpps.test\n","identities":[{"account":"root","email":"dpps-test@cta-observatory.org","id":"CN=DPPS User","type":"X509"}],"rse_distances":[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]],"rses":{"STORAGE-1":{"attributes":{"ANY":true,"ONSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-1","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"rse_type":"DISK"},"STORAGE-2":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-2","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true},"STORAGE-3":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-3","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true}}}` | a list of Rucio Storage Elements (RSE) TODO: make more clear mechanism to handle different upgrade scenarios If there is a conflict between existing configuration, the configuration will fail. In this case, likely the configuration should be deleted and re-created. |
-| configure.extra_script | string | `"# add a scope\nrucio-admin scope add --account root --scope root\nrucio add-container /ctao.dpps.test\n"` | This script is executed after the Rucio server is deployed and configured. It can be used to perform additional configuration or setup tasks if they currently cannot be done with the chart values. |
+| configure | object | `{"as_hook":false,"extra_script":"# add a scope\nrucio-admin scope add --account root --scope root || echo \"Scope 'root' already exists\"\nrucio add-container /ctao.dpps.test || echo \"Container /ctao.dpps.test already exists\"\n","identities":[{"account":"root","email":"dpps-test@cta-observatory.org","id":"CN=DPPS User","type":"X509"}],"rse_distances":[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]],"rses":{"STORAGE-1":{"attributes":{"ANY":true,"ONSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-1","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"rse_type":"DISK"},"STORAGE-2":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-2","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true},"STORAGE-3":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-3","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true}}}` | a list of Rucio Storage Elements (RSE) TODO: make more clear mechanism to handle different upgrade scenarios If there is a conflict between existing configuration, the configuration will fail. In this case, likely the configuration should be deleted and re-created. |
+| configure.extra_script | string | `"# add a scope\nrucio-admin scope add --account root --scope root || echo \"Scope 'root' already exists\"\nrucio add-container /ctao.dpps.test || echo \"Container /ctao.dpps.test already exists\"\n"` | This script is executed after the Rucio server is deployed and configured. It can be used to perform additional configuration or setup tasks if they currently cannot be done with the chart values. |
 | configure.rse_distances | list | `[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]]` | A list of RSE distance specifications, each a list of 4 values: source RSE, destination RSE, distance (integer), and ranking (integer) |
-| configure_test_setup | bool | `true` | This will configure the rucio server with the storages |
+| configure_rucio | bool | `true` | This will configure the rucio server with the storages |
 | database | object | `{"default":"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"}` | Databases Credentials used by Rucio to access the database. If postgresql subchart is deployed, these credentials should match those in postgresql.global.postgresql.auth. If postgresql subchart is not deployed, an external database must be provided |
 | database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The Rucio database connection URI |
 | dev.client_image_tag | string | `nil` |  |
-| dev.mount_repo | bool | `true` |  |
-| dev.run_tests | bool | `true` |  |
+| dev.mount_repo | bool | `true` | mount the repository into the container, useful for development and debugging |
+| dev.n_test_jobs | int | `1` | number of jobs to use for pytest |
+| dev.run_tests | bool | `true` | run tests during helm test (otherwise, the tests can be run manually after exec into the pod) |
 | dev.sleep | bool | `false` | sleep after test to allow interactive development |
 | fts.enabled | bool | `true` | Specifies the configuration for FTS test step (FTS server, FTS database, and ActiveMQ broker containers). Enables or disables the deployment of a FTS instance for testing. This is set to 'False' if an external FTS is used |
 | fts.ftsdb_password | string | `"SDP2RQkbJE2f+ohUb2nUu6Ae10BpQH0VD70CsIQcDtM"` | Defines the password for the FTS database user |
@@ -56,10 +84,8 @@ A Helm chart for the bdms project
 | postgresql.global.postgresql.auth.database | string | `"rucio"` | The name of the database to be created and used by Rucio |
 | postgresql.global.postgresql.auth.password | string | `"XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM"` | The password for the database user |
 | postgresql.global.postgresql.auth.username | string | `"rucio"` | The database username for authentication |
+| postgresql.image.registry | string | `"harbor.cta-observatory.org/proxy_cache"` |  |
 | prepuller_enabled | bool | `true` | Starts containers with the same image as the one used in the deployment before all volumes are available. Saves time in the first deployment |
-| rethinkdb.enabled | bool | `false` |  |
-| rethinkdb.storageClassName | string | `nil` |  |
-| rethinkdb.storageSize | string | `"1Gi"` |  |
 | rucio-daemons.config.common.extract_scope | string | `"ctao_bdms"` |  |
 | rucio-daemons.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | Specifies the connection URI for the Rucio database, these settings will be written to 'rucio.cfg' |
 | rucio-daemons.config.messaging-fts3.brokers | string | `"fts-activemq"` | Specifies the message broker used for FTS messaging |
@@ -136,7 +162,8 @@ A Helm chart for the bdms project
 | rucio_db.existingSecret.enabled | bool | `false` | If true, the database connection URI is obtained from a kubernetes secret in |
 | rucio_db.existingSecret.key | string | `"connection"` | The key inside the kubernetes secret that holds the database connection URI |
 | rucio_db.existingSecret.secretName | string | `"rucio-db"` | The name of the kubernetes secret storing the database connection URI. Its in use only if 'existingSecret.enabled: true' |
-| safe_to_bootstrap_rucio | bool | `true` | This is a destructive operation, it will delete all data in the database |
+| safe_to_bootstrap_rucio | bool | `false` | This is a destructive operation, it will delete all data in the database |
+| safe_to_bootstrap_rucio_on_install | bool | `true` | This is will delete all data in the database only on the first install |
 | server.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
 | server.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
 | server.certificate.existingSecret.key | string | `"tls.key"` | The key inside the kubernetes secret that stores the private key |
@@ -145,7 +172,9 @@ A Helm chart for the bdms project
 | server.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
 | server.rucioHost | string | `"rucio-server-manual-tc.local"` | The hostname of the Rucio server. It is used by clients and services to communicate with Rucio |
 | suffix_namespace | string | `"default"` | Specifies the Namespace suffix used for managing deployments in kubernetes |
-| test_storages | object | `{"xrootd":{"image":{"repository":"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd","tag":"37.1.0"},"instances":["rucio-storage-1","rucio-storage-2","rucio-storage-3"]}}` | - A list of test storages, deployed in the test setup |
+| test_storages | object | `{"enabled":true,"xrootd":{"image":{"repository":"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd","tag":"37.1.0"},"instances":["rucio-storage-1","rucio-storage-2","rucio-storage-3"],"rucio_storage_1_storage_class":"standard"}}` | - A list of test storages, deployed in the test setup |
+| test_storages.enabled | bool | `true` | If true, deploys test storages for testing purposes. This is set to 'False' if an external storage is used as in the production setup |
 | test_storages.xrootd.image.repository | string | `"harbor.cta-observatory.org/proxy_cache/rucio/test-xrootd"` | The container image repository for the XRootD storage deployment |
 | test_storages.xrootd.image.tag | string | `"37.1.0"` | Defines the specific version of the XRootD image to use |
+| test_storages.xrootd.rucio_storage_1_storage_class | string | `"standard"` | The storage class name for the PVC used by rucio-storage-1 |
 

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/chart/scripts/bootstrap_rucio/wait_for_rucio.sh

@@ -9,7 +9,7 @@ if [ -z "$WAIT_RUCIO_PING" ]; then
     echo "Skipping rucio ping check"
 else
     while true; do
-        rucio ping && break
+        rucio ping && rucio whoami && break
         sleep 3
     done
 

{ctao_bdms_clients-0.2.0rc1 → ctao_bdms_clients-0.3.0rc1}/chart/scripts/certificates/install_ca.sh

@@ -1,7 +1,6 @@
 hash=$(openssl x509 -noout -hash -in /etc/grid-security/ca.pem)
 
 mkdir -pv /etc/grid-security/certificates
-update-ca-trust force-enable
 
 cp -fv /etc/grid-security/ca.pem  /etc/pki/ca-trust/source/anchors/rucio-ca.pem
 update-ca-trust extract