ctao-bdms-clients 0.2.1__tar.gz → 0.3.0rc1__tar.gz

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/.gitignore

@@ -7,9 +7,13 @@ Chart.lock
 helm
 helm.tar.gz
 kind
+yq
 kubectl
 linux-amd64
+pull-images-statistics.json
 # generated file
+helm-debug.txt
+chart-image-list.txt
 chart/**/*.tgz
 report.xml
 # setuptools_scm generated version file

ctao_bdms_clients-0.3.0rc1/.gitlab-ci.yml

@@ -0,0 +1,60 @@
+include:
+  - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
+    ref: 1e9b640800ac73c215ce59df01330085d05b335e
+    file: 'ci-functions.yml'
+  - "aiv-config.yml"
+
+
+variables:
+  CHART_LOCATION: chart
+  CHART_NAME: bdms
+  CHART_EXTRA_VALUES: "--set dev.client_image_tag=${DOCKER_TAG} --set acada_ingest.image.tag=${DOCKER_TAG}"
+  DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
+  RUCIO_VERSION: "35.4.1"
+  RUCIO_TAG: "release-${RUCIO_VERSION}"
+
+stages:
+  - prepare
+  - lint
+  - build
+  - sign
+  - tests
+  - sonarqube
+  - publish
+  - report
+  - changelog
+
+k8s-integration-tests:
+  # override from toolkit to add .env file with CI secrets
+  script:
+    - echo -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY\nMINIO_SECRET_KEY=$MINIO_SECRET_KEY\n" > .env
+    - ${MAKE} test-chart 2>&1 | tee test-output.log
+
+k8s-integration-tests-with-upgrade:
+  extends: k8s-integration-tests
+  script:
+    - echo -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY\nMINIO_SECRET_KEY=$MINIO_SECRET_KEY\n" > .env
+    - ${MAKE} test-chart 2>&1 | tee test-output.log
+    - find -name Chart.lock -delete
+    # seconds test, upgrades current cluster
+    - ${MAKE} test-chart 2>&1 | tee test-output.log
+
+
+build:
+  variables:
+    CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-client:${DOCKER_TAG}'
+    KANIKO_EXTRA_ARGS: --build-arg RUCIO_TAG=${RUCIO_TAG}
+
+build-ingestion-daemon:
+  extends: build
+  variables:
+    CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-ingestion-daemon:${DOCKER_TAG}'
+    KANIKO_EXTRA_ARGS: --build-arg RUCIO_TAG=${RUCIO_TAG}
+
+hadolint:
+  rules:
+    - when: never
+
+sign:
+  rules:
+    - when: never

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/Dockerfile

@@ -20,7 +20,7 @@ ARG RUCIO_TAG
 # server and daemons use root, clients use "user", switch to root, install, then back
 USER root
 COPY --from=builder /tmp/dist/ /tmp/dist/
-RUN  dnf install -y --setopt=install_weak_deps=False git \
+RUN  dnf install -y --setopt=install_weak_deps=False git procps \
   && python3 -m pip install --no-cache-dir /tmp/dist/ctao_bdms* \
   && dnf autoremove -y \
   && dnf clean all

ctao_bdms_clients-0.3.0rc1/Makefile

@@ -0,0 +1,20 @@
+###
+# TODO: duplicate in config?
+export CHART_NAME=bdms
+export CHART_LOCATION=chart
+include dpps-aiv-toolkit/Makefile
+
+
+# TODO: move this to kit
+export TEST_ARTIFACTS_PATH ?= $(PWD)
+export TEST_REPORT_CONFIG ?= $(PWD)/aiv-config.yml
+export TEX_CONTENT_PATH ?= $(PWD)/report
+
+build-dev-server-images:
+	# TODO: should it be a separate image? it should have a different name
+	docker build . -t harbor.cta-observatory.org/dpps/bdms-ingestion-daemon:dev
+	./kind -n $(KUBECLUSTER) load docker-image \
+		harbor.cta-observatory.org/dpps/bdms-ingestion-daemon:dev
+
+
+dev: build-dev-server-images

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ctao-bdms-clients
-Version: 0.2.1
+Version: 0.3.0rc1
 Summary: Client module for the CTAO DPPS Bulk Data Management System
 Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
 License-Expression: BSD-3-Clause
@@ -13,6 +13,10 @@ Requires-Dist: astropy<8.0.0a0,>=6.0.1
 Requires-Dist: ctao-bdms-rucio-policy~=0.1.0
 Requires-Dist: rucio-clients~=35.7.0
 Requires-Dist: protozfits>=2.7.2
+Requires-Dist: watchdog>=6.0.0
+Requires-Dist: filelock>=3.18.0
+Requires-Dist: prometheus-client>=0.22.1
+Requires-Dist: ruamel.yaml
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/aiv-config.yml

@@ -3,7 +3,7 @@
 variables:
   # Comma-separated list of "UC Groups" to select UCs and Requirements, e.g. "BDMS,AIV"
   DPPS_UC_GROUPS: BDMS
-  DPPS_RELEASE: v0.1
+  DPPS_RELEASE: v0.2.0
 
   # these paths are relative to TEST_ARTIFACTS_PATH
   REPORT_XML: report.xml

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/chart/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: bdms
 version: 0.1.0
-appVersion: 0.1.0
+appVersion: dev
 description: A Helm chart for the bdms project
 type: application
 keywords:
@@ -18,7 +18,7 @@ dependencies:
 - name: postgresql
   condition: postgresql.enabled
   version: 15.5.10
-  repository: oci://registry-1.docker.io/bitnamicharts
+  repository: oci://harbor.cta-observatory.org/proxy_cache/bitnamicharts
 
 - name: rucio-server
   version: 35.0.0
@@ -30,10 +30,10 @@ dependencies:
 
 - name: cert-generator-grid
   condition: cert-generator-grid.enabled
-  version: v2.0.0
+  version: v2.1.0
   repository: oci://harbor.cta-observatory.org/dpps
 
 - name: fts
   condition: fts.enabled
-  version: v0.3.0
+  version: v0.3.1
   repository: oci://harbor.cta-observatory.org/dpps

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/chart/Makefile

@@ -11,5 +11,5 @@ publish:
 reset:
 	helm delete bdms || true
 	kubectl delete secrets bdms-rucio-server bdms-rucio-server-tls bdms-server-cafile bdms-server-hostcert bdms-server-hostkey dppsuser-certkey || true
-	kubectl delete job  generate-certificates configure-test-rucio test-rucio || true
+	kubectl delete job  generate-certificates configure-rucio test-rucio || true
 	kubectl delete pvc data-bdms-postgresql-0 || true

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/chart/README.md

@@ -1,6 +1,6 @@
 # bdms
 
-![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
+![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square)
 
 A Helm chart for the bdms project
 
@@ -14,16 +14,41 @@ A Helm chart for the bdms project
 
 | Repository | Name | Version |
 |------------|------|---------|
-| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v2.0.0 |
-| oci://harbor.cta-observatory.org/dpps | fts | v0.3.0 |
+| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v2.1.0 |
+| oci://harbor.cta-observatory.org/dpps | fts | v0.3.1 |
 | oci://harbor.cta-observatory.org/dpps | rucio-daemons | 35.0.0 |
 | oci://harbor.cta-observatory.org/dpps | rucio-server | 35.0.0 |
-| oci://registry-1.docker.io/bitnamicharts | postgresql | 15.5.10 |
+| oci://harbor.cta-observatory.org/proxy_cache/bitnamicharts | postgresql | 15.5.10 |
 
 ## Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| acada_ingest.daemon.config.check_interval | float | `1` |  |
+| acada_ingest.daemon.config.data_path | string | `"/storage-1/"` |  |
+| acada_ingest.daemon.config.disable_metrics | bool | `false` |  |
+| acada_ingest.daemon.config.lock_file | string | `"/storage-1/bdms_ingest.lock"` |  |
+| acada_ingest.daemon.config.log_file | string | `nil` | The path to the log file, if not specified, logs to stdout |
+| acada_ingest.daemon.config.log_level | string | `"DEBUG"` | The logging level for the ingestion daemon |
+| acada_ingest.daemon.config.metrics_port | int | `8000` | The port for the Prometheus metrics server |
+| acada_ingest.daemon.config.offsite_copies | int | `2` |  |
+| acada_ingest.daemon.config.polling_interval | float | `1` |  |
+| acada_ingest.daemon.config.rse | string | `"STORAGE-1"` |  |
+| acada_ingest.daemon.config.scope | string | `"test_scope_persistent"` |  |
+| acada_ingest.daemon.config.vo | string | `"ctao.dpps.test"` |  |
+| acada_ingest.daemon.config.workers | int | `4` |  |
+| acada_ingest.daemon.replicas | int | `0` | The number of replicas of the ingestion daemon to run, set to 0 to disable the daemon |
+| acada_ingest.daemon.service.enabled | bool | `true` |  |
+| acada_ingest.daemon.service.type | string | `"ClusterIP"` |  |
+| acada_ingest.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-ingestion-daemon"` | The container image repository for the ingestion daemon |
+| acada_ingest.securityContext.fsGroup | int | `0` |  |
+| acada_ingest.securityContext.runAsGroup | int | `0` |  |
+| acada_ingest.securityContext.runAsUser | int | `0` | The security context for the ingestion daemon, it defines the user and group IDs under which the container runs |
+| acada_ingest.securityContext.supplementalGroups | list | `[]` |  |
+| acada_ingest.volumeMounts[0].mountPath | string | `"/storage-1/"` |  |
+| acada_ingest.volumeMounts[0].name | string | `"storage-1-data"` |  |
+| acada_ingest.volumes[0].name | string | `"storage-1-data"` |  |
+| acada_ingest.volumes[0].persistentVolumeClaim.claimName | string | `"storage-1-pvc"` |  |
 | auth.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. It is used by clients and services to authenticate with Rucio |
 | auth.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
 | auth.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
@@ -33,18 +58,20 @@ A Helm chart for the bdms project
 | auth.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
 | bootstrap.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image for bootstrapping Rucio (initialization, configuration) with the CTAO Rucio policy package installed |
 | bootstrap.image.tag | string | `"35.7.0-v0.2.0"` | The specific image tag to use for the bootstrap container |
+| bootstrap.pg_image.repository | string | `"harbor.cta-observatory.org/proxy_cache/postgres"` | Postgres client image used to wait for db readines during bootstrap |
+| bootstrap.pg_image.tag | string | `"16.3-bookworm"` | Postgres client image tag used to wait for db readines during bootstrap |
 | cert-generator-grid.enabled | bool | `true` |  |
 | cert-generator-grid.generatePreHooks | bool | `true` |  |
-| configure | object | `{"extra_script":"# add a scope\nrucio-admin scope add --account root --scope root\nrucio add-container /ctao.dpps.test\n","identities":[{"account":"root","email":"dpps-test@cta-observatory.org","id":"CN=DPPS User","type":"X509"}],"rse_distances":[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]],"rses":{"STORAGE-1":{"attributes":{"ANY":true,"ONSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-1","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"rse_type":"DISK"},"STORAGE-2":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-2","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true},"STORAGE-3":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-3","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true}}}` | a list of Rucio Storage Elements (RSE) TODO: make more clear mechanism to handle different upgrade scenarios If there is a conflict between existing configuration, the configuration will fail. In this case, likely the configuration should be deleted and re-created. |
-| configure.extra_script | string | `"# add a scope\nrucio-admin scope add --account root --scope root\nrucio add-container /ctao.dpps.test\n"` | This script is executed after the Rucio server is deployed and configured. It can be used to perform additional configuration or setup tasks if they currently cannot be done with the chart values. |
+| configure | object | `{"as_hook":false,"extra_script":"# add a scope\nrucio-admin scope add --account root --scope root || echo \"Scope 'root' already exists\"\nrucio add-container /ctao.dpps.test || echo \"Container /ctao.dpps.test already exists\"\n","identities":[{"account":"root","email":"dpps-test@cta-observatory.org","id":"CN=DPPS User","type":"X509"}],"rse_distances":[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]],"rses":{"STORAGE-1":{"attributes":{"ANY":true,"ONSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-1","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"rse_type":"DISK"},"STORAGE-2":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-2","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true},"STORAGE-3":{"attributes":{"ANY":true,"OFFSITE":true,"fts":"https://bdms-fts:8446"},"limits_by_account":{"root":-1},"protocols":[{"domains":{"lan":{"delete":1,"read":1,"write":1},"wan":{"delete":1,"read":1,"third_party_copy_read":1,"third_party_copy_write":1,"write":1}},"extended_attributes":"None","hostname":"rucio-storage-3","impl":"rucio.rse.protocols.gfal.Default","port":1094,"prefix":"//rucio","scheme":"root"}],"recreate_if_exists":true}}}` | a list of Rucio Storage Elements (RSE) TODO: make more clear mechanism to handle different upgrade scenarios If there is a conflict between existing configuration, the configuration will fail. In this case, likely the configuration should be deleted and re-created. |
+| configure.extra_script | string | `"# add a scope\nrucio-admin scope add --account root --scope root || echo \"Scope 'root' already exists\"\nrucio add-container /ctao.dpps.test || echo \"Container /ctao.dpps.test already exists\"\n"` | This script is executed after the Rucio server is deployed and configured. It can be used to perform additional configuration or setup tasks if they currently cannot be done with the chart values. |
 | configure.rse_distances | list | `[["STORAGE-1","STORAGE-2",1,1],["STORAGE-2","STORAGE-1",1,1],["STORAGE-1","STORAGE-3",1,1],["STORAGE-3","STORAGE-1",1,1],["STORAGE-2","STORAGE-3",1,1],["STORAGE-3","STORAGE-2",1,1]]` | A list of RSE distance specifications, each a list of 4 values: source RSE, destination RSE, distance (integer), and ranking (integer) |
-| configure_test_setup | bool | `true` | This will configure the rucio server with the storages |
+| configure_rucio | bool | `true` | This will configure the rucio server with the storages |
 | database | object | `{"default":"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"}` | Databases Credentials used by Rucio to access the database. If postgresql subchart is deployed, these credentials should match those in postgresql.global.postgresql.auth. If postgresql subchart is not deployed, an external database must be provided |
 | database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The Rucio database connection URI |
 | dev.client_image_tag | string | `nil` |  |
-| dev.mount_repo | bool | `true` |  |
-| dev.n_test_jobs | int | `4` | number of jobs to use for pytest |
-| dev.run_tests | bool | `true` |  |
+| dev.mount_repo | bool | `true` | mount the repository into the container, useful for development and debugging |
+| dev.n_test_jobs | int | `1` | number of jobs to use for pytest |
+| dev.run_tests | bool | `true` | run tests during helm test (otherwise, the tests can be run manually after exec into the pod) |
 | dev.sleep | bool | `false` | sleep after test to allow interactive development |
 | fts.enabled | bool | `true` | Specifies the configuration for FTS test step (FTS server, FTS database, and ActiveMQ broker containers). Enables or disables the deployment of a FTS instance for testing. This is set to 'False' if an external FTS is used |
 | fts.ftsdb_password | string | `"SDP2RQkbJE2f+ohUb2nUu6Ae10BpQH0VD70CsIQcDtM"` | Defines the password for the FTS database user |
@@ -57,10 +84,8 @@ A Helm chart for the bdms project
 | postgresql.global.postgresql.auth.database | string | `"rucio"` | The name of the database to be created and used by Rucio |
 | postgresql.global.postgresql.auth.password | string | `"XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM"` | The password for the database user |
 | postgresql.global.postgresql.auth.username | string | `"rucio"` | The database username for authentication |
+| postgresql.image.registry | string | `"harbor.cta-observatory.org/proxy_cache"` |  |
 | prepuller_enabled | bool | `true` | Starts containers with the same image as the one used in the deployment before all volumes are available. Saves time in the first deployment |
-| rethinkdb.enabled | bool | `false` |  |
-| rethinkdb.storageClassName | string | `nil` |  |
-| rethinkdb.storageSize | string | `"1Gi"` |  |
 | rucio-daemons.config.common.extract_scope | string | `"ctao_bdms"` |  |
 | rucio-daemons.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | Specifies the connection URI for the Rucio database, these settings will be written to 'rucio.cfg' |
 | rucio-daemons.config.messaging-fts3.brokers | string | `"fts-activemq"` | Specifies the message broker used for FTS messaging |
@@ -137,7 +162,8 @@ A Helm chart for the bdms project
 | rucio_db.existingSecret.enabled | bool | `false` | If true, the database connection URI is obtained from a kubernetes secret in |
 | rucio_db.existingSecret.key | string | `"connection"` | The key inside the kubernetes secret that holds the database connection URI |
 | rucio_db.existingSecret.secretName | string | `"rucio-db"` | The name of the kubernetes secret storing the database connection URI. Its in use only if 'existingSecret.enabled: true' |
-| safe_to_bootstrap_rucio | bool | `true` | This is a destructive operation, it will delete all data in the database |
+| safe_to_bootstrap_rucio | bool | `false` | This is a destructive operation, it will delete all data in the database |
+| safe_to_bootstrap_rucio_on_install | bool | `true` | This is will delete all data in the database only on the first install |
 | server.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
 | server.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
 | server.certificate.existingSecret.key | string | `"tls.key"` | The key inside the kubernetes secret that stores the private key |

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/chart/scripts/bootstrap_rucio/wait_for_rucio.sh

@@ -9,7 +9,7 @@ if [ -z "$WAIT_RUCIO_PING" ]; then
     echo "Skipping rucio ping check"
 else
     while true; do
-        rucio ping && break
+        rucio ping && rucio whoami && break
         sleep 3
     done
 

{ctao_bdms_clients-0.2.1 → ctao_bdms_clients-0.3.0rc1}/chart/templates/bootstrap_jobs.yaml

@@ -1,23 +1,32 @@
 
-{{ if .Values.safe_to_bootstrap_rucio | default false }}
+{{ if .Values.configure_rucio | default false }}
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: {{ template "bdms.fullname" . }}-bootstrap-rucio
+  name: {{ template "bdms.fullname" . }}-configure-rucio-{{ .Release.Revision  }}
+  {{- if .Values.configure.as_hook }}
   annotations:
     "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "-10"
+    "helm.sh/hook-weight": "-5"
     "helm.sh/hook-delete-policy": before-hook-creation
+  {{- end }}
 spec:
   backoffLimit: 0
   template:
     spec:
       volumes:
       {{ include "volumes_cert" . | indent 8 }}
+      resources:
+        requests:
+          memory: "2Gi"
+          cpu: "500m"
+        limits:
+          memory: "8Gi"
+          cpu: "2000m"
       initContainers:
       - name: test-connection
-        image: postgres:latest
+        image: {{ .Values.bootstrap.pg_image.repository }}:{{ .Values.bootstrap.pg_image.tag }}
         command:
         - /bin/sh
         - -c
@@ -39,92 +48,52 @@ spec:
             secretKeyRef:
               name: {{ .Release.Name }}-postgresql
               key: postgres-password
-
       containers:
-      - name: bootstrap-rucio
+      - name: configure-rucio
         image: {{ .Values.bootstrap.image.repository }}:{{ .Values.bootstrap.image.tag }}
+        securityContext:
+          runAsUser: 0
+        env:
+        {{ include "env_helm_release" . | indent 8 }}
         command:
         - /bin/sh
         - -c
         - |
-          set -ex
+          set -eux -o pipefail
 
           {{ .Files.Get "scripts/certificates/install_ca.sh" | indent 10 }}
+
+          WAIT_RUCIO_PING=
           {{ .Files.Get "scripts/bootstrap_rucio/wait_for_rucio.sh" | indent 10 }}
 
+          {{ if or (.Values.safe_to_bootstrap_rucio | default false) (and (.Values.safe_to_bootstrap_rucio_on_install | default false) .Release.IsInstall) }}
           echo "Running reset database script..."
 
           python3 /usr/local/rucio/tools/reset_database.py
 
+          {{ if .Release.IsUpgrade }}
+          # bootstrapping the DB while the rucio server is running causes temporarily inconsistent state of the server
+          # so we need to restart the server after the bootstrap
           curl -LO https://dl.k8s.io/release/v1.33.0/bin/linux/amd64/kubectl -o ./kubectl
           chmod +x ./kubectl
 
           # could be also in post-upgrade hook but only if bootstrap was performed
           ./kubectl rollout restart deployment {{ .Release.Name }}-rucio-server
+          {{ end }}
 
-        volumeMounts:
-        {{ include "volume_mounts_rucio_config" . | indent 8 }}
-        {{ include "volume_mounts_cert" . | indent 8 }}
-        env:
-        {{ include "env_helm_release" . | indent 8 }}
-        - name: POSTGRES_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-postgresql
-              key: postgres-password
-      restartPolicy: OnFailure
-{{ end }}
-{{ if .Values.configure_test_setup | default false }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: {{ template "bdms.fullname" . }}-configure-test-rucio
-  annotations:
-    "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": before-hook-creation
-spec:
-  backoffLimit: 0
-  template:
-    spec:
-      volumes:
-      {{ include "volumes_cert" . | indent 8 }}
-      resources:
-        requests:
-          memory: "2Gi"
-          cpu: "500m"
-        limits:
-          memory: "8Gi"
-          cpu: "2000m"
-      containers:
-      - name: configure-test-rucio
-        image: {{ .Values.bootstrap.image.repository }}:{{ .Values.bootstrap.image.tag }}
-        securityContext:
-          runAsUser: 0
-        env:
-        {{ include "env_helm_release" . | indent 8 }}
-        command:
-        - /bin/sh
-        - -c
-        - |
-          set -ex
+          {{ end }}
 
           # TODO: make or find an image?
           dnf install -y voms-clients
 
-          {{ .Files.Get "scripts/certificates/install_ca.sh" | indent 10 }}
+          voms-proxy-init -valid 9999:00 -cert /opt/rucio/etc/usercert.pem -key /opt/rucio/etc/userkey.pem -out /tmp/x509up
+          cp -fv /tmp/x509up /tmp/x509up_u$(id -u)
 
           WAIT_RUCIO_PING=1
           {{ .Files.Get "scripts/bootstrap_rucio/wait_for_rucio.sh" | indent 10 }}
 
           echo "Configuring test rucio setup ..."
 
-          set -eux -o pipefail
-
-          voms-proxy-init -valid 9999:00 -cert /opt/rucio/etc/usercert.pem -key /opt/rucio/etc/userkey.pem -out /tmp/x509up
-          cp -fv /tmp/x509up /tmp/x509up_u$(id -u)
-
           echo "Configuring identities ..."
           {{ range .Values.configure.identities -}}
           rucio-admin -v identity add
@@ -135,39 +104,35 @@ spec:
 
           echo "Configuring RSE {{ $rse_name }} ..."
 
-          rucio-admin -v rse add "{{ $rse_name }}"
-
-          # TODO: there is a strange race condition here, where the rse is not yet available
-          # in some sequences it does not happen, depending on the order of the rse creation
-          # this time it started to happen after FTS container was separated?
-          while true; do
-            rucio-admin -v rse info "{{ $rse_name }}" && break
-            sleep 3
-          done
+          if rucio-admin -v rse add "{{ $rse_name }}"; then
+            echo "RSE {{ $rse_name }} created"
 
-          {{- range $rse_spec.protocols }}
-          rucio-admin -v rse add-protocol \
-              --hostname "{{ .hostname }}" \
-              --scheme {{ .scheme }} \
-              --prefix {{ .prefix }} \
-              --port {{ .port }} \
-              --impl {{ .impl | default "rucio.rse.protocols.gfal.Default" }} \
-              --domain-json '{{ .domains | toJson }}' \
-              "{{ $rse_name }}"
-          {{- end }}
+            {{- range $rse_spec.protocols }}
+            rucio-admin -v rse add-protocol \
+                --hostname "{{ .hostname }}" \
+                --scheme {{ .scheme }} \
+                --prefix {{ .prefix }} \
+                --port {{ .port }} \
+                --impl {{ .impl | default "rucio.rse.protocols.gfal.Default" }} \
+                --domain-json '{{ .domains | toJson }}' \
+                "{{ $rse_name }}"
+            {{- end }}
 
-          {{- range $k, $v := $rse_spec.attributes }}
-          rucio-admin rse set-attribute --rse "{{ $rse_name }}" --key "{{ $k }}" --value "{{ $v }}"
-          {{- end }}
+            {{- range $k, $v := $rse_spec.attributes }}
+            rucio-admin rse set-attribute --rse "{{ $rse_name }}" --key "{{ $k }}" --value "{{ $v }}"
+            {{- end }}
 
 
-          {{ range $account, $limit := $rse_spec.limits_by_account }}
-          rucio-admin account set-limits {{ $account }} "{{ $rse_name }}" {{ $limit }}
-          {{ end }}
+            {{ range $account, $limit := $rse_spec.limits_by_account }}
+            rucio-admin account set-limits {{ $account }} "{{ $rse_name }}" {{ $limit }}
+            {{ end }}
 
-          echo "Configuring RSE {{ $rse_name }} done"
-          rucio-admin rse info "{{ $rse_name }}"
+            echo "Configuring RSE {{ $rse_name }} done"
+            rucio-admin rse info "{{ $rse_name }}"
 
+          else
+            echo "RSE {{ $rse_name }} already exists, skipping creation and protocols configuration"
+          fi
           {{- end }}
 
           {{- range $distance_tuple := .Values.configure.rse_distances }}
@@ -176,7 +141,7 @@ spec:
             "{{ index $distance_tuple 0 }}" \
             "{{ index $distance_tuple 1 }}" \
             --distance {{ index $distance_tuple 2 }} \
-            --ranking {{ index $distance_tuple 3 }}
+            --ranking {{ index $distance_tuple 3 }} || echo "RSE distance {{ $distance_tuple }} already exists"
           {{- end }}
 
           {{- .Values.configure.extra_script | nindent 10 }}
@@ -193,6 +158,11 @@ spec:
           runAsUser: 0
         env:
         {{ include "env_helm_release" . | indent 8 }}
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-postgresql
+              key: postgres-password
         command:
         - /bin/sh
         - -c
@@ -230,6 +200,8 @@ spec:
         {{ include "volume_mounts_rucio_config" . | indent 8 }}
         {{ include "volume_mounts_cert" . | indent 8 }}
 
+
+      # rucio server might restart during initialization for various reasons
       restartPolicy: Never
 
 {{ end }}