ctao-bdms-clients 0.0.0a0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. ctao_bdms_clients-0.0.0a0/.codespell-ignores +1 -0
  2. ctao_bdms_clients-0.0.0a0/.dockerignore +5 -0
  3. ctao_bdms_clients-0.0.0a0/.flake8 +2 -0
  4. ctao_bdms_clients-0.0.0a0/.gitignore +143 -0
  5. ctao_bdms_clients-0.0.0a0/.gitlab-ci.yml +37 -0
  6. ctao_bdms_clients-0.0.0a0/.gitmodules +9 -0
  7. ctao_bdms_clients-0.0.0a0/.pre-commit-config.yaml +61 -0
  8. ctao_bdms_clients-0.0.0a0/CHANGES.rst +15 -0
  9. ctao_bdms_clients-0.0.0a0/Dockerfile +30 -0
  10. ctao_bdms_clients-0.0.0a0/LICENSE +29 -0
  11. ctao_bdms_clients-0.0.0a0/MANIFEST.in +2 -0
  12. ctao_bdms_clients-0.0.0a0/Makefile +11 -0
  13. ctao_bdms_clients-0.0.0a0/PKG-INFO +35 -0
  14. ctao_bdms_clients-0.0.0a0/README.md +5 -0
  15. ctao_bdms_clients-0.0.0a0/aiv-config-dependencies.yml +4 -0
  16. ctao_bdms_clients-0.0.0a0/aiv-config.yml +18 -0
  17. ctao_bdms_clients-0.0.0a0/chart/Chart.yaml +39 -0
  18. ctao_bdms_clients-0.0.0a0/chart/Makefile +15 -0
  19. ctao_bdms_clients-0.0.0a0/chart/README.md +140 -0
  20. ctao_bdms_clients-0.0.0a0/chart/scripts/bootstrap_rucio/setup_rucio.sh +62 -0
  21. ctao_bdms_clients-0.0.0a0/chart/scripts/certificates/install_ca.sh +18 -0
  22. ctao_bdms_clients-0.0.0a0/chart/templates/_helpers.tpl +63 -0
  23. ctao_bdms_clients-0.0.0a0/chart/templates/bootstrap_jobs.yaml +171 -0
  24. ctao_bdms_clients-0.0.0a0/chart/templates/configmap.yaml +100 -0
  25. ctao_bdms_clients-0.0.0a0/chart/templates/prepuller.yaml +24 -0
  26. ctao_bdms_clients-0.0.0a0/chart/templates/test_storages.yaml +128 -0
  27. ctao_bdms_clients-0.0.0a0/chart/templates/tests/test_jobs.yaml +127 -0
  28. ctao_bdms_clients-0.0.0a0/chart/values.yaml +330 -0
  29. ctao_bdms_clients-0.0.0a0/docs/Makefile +23 -0
  30. ctao_bdms_clients-0.0.0a0/docs/changelog.rst +6 -0
  31. ctao_bdms_clients-0.0.0a0/docs/changes/template.rst +43 -0
  32. ctao_bdms_clients-0.0.0a0/docs/chart.rst +6 -0
  33. ctao_bdms_clients-0.0.0a0/docs/conf.py +74 -0
  34. ctao_bdms_clients-0.0.0a0/docs/getting_started.rst +43 -0
  35. ctao_bdms_clients-0.0.0a0/docs/index.rst +23 -0
  36. ctao_bdms_clients-0.0.0a0/docs/reference.rst +7 -0
  37. ctao_bdms_clients-0.0.0a0/docs/server_setup/bdms_repos_interaction.rst +59 -0
  38. ctao_bdms_clients-0.0.0a0/docs/server_setup/certificates.rst +20 -0
  39. ctao_bdms_clients-0.0.0a0/docs/server_setup/fts.rst +35 -0
  40. ctao_bdms_clients-0.0.0a0/docs/server_setup/index.rst +12 -0
  41. ctao_bdms_clients-0.0.0a0/docs/server_setup/rucio_configuration.rst +88 -0
  42. ctao_bdms_clients-0.0.0a0/docs/server_setup/storage_elements.rst +48 -0
  43. ctao_bdms_clients-0.0.0a0/pyproject.toml +175 -0
  44. ctao_bdms_clients-0.0.0a0/report/inspection.tex +0 -0
  45. ctao_bdms_clients-0.0.0a0/report/performance_verification.tex +0 -0
  46. ctao_bdms_clients-0.0.0a0/report/preamble.tex +0 -0
  47. ctao_bdms_clients-0.0.0a0/setup.cfg +4 -0
  48. ctao_bdms_clients-0.0.0a0/sonar-project.properties +11 -0
  49. ctao_bdms_clients-0.0.0a0/src/bdms/__init__.py +7 -0
  50. ctao_bdms_clients-0.0.0a0/src/bdms/_dev_version/__init__.py +9 -0
  51. ctao_bdms_clients-0.0.0a0/src/bdms/_version.py +16 -0
  52. ctao_bdms_clients-0.0.0a0/src/bdms/tests/__init__.py +0 -0
  53. ctao_bdms_clients-0.0.0a0/src/bdms/tests/conftest.py +53 -0
  54. ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_basic_rucio_functionality.py +143 -0
  55. ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_dpps_rel_0_0.py +90 -0
  56. ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_file_replicas.py +72 -0
  57. ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_metadata.py +110 -0
  58. ctao_bdms_clients-0.0.0a0/src/bdms/version.py +23 -0
  59. ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/PKG-INFO +35 -0
  60. ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/SOURCES.txt +62 -0
  61. ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/dependency_links.txt +1 -0
  62. ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/entry_points.txt +2 -0
  63. ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/requires.txt +22 -0
  64. ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/top_level.txt +1 -0
@@ -0,0 +1 @@
1
+ anull
@@ -0,0 +1,5 @@
1
+ **/__pycache__
2
+ **/*.pyc
3
+ **/*.egg-info
4
+ **/build
5
+ **/_version.py
@@ -0,0 +1,2 @@
1
+ [flake8]
2
+ max-line-length=88
@@ -0,0 +1,143 @@
1
+ # Files downloaded by the dpps-aiv-toolkit
2
+ Chart.lock
3
+ helm
4
+ helm.tar.gz
5
+ kind
6
+ kubectl
7
+ linux-amd64
8
+ # generated file
9
+ chart/**/*.tgz
10
+ report.xml
11
+ # setuptools_scm generated version file
12
+ _version.py
13
+
14
+ # Byte-compiled / optimized / DLL files
15
+ __pycache__/
16
+ *.py[cod]
17
+ *$py.class
18
+
19
+ # C extensions
20
+ *.so
21
+
22
+ # Distribution / packaging
23
+ .Python
24
+ build/
25
+ develop-eggs/
26
+ dist/
27
+ downloads/
28
+ eggs/
29
+ .eggs/
30
+ lib/
31
+ lib64/
32
+ parts/
33
+ sdist/
34
+ var/
35
+ wheels/
36
+ pip-wheel-metadata/
37
+ share/python-wheels/
38
+ *.egg-info/
39
+ .installed.cfg
40
+ *.egg
41
+ MANIFEST
42
+
43
+ # PyInstaller
44
+ # Usually these files are written by a python script from a template
45
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
46
+ *.manifest
47
+ *.spec
48
+
49
+ # Installer logs
50
+ pip-log.txt
51
+ pip-delete-this-directory.txt
52
+
53
+ # Unit test / coverage reports
54
+ htmlcov/
55
+ .tox/
56
+ .nox/
57
+ .coverage
58
+ .coverage.*
59
+ .cache
60
+ nosetests.xml
61
+ coverage.xml
62
+ *.cover
63
+ *.py,cover
64
+ .hypothesis/
65
+ .pytest_cache/
66
+
67
+ # Translations
68
+ *.mo
69
+ *.pot
70
+
71
+ # Django stuff:
72
+ *.log
73
+ local_settings.py
74
+ db.sqlite3
75
+ db.sqlite3-journal
76
+
77
+ # Flask stuff:
78
+ instance/
79
+ .webassets-cache
80
+
81
+ # Scrapy stuff:
82
+ .scrapy
83
+
84
+ # Sphinx documentation
85
+ docs/_build/
86
+ docs/api
87
+
88
+ # PyBuilder
89
+ target/
90
+
91
+ # Jupyter Notebook
92
+ .ipynb_checkpoints
93
+
94
+ # IPython
95
+ profile_default/
96
+ ipython_config.py
97
+
98
+ # pyenv
99
+ .python-version
100
+
101
+ # pipenv
102
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
103
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
104
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
105
+ # install all needed dependencies.
106
+ #Pipfile.lock
107
+
108
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
109
+ __pypackages__/
110
+
111
+ # Celery stuff
112
+ celerybeat-schedule
113
+ celerybeat.pid
114
+
115
+ # SageMath parsed files
116
+ *.sage.py
117
+
118
+ # Environments
119
+ .env
120
+ .venv
121
+ env/
122
+ venv/
123
+ ENV/
124
+ env.bak/
125
+ venv.bak/
126
+
127
+ # Spyder project settings
128
+ .spyderproject
129
+ .spyproject
130
+
131
+ # Rope project settings
132
+ .ropeproject
133
+
134
+ # mkdocs documentation
135
+ /site
136
+
137
+ # mypy
138
+ .mypy_cache/
139
+ .dmypy.json
140
+ dmypy.json
141
+
142
+ # Pyre type checker
143
+ .pyre/
@@ -0,0 +1,37 @@
1
+ include:
2
+ - project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
3
+ ref: 705e92d15a41d000f458dcb11d662f83d5718aaf
4
+ file: 'ci-functions.yml'
5
+ - "aiv-config.yml"
6
+
7
+
8
+ variables:
9
+ CHART_LOCATION: chart
10
+ CHART_NAME: bdms
11
+ CHART_EXTRA_VALUES: "--set client_image_tag=${DOCKER_TAG}"
12
+ DPPS_AIV_TOOLKIT_DIR: dpps-aiv-toolkit
13
+ DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
14
+ RUCIO_VERSION: "35.4.1"
15
+ RUCIO_TAG: "release-${RUCIO_VERSION}"
16
+
17
+ stages:
18
+ - lint
19
+ - build
20
+ - sign
21
+ - tests
22
+ - sonarqube
23
+ - publish
24
+ - report
25
+
26
+ build:
27
+ variables:
28
+ CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-client:${DOCKER_TAG}'
29
+ KANIKO_EXTRA_ARGS: --build-arg RUCIO_TAG=${RUCIO_TAG}
30
+
31
+ hadolint:
32
+ rules:
33
+ - when: never
34
+
35
+ sign:
36
+ rules:
37
+ - when: never
@@ -0,0 +1,9 @@
1
+ [submodule "dpps-aiv-toolkit"]
2
+ path = dpps-aiv-toolkit
3
+ url = ../../aiv/dpps-aiv-toolkit.git
4
+ [submodule "dpps-release-plan"]
5
+ path = dpps-release-plan
6
+ url = ../../dpps-project-management/dpps-release-plan.git
7
+ [submodule "requirements-traceability"]
8
+ path = requirements-traceability
9
+ url = ../../dpps-project-management/requirements-traceability.git
@@ -0,0 +1,61 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v5.0.0 # Use the ref you want to point at
4
+ hooks:
5
+ - id: trailing-whitespace
6
+ exclude: '(.*\.fits(\.fz)?)|(.*.patch.*)$'
7
+ - id: check-added-large-files
8
+ - id: check-case-conflict
9
+ - id: check-merge-conflict
10
+ - id: end-of-file-fixer
11
+ exclude: '(.*\.fits(\.fz)?)|(.*.patch.*)|(chart\/.*README.md)$'
12
+
13
+ - repo: https://github.com/codespell-project/codespell
14
+ rev: v2.3.0
15
+ hooks:
16
+ - id: codespell
17
+ additional_dependencies:
18
+ - tomli
19
+
20
+ - repo: https://github.com/astral-sh/ruff-pre-commit
21
+ rev: v0.8.0
22
+ hooks:
23
+ - id: ruff
24
+ args: [ --fix, --show-fixes ]
25
+ exclude: chart/charts/rucio-daemons/patches/request_patched.py
26
+ - id: ruff-format
27
+ exclude: chart/charts/rucio-daemons/patches/request_patched.py
28
+ exclude: '.*.patch.*'
29
+ - id: ruff-format
30
+ exclude: '.*.patch.*'
31
+
32
+ - repo: https://github.com/scientific-python/cookie
33
+ rev: "2024.08.19"
34
+ hooks:
35
+ - id: sp-repo-review
36
+
37
+ - repo: local
38
+ hooks:
39
+ - id: CI yaml check
40
+ args: [ dpps-aiv-toolkit ]
41
+ name: CI yaml check
42
+ description: CI yaml check
43
+ language: script
44
+ entry: dpps-aiv-toolkit/cicheck.sh
45
+ pass_filenames: true
46
+ files: ^.gitlab-ci.yml$
47
+ always_run: true
48
+
49
+ - repo: https://github.com/norwoodj/helm-docs
50
+ rev: "v1.14.2"
51
+ hooks:
52
+ - id: helm-docs-built
53
+ args:
54
+ - "--chart-search-root=chart"
55
+
56
+ # - id: helm-lint
57
+ # name: helm-lint
58
+ # language: docker_image
59
+ # entry: alpine/helm:3.11.1 lint chart/ # --set
60
+ # always_run: true
61
+ # pass_filenames: false
@@ -0,0 +1,15 @@
1
+ BDMS v0.1.0rc1 (2025-02-12)
2
+ ---------------------------
3
+
4
+ First release candidate for the Bulk Data Management System (BDMS).
5
+
6
+ * Deployment of Rucio 35.4 using helm
7
+
8
+
9
+
10
+ Maintenance
11
+ ~~~~~~~~~~~
12
+
13
+ - Write and publish BDMS documentation for DPPS Release 0. [`!47 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/47>`__]
14
+
15
+ - Setup ``towncrier`` for changelog generation. [`!49 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/49>`__]
@@ -0,0 +1,30 @@
1
+ # rucio version used as base for the final image
2
+ ARG RUCIO_TAG=release-35.4.1
3
+
4
+ FROM python:3.9 AS builder
5
+
6
+ COPY pyproject.toml MANIFEST.in /tmp/bdms/
7
+ COPY .git /tmp/bdms/.git/
8
+ COPY src /tmp/bdms/src/
9
+
10
+ RUN python -m pip install build \
11
+ && python -m build /tmp/bdms --wheel -o /tmp/dist
12
+
13
+ # second stage, copy and install wheel
14
+ # We are using the official python 3.11 image
15
+ # as base image in the slim variant to reduce image size.
16
+ FROM rucio/rucio-clients:${RUCIO_TAG}
17
+
18
+ ARG RUCIO_TAG
19
+
20
+ # server and daemons use root, clients use "user", switch to root, install, then back
21
+ USER root
22
+ COPY --from=builder /tmp/dist/ /tmp/dist/
23
+ RUN dnf install -y --setopt=install_weak_deps=False git \
24
+ && python3 -m pip install --no-cache-dir /tmp/dist/ctao_bdms* \
25
+ && dnf autoremove \
26
+ && dnf clean all
27
+
28
+ USER user
29
+
30
+ ENV RUCIO_POLICY_PACKAGE=bdms_rucio_policy
@@ -0,0 +1,29 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2022, Cherenkov Telescope Array Observatory Consortium
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ 1. Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ 2. Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ 3. Neither the name of the copyright holder nor the names of its
17
+ contributors may be used to endorse or promote products derived from
18
+ this software without specific prior written permission.
19
+
20
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,2 @@
1
+ prune src/template/_dev_version
2
+ prune .github
@@ -0,0 +1,11 @@
1
+ ###
2
+ # TODO: duplicate in config?
3
+ export CHART_NAME=bdms
4
+ export CHART_LOCATION=chart
5
+ include dpps-aiv-toolkit/Makefile
6
+
7
+
8
+ # TODO: move this to kit
9
+ export TEST_ARTIFACTS_PATH ?= $(PWD)
10
+ export TEST_REPORT_CONFIG ?= $(PWD)/aiv-config.yml
11
+ export TEX_CONTENT_PATH ?= $(PWD)/report
@@ -0,0 +1,35 @@
1
+ Metadata-Version: 2.2
2
+ Name: ctao-bdms-clients
3
+ Version: 0.0.0a0
4
+ Summary: Client module for the CTAO DPPS Bulk Data Management System
5
+ Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
6
+ License: BSD-3-Clause
7
+ Project-URL: repository, https://github.com/cta-observatory/...
8
+ Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/documentation/...
9
+ Requires-Python: >=3.9
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: rucio-clients~=35.4.1
13
+ Requires-Dist: ctao-bdms-rucio-policy==0.1.0
14
+ Provides-Extra: test
15
+ Requires-Dist: pytest; extra == "test"
16
+ Requires-Dist: pytest-cov; extra == "test"
17
+ Requires-Dist: pytest-mock; extra == "test"
18
+ Requires-Dist: pytest-requirements; extra == "test"
19
+ Provides-Extra: doc
20
+ Requires-Dist: sphinx; extra == "doc"
21
+ Requires-Dist: numpydoc; extra == "doc"
22
+ Requires-Dist: ctao-sphinx-theme; extra == "doc"
23
+ Requires-Dist: myst-parser; extra == "doc"
24
+ Requires-Dist: sphinx-changelog; extra == "doc"
25
+ Provides-Extra: dev
26
+ Requires-Dist: setuptools_scm; extra == "dev"
27
+ Requires-Dist: sphinx-autobuild; extra == "dev"
28
+ Provides-Extra: all
29
+ Requires-Dist: bdms[dev,doc,test]; extra == "all"
30
+
31
+ # Bulk Data Management System
32
+
33
+ [![CI](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/badges/main/pipeline.svg?key_text=CI&key_width=25)](https://gitlab.cta-observatory.org/dpps/bdms/bdms/-/pipelines/main/latest)
34
+ [![Test Report](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/badges/main/pipeline.svg?key_text=Test%20Report)](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/jobs/artifacts/main/file/test_report.pdf?job=build-test-report)
35
+ [![Docs](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/badges/main/pipeline.svg?key_text=docs&key_width=30)](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms/)
@@ -0,0 +1,5 @@
1
+ # Bulk Data Management System
2
+
3
+ [![CI](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/badges/main/pipeline.svg?key_text=CI&key_width=25)](https://gitlab.cta-observatory.org/dpps/bdms/bdms/-/pipelines/main/latest)
4
+ [![Test Report](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/badges/main/pipeline.svg?key_text=Test%20Report)](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/jobs/artifacts/main/file/test_report.pdf?job=build-test-report)
5
+ [![Docs](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/badges/main/pipeline.svg?key_text=docs&key_width=30)](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms/)
@@ -0,0 +1,4 @@
1
+ dependencies:
2
+ - gitlab_path: cta-computing/dpps/bdms/bdms-rucio-policy
3
+ revision: main
4
+ sonar_key: cta-computing_dpps_bdms_bdms-rucio-policy_AZPO4IWyAfB1AuE28R3d
@@ -0,0 +1,18 @@
1
+ # This is the configuration used both in CI and local Makefile
2
+
3
+ variables:
4
+ # Comma-separated list of "UC Groups" to select UCs and Requirements, e.g. "BDMS,AIV"
5
+ DPPS_UC_GROUPS: BDMS
6
+ DPPS_RELEASE: v0.0
7
+
8
+ # these paths are relative to TEST_ARTIFACTS_PATH
9
+ REPORT_XML: report.xml
10
+ RELEASE_PLAN_FN: dpps-release-plan/release_development_document/dpps-release-plan.tex
11
+ TRACEABILITY_CSV_FN: requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
12
+
13
+ APPLICATION_NAME: BDMS
14
+ APPLICATION_AUTHOR: BDMS Team
15
+ APPLICATION_AUTHOR_ORGANIZATION: CTAO
16
+ APPLICATION_VERSION: 0.0.0
17
+
18
+ EXTRA_CONFIG_FILES: aiv-config-dependencies.yml
@@ -0,0 +1,39 @@
1
+ apiVersion: v2
2
+ name: bdms
3
+ version: 0.1.0
4
+ appVersion: 0.1.0
5
+ description: A Helm chart for the bdms project
6
+ type: application
7
+ keywords:
8
+ - bdms
9
+ - rucio
10
+ - monitoring
11
+ - postgresql
12
+
13
+ maintainers:
14
+ - name: The BDMS Authors
15
+ email:
16
+
17
+ dependencies:
18
+ - name: postgresql
19
+ condition: postgresql.enabled
20
+ version: 15.5.10
21
+ repository: oci://registry-1.docker.io/bitnamicharts
22
+
23
+ - name: rucio-server
24
+ version: 35.0.0
25
+ repository: oci://harbor.cta-observatory.org/dpps
26
+
27
+ - name: rucio-daemons
28
+ version: 35.0.0
29
+ repository: oci://harbor.cta-observatory.org/dpps
30
+
31
+ - name: cert-generator-grid
32
+ condition: cert-generator-grid.enabled
33
+ version: 0.0.0-75a4994-75a4994c
34
+ repository: oci://harbor.cta-observatory.org/dpps
35
+
36
+ - name: fts
37
+ condition: fts.enabled
38
+ version: 0.0.0-c435e63-c435e636
39
+ repository: oci://harbor.cta-observatory.org/dpps
@@ -0,0 +1,15 @@
1
+ install:
2
+ helm upgrade --install --wait bdms .
3
+
4
+ test:
5
+ helm test bdms
6
+
7
+ publish:
8
+ helm package .
9
+ helm push bdms-0.1.0.tgz oci://harbor.cta-observatory.org/dpps
10
+
11
+ reset:
12
+ helm delete bdms || true
13
+ kubectl delete secrets bdms-rucio-server bdms-rucio-server-tls bdms-server-cafile bdms-server-hostcert bdms-server-hostkey dppsuser-certkey || true
14
+ kubectl delete job generate-certificates configure-test-rucio test-rucio || true
15
+ kubectl delete pvc data-bdms-postgresql-0 || true
@@ -0,0 +1,140 @@
1
+ # bdms
2
+
3
+ ![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square)
4
+
5
+ A Helm chart for the bdms project
6
+
7
+ ## Maintainers
8
+
9
+ | Name | Email | Url |
10
+ | ---- | ------ | --- |
11
+ | The BDMS Authors | | |
12
+
13
+ ## Requirements
14
+
15
+ | Repository | Name | Version |
16
+ |------------|------|---------|
17
+ | oci://harbor.cta-observatory.org/dpps | cert-generator-grid | 0.0.0-75a4994-75a4994c |
18
+ | oci://harbor.cta-observatory.org/dpps | fts | 0.0.0-c435e63-c435e636 |
19
+ | oci://harbor.cta-observatory.org/dpps | rucio-daemons | 35.0.0 |
20
+ | oci://harbor.cta-observatory.org/dpps | rucio-server | 35.0.0 |
21
+ | oci://registry-1.docker.io/bitnamicharts | postgresql | 15.5.10 |
22
+
23
+ ## Values
24
+
25
+ | Key | Type | Default | Description |
26
+ |-----|------|---------|-------------|
27
+ | auth.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. It is used by clients and services to authenticate with Rucio |
28
+ | auth.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
29
+ | auth.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
30
+ | auth.certificate.existingSecret.key | string | `"tls.key"` | The key inside the kubernetes secret that stores the private key |
31
+ | auth.certificate.existingSecret.secretName | string | `"rucio-server.local"` | The name of the kubernetes secret containing the TLS certificate and key |
32
+ | auth.certificate.letsencrypt.email | string | `""` | Email address for Let's encrypt registration and renewal reminders |
33
+ | auth.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
34
+ | bootstrap.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image for bootstrapping Rucio (initialization, configuration) with the CTAO Rucio policy package installed |
35
+ | bootstrap.image.tag | string | `"35.4.1-v0.1.0"` | The specific image tag to use for the bootstrap container |
36
+ | cert-generator-grid.enabled | bool | `true` | |
37
+ | cert-generator-grid.generatePreHooks | bool | `true` | |
38
+ | client_image_tag | string | `"6cbd744c"` | |
39
+ | configure_test_setup | bool | `true` | This will configure the rucio server with the storages |
40
+ | database | object | `{"default":"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"}` | Databases Credentials used by Rucio to access the database. If postgresql subchart is deployed, these credentials should match those in postgresql.global.postgresql.auth. If postgresql subchart is not deployed, an external database must be provided |
41
+ | database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The Rucio database connection URI |
42
+ | dev.mount_repo | bool | `true` | |
43
+ | dev.run_tests | bool | `true` | |
44
+ | dev.sleep | bool | `false` | sleep after test to allow interactive development |
45
+ | fts.enabled | bool | `true` | Specifies the configuration for FTS test step (FTS server, FTS database, and ActiveMQ broker containers). Enables or disables the deployment of a FTS instance for testing. This is set to 'False' if an external FTS is used |
46
+ | fts.ftsdb_password | string | `"SDP2RQkbJE2f+ohUb2nUu6Ae10BpQH0VD70CsIQcDtM"` | Defines the password for the FTS database user |
47
+ | fts.ftsdb_root_password | string | `"iB7dMiIybdoaozWZMkvRo0eg9HbQzG9+5up50zUDjE4"` | Defines the root password for the FTS database |
48
+ | fts.image.pullPolicy | string | `"Always"` | |
49
+ | fts.image.repository | string | `"harbor.cta-observatory.org/proxy_cache/rucio/fts"` | The container image repository for the FTS deployment |
50
+ | fts.image.tag | string | `"35.4.1"` | Defines the specific version of the FTS image to use |
51
+ | fts.messaging.broker | string | `"localhost:61613"` | |
52
+ | fts.messaging.password | string | `"topsecret"` | |
53
+ | fts.messaging.use_broker_credentials | string | `"true"` | |
54
+ | fts.messaging.username | string | `"fts"` | |
55
+ | postgresql.enabled | bool | `true` | Configuration of built-in postgresql database. If 'enabled: true', a postgresql instance will be deployed, otherwise, an external database must be provided in database.default value |
56
+ | postgresql.global.postgresql.auth.database | string | `"rucio"` | The name of the database to be created and used by Rucio |
57
+ | postgresql.global.postgresql.auth.password | string | `"XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM"` | The password for the database user |
58
+ | postgresql.global.postgresql.auth.username | string | `"rucio"` | The database username for authentication |
59
+ | prepuller_enabled | bool | `true` | Starts containers with the same image as the one used in the deployment before all volumes are available. Saves time in the first deployment |
60
+ | rethinkdb.enabled | bool | `false` | |
61
+ | rethinkdb.storageClassName | string | `nil` | |
62
+ | rethinkdb.storageSize | string | `"1Gi"` | |
63
+ | rucio-daemons.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | Specifies the connection URI for the Rucio database, these settings will be written to 'rucio.cfg' |
64
+ | rucio-daemons.config.messaging-fts3.brokers | string | `"fts-activemq"` | Specifies the message broker used for FTS messaging |
65
+ | rucio-daemons.config.messaging-fts3.destination | string | `"/topic/transfer.fts_monitoring_complete"` | Specifies the message broker queue path where FTS sends transfer status updates. This is the place where Rucio listens for completed transfer notifications |
66
+ | rucio-daemons.config.messaging-fts3.nonssl_port | int | `61613` | Specifies the non-SSL port |
67
+ | rucio-daemons.config.messaging-fts3.password | string | `"topsecret"` | Specifies the authentication credential (password) for connecting to the message broker |
68
+ | rucio-daemons.config.messaging-fts3.port | int | `61613` | Defines the port used for the broker |
69
+ | rucio-daemons.config.messaging-fts3.use_ssl | bool | `false` | Determines whether to use SSL for message broker connections. If true, valid certificates are required for securing the connection |
70
+ | rucio-daemons.config.messaging-fts3.username | string | `"fts"` | Specifies the authentication credential (username) for connecting to the message broker |
71
+ | rucio-daemons.config.policy.permission | string | `"ctao"` | Defines the policy permission model for Rucio for determining how authorization and access controls are applied, its value should be taken from the installed Rucio policy package |
72
+ | rucio-daemons.conveyorFinisher.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
73
+ | rucio-daemons.conveyorFinisher.resources.limits.cpu | string | `"3000m"` | |
74
+ | rucio-daemons.conveyorFinisher.resources.limits.memory | string | `"4Gi"` | |
75
+ | rucio-daemons.conveyorFinisher.resources.requests.cpu | string | `"700m"` | |
76
+ | rucio-daemons.conveyorFinisher.resources.requests.memory | string | `"200Mi"` | |
77
+ | rucio-daemons.conveyorFinisher.sleepTime | int | `5` | Defines how often (in seconds) the daemon processes finished transfers |
78
+ | rucio-daemons.conveyorFinisherCount | int | `1` | Marks completed transfers and updates metadata |
79
+ | rucio-daemons.conveyorPoller.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
80
+ | rucio-daemons.conveyorPoller.olderThan | int | `600` | Filters transfers that are older than the specified time (in seconds) before polling |
81
+ | rucio-daemons.conveyorPoller.resources.limits.cpu | string | `"3000m"` | |
82
+ | rucio-daemons.conveyorPoller.resources.limits.memory | string | `"4Gi"` | |
83
+ | rucio-daemons.conveyorPoller.resources.requests.cpu | string | `"700m"` | |
84
+ | rucio-daemons.conveyorPoller.resources.requests.memory | string | `"200Mi"` | |
85
+ | rucio-daemons.conveyorPoller.sleepTime | int | `60` | Defines how often (in seconds) the daemon polls for transfer status updates |
86
+ | rucio-daemons.conveyorPollerCount | int | `1` | Polls FTS to check the status of ongoing transfers |
87
+ | rucio-daemons.conveyorReceiverCount | int | `1` | Listens to messages from ActiveMQ, which FTS uses to publish transfer status updates. This ensures Rucio is notified of completed or failed transfers in real time |
88
+ | rucio-daemons.conveyorTransferSubmitter.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
89
+ | rucio-daemons.conveyorTransferSubmitter.archiveTimeout | string | `""` | Sets the timeout if required for archiving completed transfers |
90
+ | rucio-daemons.conveyorTransferSubmitter.resources.limits.cpu | string | `"3000m"` | |
91
+ | rucio-daemons.conveyorTransferSubmitter.resources.limits.memory | string | `"4Gi"` | |
92
+ | rucio-daemons.conveyorTransferSubmitter.resources.requests.cpu | string | `"700m"` | |
93
+ | rucio-daemons.conveyorTransferSubmitter.resources.requests.memory | string | `"200Mi"` | |
94
+ | rucio-daemons.conveyorTransferSubmitter.sleepTime | int | `5` | Defines the interval (in seconds) the daemon waits before checking for new transfers |
95
+ | rucio-daemons.conveyorTransferSubmitterCount | int | `1` | Number of container instances to deploy for each Rucio daemon, this daemon submits new transfer requests to the FTS |
96
+ | rucio-daemons.image.pullPolicy | string | `"Always"` | It defines when kubernetes should pull the container image, the options available are: Always, IfNotPresent, and Never |
97
+ | rucio-daemons.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-daemons"` | Specifies the container image repository for Rucio daemons |
98
+ | rucio-daemons.image.tag | string | `"35.4.1-v0.1.0"` | Specific image tag to use for deployment |
99
+ | rucio-daemons.judgeEvaluatorCount | int | `1` | Evaluates Rucio replication rules and triggers transfers |
100
+ | rucio-daemons.useDeprecatedImplicitSecrets | bool | `true` | Enables the use of deprecated implicit secrets for authentication |
101
+ | rucio-server.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. |
102
+ | rucio-server.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The database connection URI for Rucio |
103
+ | rucio-server.config.persistence.accessMode | string | `"ReadWriteOnce"` | Defines how storage can be accessed, the options available are: ReadWriteOnce, ReadOnlyMany, ReadWriteMany |
104
+ | rucio-server.config.persistence.enabled | bool | `true` | Enables persistent storage for Rucio server, setting it to false means using temporary storage ('ephemeral' in the context of kubernetes) that gets wiped out and lost when the container is stopped or restarted |
105
+ | rucio-server.config.persistence.size | string | `"1Gi"` | Defines the size of persistent volume claim (PVC) or the amount of the storage capacity provisioned to the Rucio server |
106
+ | rucio-server.config.persistence.storageClass | string | `""` | Specifies the kubernetes storage class for persistent volume, default storage class is used when its value is left empty |
107
+ | rucio-server.ftsRenewal.enabled | bool | `false` | Enables automatic renewal of FTS credentials using X.509 certificates and proxy |
108
+ | rucio-server.httpd_config.encoded_slashes | string | `"True"` | Allows for custom LFNs with slashes in request URLs so that Rucio server (Apache) can decode and handle such requests properly |
109
+ | rucio-server.httpd_config.grid_site_enabled | string | `"True"` | Enables Rucio server to support and interact with grid middleware (storages) for X509 authentication with proxies |
110
+ | rucio-server.image.pullPolicy | string | `"Always"` | It defines when kubernetes should pull the container image, the options available are: Always, IfNotPresent, and Never |
111
+ | rucio-server.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image repository for Rucio server with the CTAO Rucio policy package installed |
112
+ | rucio-server.image.tag | string | `"35.4.1-v0.1.0"` | The specific image tag to deploy |
113
+ | rucio-server.ingress.enabled | bool | `true` | Enables an ingress resource (controller) for exposing the Rucio server externally to allow clients connect to the Rucio server. It needs one of the ingress controllers (NGINX, Traefik) to be installed |
114
+ | rucio-server.ingress.hosts | list | `["rucio-server-manual-tc.local"]` | Defines the hostname to be used to access the Rucio server. It should match DNS configuration and TLS certificates |
115
+ | rucio-server.replicaCount | int | `1` | Number of replicas of the Rucio server to deploy. We can increase it to meet higher availability goals |
116
+ | rucio-server.service.name | string | `"https"` | The name of the service port |
117
+ | rucio-server.service.port | int | `443` | The port exposed by the kubernetes service, making the Rucio server accessible within the cluster |
118
+ | rucio-server.service.protocol | string | `"TCP"` | The network protocol used for HTTPS based communication |
119
+ | rucio-server.service.targetPort | int | `443` | The port inside the Rucio server container that listens for incoming traffic |
120
+ | rucio-server.service.type | string | `"ClusterIP"` | Specifies the kubernetes service type for making the Rucio server accessible within or outside the kubernetes cluster, available options include clusterIP (internal access only, default), NodePort (exposes the service on port across all cluster nodes), and LoadBalancer (Uses an external load balancer) |
121
+ | rucio-server.useSSL | bool | `true` | Enables the Rucio server to use SSL/TLS for secure communication, requiring valid certificates to be configured |
122
+ | rucio.password | string | `"secret"` | |
123
+ | rucio.username | string | `"dpps"` | Specifies the username for Rucio operations as part of Rucio configuration |
124
+ | rucio.version | string | `"35.4.1"` | The version of Rucio being deployed |
125
+ | rucio_db.connection | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The database connection URI for Rucio. It is of the format: `postgresql://<user>:<password>@<host>:<port>/<database>`, this field in use only if 'existingSecret.enabled' is set to 'false', otherwise ignored |
126
+ | rucio_db.deploy | bool | `true` | If true, deploys a postgresql instance for the Rucio database, otherwise use an external database |
127
+ | rucio_db.existingSecret.enabled | bool | `false` | If true, the database connection URI is obtained from a kubernetes secret in |
128
+ | rucio_db.existingSecret.key | string | `"connection"` | The key inside the kubernetes secret that holds the database connection URI |
129
+ | rucio_db.existingSecret.secretName | string | `"rucio-db"` | The name of the kubernetes secret storing the database connection URI. Its in use only if 'existingSecret.enabled: true' |
130
+ | safe_to_bootstrap_rucio | bool | `true` | This is a destructive operation, it will delete all data in the database |
131
+ | server.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
132
+ | server.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
133
+ | server.certificate.existingSecret.key | string | `"tls.key"` | The key inside the kubernetes secret that stores the private key |
134
+ | server.certificate.existingSecret.secretName | string | `"rucio-server-manual-tc.local"` | The name of the kubernetes secret containing the TLS certificate and key |
135
+ | server.certificate.letsencrypt.email | string | `""` | |
136
+ | server.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
137
+ | server.rucioHost | string | `"rucio-server-manual-tc.local"` | The hostname of the Rucio server. It is used by clients and services to communicate with Rucio |
138
+ | storages | list | `["rucio-storage-1","rucio-storage-2","rucio-storage-3"]` | a list of storage element (RSE) hostnames names, for each RSE, one deployment and service are configured, two configmaps xrdconfig and xrd-entrypoint for those three storages |
139
+ | suffix_namespace | string | `"default"` | Specifies the Namespace suffix used for managing deployments in kubernetes |
140
+