ctao-bdms-clients 0.0.0a0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctao_bdms_clients-0.0.0a0/.codespell-ignores +1 -0
- ctao_bdms_clients-0.0.0a0/.dockerignore +5 -0
- ctao_bdms_clients-0.0.0a0/.flake8 +2 -0
- ctao_bdms_clients-0.0.0a0/.gitignore +143 -0
- ctao_bdms_clients-0.0.0a0/.gitlab-ci.yml +37 -0
- ctao_bdms_clients-0.0.0a0/.gitmodules +9 -0
- ctao_bdms_clients-0.0.0a0/.pre-commit-config.yaml +61 -0
- ctao_bdms_clients-0.0.0a0/CHANGES.rst +15 -0
- ctao_bdms_clients-0.0.0a0/Dockerfile +30 -0
- ctao_bdms_clients-0.0.0a0/LICENSE +29 -0
- ctao_bdms_clients-0.0.0a0/MANIFEST.in +2 -0
- ctao_bdms_clients-0.0.0a0/Makefile +11 -0
- ctao_bdms_clients-0.0.0a0/PKG-INFO +35 -0
- ctao_bdms_clients-0.0.0a0/README.md +5 -0
- ctao_bdms_clients-0.0.0a0/aiv-config-dependencies.yml +4 -0
- ctao_bdms_clients-0.0.0a0/aiv-config.yml +18 -0
- ctao_bdms_clients-0.0.0a0/chart/Chart.yaml +39 -0
- ctao_bdms_clients-0.0.0a0/chart/Makefile +15 -0
- ctao_bdms_clients-0.0.0a0/chart/README.md +140 -0
- ctao_bdms_clients-0.0.0a0/chart/scripts/bootstrap_rucio/setup_rucio.sh +62 -0
- ctao_bdms_clients-0.0.0a0/chart/scripts/certificates/install_ca.sh +18 -0
- ctao_bdms_clients-0.0.0a0/chart/templates/_helpers.tpl +63 -0
- ctao_bdms_clients-0.0.0a0/chart/templates/bootstrap_jobs.yaml +171 -0
- ctao_bdms_clients-0.0.0a0/chart/templates/configmap.yaml +100 -0
- ctao_bdms_clients-0.0.0a0/chart/templates/prepuller.yaml +24 -0
- ctao_bdms_clients-0.0.0a0/chart/templates/test_storages.yaml +128 -0
- ctao_bdms_clients-0.0.0a0/chart/templates/tests/test_jobs.yaml +127 -0
- ctao_bdms_clients-0.0.0a0/chart/values.yaml +330 -0
- ctao_bdms_clients-0.0.0a0/docs/Makefile +23 -0
- ctao_bdms_clients-0.0.0a0/docs/changelog.rst +6 -0
- ctao_bdms_clients-0.0.0a0/docs/changes/template.rst +43 -0
- ctao_bdms_clients-0.0.0a0/docs/chart.rst +6 -0
- ctao_bdms_clients-0.0.0a0/docs/conf.py +74 -0
- ctao_bdms_clients-0.0.0a0/docs/getting_started.rst +43 -0
- ctao_bdms_clients-0.0.0a0/docs/index.rst +23 -0
- ctao_bdms_clients-0.0.0a0/docs/reference.rst +7 -0
- ctao_bdms_clients-0.0.0a0/docs/server_setup/bdms_repos_interaction.rst +59 -0
- ctao_bdms_clients-0.0.0a0/docs/server_setup/certificates.rst +20 -0
- ctao_bdms_clients-0.0.0a0/docs/server_setup/fts.rst +35 -0
- ctao_bdms_clients-0.0.0a0/docs/server_setup/index.rst +12 -0
- ctao_bdms_clients-0.0.0a0/docs/server_setup/rucio_configuration.rst +88 -0
- ctao_bdms_clients-0.0.0a0/docs/server_setup/storage_elements.rst +48 -0
- ctao_bdms_clients-0.0.0a0/pyproject.toml +175 -0
- ctao_bdms_clients-0.0.0a0/report/inspection.tex +0 -0
- ctao_bdms_clients-0.0.0a0/report/performance_verification.tex +0 -0
- ctao_bdms_clients-0.0.0a0/report/preamble.tex +0 -0
- ctao_bdms_clients-0.0.0a0/setup.cfg +4 -0
- ctao_bdms_clients-0.0.0a0/sonar-project.properties +11 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/__init__.py +7 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/_dev_version/__init__.py +9 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/_version.py +16 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/tests/__init__.py +0 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/tests/conftest.py +53 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_basic_rucio_functionality.py +143 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_dpps_rel_0_0.py +90 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_file_replicas.py +72 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/tests/test_metadata.py +110 -0
- ctao_bdms_clients-0.0.0a0/src/bdms/version.py +23 -0
- ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/PKG-INFO +35 -0
- ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/SOURCES.txt +62 -0
- ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/dependency_links.txt +1 -0
- ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/entry_points.txt +2 -0
- ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/requires.txt +22 -0
- ctao_bdms_clients-0.0.0a0/src/ctao_bdms_clients.egg-info/top_level.txt +1 -0
@@ -0,0 +1 @@
|
|
1
|
+
anull
|
@@ -0,0 +1,143 @@
|
|
1
|
+
# Files downloaded by the dpps-aiv-toolkit
|
2
|
+
Chart.lock
|
3
|
+
helm
|
4
|
+
helm.tar.gz
|
5
|
+
kind
|
6
|
+
kubectl
|
7
|
+
linux-amd64
|
8
|
+
# generated file
|
9
|
+
chart/**/*.tgz
|
10
|
+
report.xml
|
11
|
+
# setuptools_scm generated version file
|
12
|
+
_version.py
|
13
|
+
|
14
|
+
# Byte-compiled / optimized / DLL files
|
15
|
+
__pycache__/
|
16
|
+
*.py[cod]
|
17
|
+
*$py.class
|
18
|
+
|
19
|
+
# C extensions
|
20
|
+
*.so
|
21
|
+
|
22
|
+
# Distribution / packaging
|
23
|
+
.Python
|
24
|
+
build/
|
25
|
+
develop-eggs/
|
26
|
+
dist/
|
27
|
+
downloads/
|
28
|
+
eggs/
|
29
|
+
.eggs/
|
30
|
+
lib/
|
31
|
+
lib64/
|
32
|
+
parts/
|
33
|
+
sdist/
|
34
|
+
var/
|
35
|
+
wheels/
|
36
|
+
pip-wheel-metadata/
|
37
|
+
share/python-wheels/
|
38
|
+
*.egg-info/
|
39
|
+
.installed.cfg
|
40
|
+
*.egg
|
41
|
+
MANIFEST
|
42
|
+
|
43
|
+
# PyInstaller
|
44
|
+
# Usually these files are written by a python script from a template
|
45
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
46
|
+
*.manifest
|
47
|
+
*.spec
|
48
|
+
|
49
|
+
# Installer logs
|
50
|
+
pip-log.txt
|
51
|
+
pip-delete-this-directory.txt
|
52
|
+
|
53
|
+
# Unit test / coverage reports
|
54
|
+
htmlcov/
|
55
|
+
.tox/
|
56
|
+
.nox/
|
57
|
+
.coverage
|
58
|
+
.coverage.*
|
59
|
+
.cache
|
60
|
+
nosetests.xml
|
61
|
+
coverage.xml
|
62
|
+
*.cover
|
63
|
+
*.py,cover
|
64
|
+
.hypothesis/
|
65
|
+
.pytest_cache/
|
66
|
+
|
67
|
+
# Translations
|
68
|
+
*.mo
|
69
|
+
*.pot
|
70
|
+
|
71
|
+
# Django stuff:
|
72
|
+
*.log
|
73
|
+
local_settings.py
|
74
|
+
db.sqlite3
|
75
|
+
db.sqlite3-journal
|
76
|
+
|
77
|
+
# Flask stuff:
|
78
|
+
instance/
|
79
|
+
.webassets-cache
|
80
|
+
|
81
|
+
# Scrapy stuff:
|
82
|
+
.scrapy
|
83
|
+
|
84
|
+
# Sphinx documentation
|
85
|
+
docs/_build/
|
86
|
+
docs/api
|
87
|
+
|
88
|
+
# PyBuilder
|
89
|
+
target/
|
90
|
+
|
91
|
+
# Jupyter Notebook
|
92
|
+
.ipynb_checkpoints
|
93
|
+
|
94
|
+
# IPython
|
95
|
+
profile_default/
|
96
|
+
ipython_config.py
|
97
|
+
|
98
|
+
# pyenv
|
99
|
+
.python-version
|
100
|
+
|
101
|
+
# pipenv
|
102
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
103
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
104
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
105
|
+
# install all needed dependencies.
|
106
|
+
#Pipfile.lock
|
107
|
+
|
108
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
109
|
+
__pypackages__/
|
110
|
+
|
111
|
+
# Celery stuff
|
112
|
+
celerybeat-schedule
|
113
|
+
celerybeat.pid
|
114
|
+
|
115
|
+
# SageMath parsed files
|
116
|
+
*.sage.py
|
117
|
+
|
118
|
+
# Environments
|
119
|
+
.env
|
120
|
+
.venv
|
121
|
+
env/
|
122
|
+
venv/
|
123
|
+
ENV/
|
124
|
+
env.bak/
|
125
|
+
venv.bak/
|
126
|
+
|
127
|
+
# Spyder project settings
|
128
|
+
.spyderproject
|
129
|
+
.spyproject
|
130
|
+
|
131
|
+
# Rope project settings
|
132
|
+
.ropeproject
|
133
|
+
|
134
|
+
# mkdocs documentation
|
135
|
+
/site
|
136
|
+
|
137
|
+
# mypy
|
138
|
+
.mypy_cache/
|
139
|
+
.dmypy.json
|
140
|
+
dmypy.json
|
141
|
+
|
142
|
+
# Pyre type checker
|
143
|
+
.pyre/
|
@@ -0,0 +1,37 @@
|
|
1
|
+
include:
|
2
|
+
- project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
|
3
|
+
ref: 705e92d15a41d000f458dcb11d662f83d5718aaf
|
4
|
+
file: 'ci-functions.yml'
|
5
|
+
- "aiv-config.yml"
|
6
|
+
|
7
|
+
|
8
|
+
variables:
|
9
|
+
CHART_LOCATION: chart
|
10
|
+
CHART_NAME: bdms
|
11
|
+
CHART_EXTRA_VALUES: "--set client_image_tag=${DOCKER_TAG}"
|
12
|
+
DPPS_AIV_TOOLKIT_DIR: dpps-aiv-toolkit
|
13
|
+
DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
|
14
|
+
RUCIO_VERSION: "35.4.1"
|
15
|
+
RUCIO_TAG: "release-${RUCIO_VERSION}"
|
16
|
+
|
17
|
+
stages:
|
18
|
+
- lint
|
19
|
+
- build
|
20
|
+
- sign
|
21
|
+
- tests
|
22
|
+
- sonarqube
|
23
|
+
- publish
|
24
|
+
- report
|
25
|
+
|
26
|
+
build:
|
27
|
+
variables:
|
28
|
+
CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/bdms-client:${DOCKER_TAG}'
|
29
|
+
KANIKO_EXTRA_ARGS: --build-arg RUCIO_TAG=${RUCIO_TAG}
|
30
|
+
|
31
|
+
hadolint:
|
32
|
+
rules:
|
33
|
+
- when: never
|
34
|
+
|
35
|
+
sign:
|
36
|
+
rules:
|
37
|
+
- when: never
|
@@ -0,0 +1,9 @@
|
|
1
|
+
[submodule "dpps-aiv-toolkit"]
|
2
|
+
path = dpps-aiv-toolkit
|
3
|
+
url = ../../aiv/dpps-aiv-toolkit.git
|
4
|
+
[submodule "dpps-release-plan"]
|
5
|
+
path = dpps-release-plan
|
6
|
+
url = ../../dpps-project-management/dpps-release-plan.git
|
7
|
+
[submodule "requirements-traceability"]
|
8
|
+
path = requirements-traceability
|
9
|
+
url = ../../dpps-project-management/requirements-traceability.git
|
@@ -0,0 +1,61 @@
|
|
1
|
+
repos:
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
3
|
+
rev: v5.0.0 # Use the ref you want to point at
|
4
|
+
hooks:
|
5
|
+
- id: trailing-whitespace
|
6
|
+
exclude: '(.*\.fits(\.fz)?)|(.*.patch.*)$'
|
7
|
+
- id: check-added-large-files
|
8
|
+
- id: check-case-conflict
|
9
|
+
- id: check-merge-conflict
|
10
|
+
- id: end-of-file-fixer
|
11
|
+
exclude: '(.*\.fits(\.fz)?)|(.*.patch.*)|(chart\/.*README.md)$'
|
12
|
+
|
13
|
+
- repo: https://github.com/codespell-project/codespell
|
14
|
+
rev: v2.3.0
|
15
|
+
hooks:
|
16
|
+
- id: codespell
|
17
|
+
additional_dependencies:
|
18
|
+
- tomli
|
19
|
+
|
20
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
21
|
+
rev: v0.8.0
|
22
|
+
hooks:
|
23
|
+
- id: ruff
|
24
|
+
args: [ --fix, --show-fixes ]
|
25
|
+
exclude: chart/charts/rucio-daemons/patches/request_patched.py
|
26
|
+
- id: ruff-format
|
27
|
+
exclude: chart/charts/rucio-daemons/patches/request_patched.py
|
28
|
+
exclude: '.*.patch.*'
|
29
|
+
- id: ruff-format
|
30
|
+
exclude: '.*.patch.*'
|
31
|
+
|
32
|
+
- repo: https://github.com/scientific-python/cookie
|
33
|
+
rev: "2024.08.19"
|
34
|
+
hooks:
|
35
|
+
- id: sp-repo-review
|
36
|
+
|
37
|
+
- repo: local
|
38
|
+
hooks:
|
39
|
+
- id: CI yaml check
|
40
|
+
args: [ dpps-aiv-toolkit ]
|
41
|
+
name: CI yaml check
|
42
|
+
description: CI yaml check
|
43
|
+
language: script
|
44
|
+
entry: dpps-aiv-toolkit/cicheck.sh
|
45
|
+
pass_filenames: true
|
46
|
+
files: ^.gitlab-ci.yml$
|
47
|
+
always_run: true
|
48
|
+
|
49
|
+
- repo: https://github.com/norwoodj/helm-docs
|
50
|
+
rev: "v1.14.2"
|
51
|
+
hooks:
|
52
|
+
- id: helm-docs-built
|
53
|
+
args:
|
54
|
+
- "--chart-search-root=chart"
|
55
|
+
|
56
|
+
# - id: helm-lint
|
57
|
+
# name: helm-lint
|
58
|
+
# language: docker_image
|
59
|
+
# entry: alpine/helm:3.11.1 lint chart/ # --set
|
60
|
+
# always_run: true
|
61
|
+
# pass_filenames: false
|
@@ -0,0 +1,15 @@
|
|
1
|
+
BDMS v0.1.0rc1 (2025-02-12)
|
2
|
+
---------------------------
|
3
|
+
|
4
|
+
First release candidate for the Bulk Data Management System (BDMS).
|
5
|
+
|
6
|
+
* Deployment of Rucio 35.4 using helm
|
7
|
+
|
8
|
+
|
9
|
+
|
10
|
+
Maintenance
|
11
|
+
~~~~~~~~~~~
|
12
|
+
|
13
|
+
- Write and publish BDMS documentation for DPPS Release 0. [`!47 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/47>`__]
|
14
|
+
|
15
|
+
- Setup ``towncrier`` for changelog generation. [`!49 <https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/merge_requests/49>`__]
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# rucio version used as base for the final image
|
2
|
+
ARG RUCIO_TAG=release-35.4.1
|
3
|
+
|
4
|
+
FROM python:3.9 AS builder
|
5
|
+
|
6
|
+
COPY pyproject.toml MANIFEST.in /tmp/bdms/
|
7
|
+
COPY .git /tmp/bdms/.git/
|
8
|
+
COPY src /tmp/bdms/src/
|
9
|
+
|
10
|
+
RUN python -m pip install build \
|
11
|
+
&& python -m build /tmp/bdms --wheel -o /tmp/dist
|
12
|
+
|
13
|
+
# second stage, copy and install wheel
|
14
|
+
# We are using the official python 3.11 image
|
15
|
+
# as base image in the slim variant to reduce image size.
|
16
|
+
FROM rucio/rucio-clients:${RUCIO_TAG}
|
17
|
+
|
18
|
+
ARG RUCIO_TAG
|
19
|
+
|
20
|
+
# server and daemons use root, clients use "user", switch to root, install, then back
|
21
|
+
USER root
|
22
|
+
COPY --from=builder /tmp/dist/ /tmp/dist/
|
23
|
+
RUN dnf install -y --setopt=install_weak_deps=False git \
|
24
|
+
&& python3 -m pip install --no-cache-dir /tmp/dist/ctao_bdms* \
|
25
|
+
&& dnf autoremove \
|
26
|
+
&& dnf clean all
|
27
|
+
|
28
|
+
USER user
|
29
|
+
|
30
|
+
ENV RUCIO_POLICY_PACKAGE=bdms_rucio_policy
|
@@ -0,0 +1,29 @@
|
|
1
|
+
BSD 3-Clause License
|
2
|
+
|
3
|
+
Copyright (c) 2022, Cherenkov Telescope Array Observatory Consortium
|
4
|
+
All rights reserved.
|
5
|
+
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
8
|
+
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
10
|
+
list of conditions and the following disclaimer.
|
11
|
+
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
14
|
+
and/or other materials provided with the distribution.
|
15
|
+
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
17
|
+
contributors may be used to endorse or promote products derived from
|
18
|
+
this software without specific prior written permission.
|
19
|
+
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
@@ -0,0 +1,11 @@
|
|
1
|
+
###
|
2
|
+
# TODO: duplicate in config?
|
3
|
+
export CHART_NAME=bdms
|
4
|
+
export CHART_LOCATION=chart
|
5
|
+
include dpps-aiv-toolkit/Makefile
|
6
|
+
|
7
|
+
|
8
|
+
# TODO: move this to kit
|
9
|
+
export TEST_ARTIFACTS_PATH ?= $(PWD)
|
10
|
+
export TEST_REPORT_CONFIG ?= $(PWD)/aiv-config.yml
|
11
|
+
export TEX_CONTENT_PATH ?= $(PWD)/report
|
@@ -0,0 +1,35 @@
|
|
1
|
+
Metadata-Version: 2.2
|
2
|
+
Name: ctao-bdms-clients
|
3
|
+
Version: 0.0.0a0
|
4
|
+
Summary: Client module for the CTAO DPPS Bulk Data Management System
|
5
|
+
Author-email: Georgios Zacharis <georgios.zacharis@inaf.it>, Stefano Gallozzi <Stefano.gallozzi@inaf.it>, Michele Mastropietro <michele.mastropietro@inaf.it>, Syed Anwar Ul Hasan <syedanwarul.hasan@cta-consortium.org>, Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>, Volodymyr Savchenko <Volodymyr.Savchenko@epfl.ch>
|
6
|
+
License: BSD-3-Clause
|
7
|
+
Project-URL: repository, https://github.com/cta-observatory/...
|
8
|
+
Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/documentation/...
|
9
|
+
Requires-Python: >=3.9
|
10
|
+
Description-Content-Type: text/markdown
|
11
|
+
License-File: LICENSE
|
12
|
+
Requires-Dist: rucio-clients~=35.4.1
|
13
|
+
Requires-Dist: ctao-bdms-rucio-policy==0.1.0
|
14
|
+
Provides-Extra: test
|
15
|
+
Requires-Dist: pytest; extra == "test"
|
16
|
+
Requires-Dist: pytest-cov; extra == "test"
|
17
|
+
Requires-Dist: pytest-mock; extra == "test"
|
18
|
+
Requires-Dist: pytest-requirements; extra == "test"
|
19
|
+
Provides-Extra: doc
|
20
|
+
Requires-Dist: sphinx; extra == "doc"
|
21
|
+
Requires-Dist: numpydoc; extra == "doc"
|
22
|
+
Requires-Dist: ctao-sphinx-theme; extra == "doc"
|
23
|
+
Requires-Dist: myst-parser; extra == "doc"
|
24
|
+
Requires-Dist: sphinx-changelog; extra == "doc"
|
25
|
+
Provides-Extra: dev
|
26
|
+
Requires-Dist: setuptools_scm; extra == "dev"
|
27
|
+
Requires-Dist: sphinx-autobuild; extra == "dev"
|
28
|
+
Provides-Extra: all
|
29
|
+
Requires-Dist: bdms[dev,doc,test]; extra == "all"
|
30
|
+
|
31
|
+
# Bulk Data Management System
|
32
|
+
|
33
|
+
[](https://gitlab.cta-observatory.org/dpps/bdms/bdms/-/pipelines/main/latest)
|
34
|
+
[](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/jobs/artifacts/main/file/test_report.pdf?job=build-test-report)
|
35
|
+
[](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms/)
|
@@ -0,0 +1,5 @@
|
|
1
|
+
# Bulk Data Management System
|
2
|
+
|
3
|
+
[](https://gitlab.cta-observatory.org/dpps/bdms/bdms/-/pipelines/main/latest)
|
4
|
+
[](https://gitlab.cta-observatory.org/cta-computing/dpps/bdms/bdms/-/jobs/artifacts/main/file/test_report.pdf?job=build-test-report)
|
5
|
+
[](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms/)
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# This is the configuration used both in CI and local Makefile
|
2
|
+
|
3
|
+
variables:
|
4
|
+
# Comma-separated list of "UC Groups" to select UCs and Requirements, e.g. "BDMS,AIV"
|
5
|
+
DPPS_UC_GROUPS: BDMS
|
6
|
+
DPPS_RELEASE: v0.0
|
7
|
+
|
8
|
+
# these paths are relative to TEST_ARTIFACTS_PATH
|
9
|
+
REPORT_XML: report.xml
|
10
|
+
RELEASE_PLAN_FN: dpps-release-plan/release_development_document/dpps-release-plan.tex
|
11
|
+
TRACEABILITY_CSV_FN: requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
|
12
|
+
|
13
|
+
APPLICATION_NAME: BDMS
|
14
|
+
APPLICATION_AUTHOR: BDMS Team
|
15
|
+
APPLICATION_AUTHOR_ORGANIZATION: CTAO
|
16
|
+
APPLICATION_VERSION: 0.0.0
|
17
|
+
|
18
|
+
EXTRA_CONFIG_FILES: aiv-config-dependencies.yml
|
@@ -0,0 +1,39 @@
|
|
1
|
+
apiVersion: v2
|
2
|
+
name: bdms
|
3
|
+
version: 0.1.0
|
4
|
+
appVersion: 0.1.0
|
5
|
+
description: A Helm chart for the bdms project
|
6
|
+
type: application
|
7
|
+
keywords:
|
8
|
+
- bdms
|
9
|
+
- rucio
|
10
|
+
- monitoring
|
11
|
+
- postgresql
|
12
|
+
|
13
|
+
maintainers:
|
14
|
+
- name: The BDMS Authors
|
15
|
+
email:
|
16
|
+
|
17
|
+
dependencies:
|
18
|
+
- name: postgresql
|
19
|
+
condition: postgresql.enabled
|
20
|
+
version: 15.5.10
|
21
|
+
repository: oci://registry-1.docker.io/bitnamicharts
|
22
|
+
|
23
|
+
- name: rucio-server
|
24
|
+
version: 35.0.0
|
25
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
26
|
+
|
27
|
+
- name: rucio-daemons
|
28
|
+
version: 35.0.0
|
29
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
30
|
+
|
31
|
+
- name: cert-generator-grid
|
32
|
+
condition: cert-generator-grid.enabled
|
33
|
+
version: 0.0.0-75a4994-75a4994c
|
34
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
35
|
+
|
36
|
+
- name: fts
|
37
|
+
condition: fts.enabled
|
38
|
+
version: 0.0.0-c435e63-c435e636
|
39
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
@@ -0,0 +1,15 @@
|
|
1
|
+
install:
|
2
|
+
helm upgrade --install --wait bdms .
|
3
|
+
|
4
|
+
test:
|
5
|
+
helm test bdms
|
6
|
+
|
7
|
+
publish:
|
8
|
+
helm package .
|
9
|
+
helm push bdms-0.1.0.tgz oci://harbor.cta-observatory.org/dpps
|
10
|
+
|
11
|
+
reset:
|
12
|
+
helm delete bdms || true
|
13
|
+
kubectl delete secrets bdms-rucio-server bdms-rucio-server-tls bdms-server-cafile bdms-server-hostcert bdms-server-hostkey dppsuser-certkey || true
|
14
|
+
kubectl delete job generate-certificates configure-test-rucio test-rucio || true
|
15
|
+
kubectl delete pvc data-bdms-postgresql-0 || true
|
@@ -0,0 +1,140 @@
|
|
1
|
+
# bdms
|
2
|
+
|
3
|
+
  
|
4
|
+
|
5
|
+
A Helm chart for the bdms project
|
6
|
+
|
7
|
+
## Maintainers
|
8
|
+
|
9
|
+
| Name | Email | Url |
|
10
|
+
| ---- | ------ | --- |
|
11
|
+
| The BDMS Authors | | |
|
12
|
+
|
13
|
+
## Requirements
|
14
|
+
|
15
|
+
| Repository | Name | Version |
|
16
|
+
|------------|------|---------|
|
17
|
+
| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | 0.0.0-75a4994-75a4994c |
|
18
|
+
| oci://harbor.cta-observatory.org/dpps | fts | 0.0.0-c435e63-c435e636 |
|
19
|
+
| oci://harbor.cta-observatory.org/dpps | rucio-daemons | 35.0.0 |
|
20
|
+
| oci://harbor.cta-observatory.org/dpps | rucio-server | 35.0.0 |
|
21
|
+
| oci://registry-1.docker.io/bitnamicharts | postgresql | 15.5.10 |
|
22
|
+
|
23
|
+
## Values
|
24
|
+
|
25
|
+
| Key | Type | Default | Description |
|
26
|
+
|-----|------|---------|-------------|
|
27
|
+
| auth.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. It is used by clients and services to authenticate with Rucio |
|
28
|
+
| auth.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
|
29
|
+
| auth.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
|
30
|
+
| auth.certificate.existingSecret.key | string | `"tls.key"` | The key inside the kubernetes secret that stores the private key |
|
31
|
+
| auth.certificate.existingSecret.secretName | string | `"rucio-server.local"` | The name of the kubernetes secret containing the TLS certificate and key |
|
32
|
+
| auth.certificate.letsencrypt.email | string | `""` | Email address for Let's encrypt registration and renewal reminders |
|
33
|
+
| auth.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
|
34
|
+
| bootstrap.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image for bootstrapping Rucio (initialization, configuration) with the CTAO Rucio policy package installed |
|
35
|
+
| bootstrap.image.tag | string | `"35.4.1-v0.1.0"` | The specific image tag to use for the bootstrap container |
|
36
|
+
| cert-generator-grid.enabled | bool | `true` | |
|
37
|
+
| cert-generator-grid.generatePreHooks | bool | `true` | |
|
38
|
+
| client_image_tag | string | `"6cbd744c"` | |
|
39
|
+
| configure_test_setup | bool | `true` | This will configure the rucio server with the storages |
|
40
|
+
| database | object | `{"default":"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"}` | Databases Credentials used by Rucio to access the database. If postgresql subchart is deployed, these credentials should match those in postgresql.global.postgresql.auth. If postgresql subchart is not deployed, an external database must be provided |
|
41
|
+
| database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The Rucio database connection URI |
|
42
|
+
| dev.mount_repo | bool | `true` | |
|
43
|
+
| dev.run_tests | bool | `true` | |
|
44
|
+
| dev.sleep | bool | `false` | sleep after test to allow interactive development |
|
45
|
+
| fts.enabled | bool | `true` | Specifies the configuration for FTS test step (FTS server, FTS database, and ActiveMQ broker containers). Enables or disables the deployment of a FTS instance for testing. This is set to 'False' if an external FTS is used |
|
46
|
+
| fts.ftsdb_password | string | `"SDP2RQkbJE2f+ohUb2nUu6Ae10BpQH0VD70CsIQcDtM"` | Defines the password for the FTS database user |
|
47
|
+
| fts.ftsdb_root_password | string | `"iB7dMiIybdoaozWZMkvRo0eg9HbQzG9+5up50zUDjE4"` | Defines the root password for the FTS database |
|
48
|
+
| fts.image.pullPolicy | string | `"Always"` | |
|
49
|
+
| fts.image.repository | string | `"harbor.cta-observatory.org/proxy_cache/rucio/fts"` | The container image repository for the FTS deployment |
|
50
|
+
| fts.image.tag | string | `"35.4.1"` | Defines the specific version of the FTS image to use |
|
51
|
+
| fts.messaging.broker | string | `"localhost:61613"` | |
|
52
|
+
| fts.messaging.password | string | `"topsecret"` | |
|
53
|
+
| fts.messaging.use_broker_credentials | string | `"true"` | |
|
54
|
+
| fts.messaging.username | string | `"fts"` | |
|
55
|
+
| postgresql.enabled | bool | `true` | Configuration of built-in postgresql database. If 'enabled: true', a postgresql instance will be deployed, otherwise, an external database must be provided in database.default value |
|
56
|
+
| postgresql.global.postgresql.auth.database | string | `"rucio"` | The name of the database to be created and used by Rucio |
|
57
|
+
| postgresql.global.postgresql.auth.password | string | `"XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM"` | The password for the database user |
|
58
|
+
| postgresql.global.postgresql.auth.username | string | `"rucio"` | The database username for authentication |
|
59
|
+
| prepuller_enabled | bool | `true` | Starts containers with the same image as the one used in the deployment before all volumes are available. Saves time in the first deployment |
|
60
|
+
| rethinkdb.enabled | bool | `false` | |
|
61
|
+
| rethinkdb.storageClassName | string | `nil` | |
|
62
|
+
| rethinkdb.storageSize | string | `"1Gi"` | |
|
63
|
+
| rucio-daemons.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | Specifies the connection URI for the Rucio database, these settings will be written to 'rucio.cfg' |
|
64
|
+
| rucio-daemons.config.messaging-fts3.brokers | string | `"fts-activemq"` | Specifies the message broker used for FTS messaging |
|
65
|
+
| rucio-daemons.config.messaging-fts3.destination | string | `"/topic/transfer.fts_monitoring_complete"` | Specifies the message broker queue path where FTS sends transfer status updates. This is the place where Rucio listens for completed transfer notifications |
|
66
|
+
| rucio-daemons.config.messaging-fts3.nonssl_port | int | `61613` | Specifies the non-SSL port |
|
67
|
+
| rucio-daemons.config.messaging-fts3.password | string | `"topsecret"` | Specifies the authentication credential (password) for connecting to the message broker |
|
68
|
+
| rucio-daemons.config.messaging-fts3.port | int | `61613` | Defines the port used for the broker |
|
69
|
+
| rucio-daemons.config.messaging-fts3.use_ssl | bool | `false` | Determines whether to use SSL for message broker connections. If true, valid certificates are required for securing the connection |
|
70
|
+
| rucio-daemons.config.messaging-fts3.username | string | `"fts"` | Specifies the authentication credential (username) for connecting to the message broker |
|
71
|
+
| rucio-daemons.config.policy.permission | string | `"ctao"` | Defines the policy permission model for Rucio for determining how authorization and access controls are applied, its value should be taken from the installed Rucio policy package |
|
72
|
+
| rucio-daemons.conveyorFinisher.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
|
73
|
+
| rucio-daemons.conveyorFinisher.resources.limits.cpu | string | `"3000m"` | |
|
74
|
+
| rucio-daemons.conveyorFinisher.resources.limits.memory | string | `"4Gi"` | |
|
75
|
+
| rucio-daemons.conveyorFinisher.resources.requests.cpu | string | `"700m"` | |
|
76
|
+
| rucio-daemons.conveyorFinisher.resources.requests.memory | string | `"200Mi"` | |
|
77
|
+
| rucio-daemons.conveyorFinisher.sleepTime | int | `5` | Defines how often (in seconds) the daemon processes finished transfers |
|
78
|
+
| rucio-daemons.conveyorFinisherCount | int | `1` | Marks completed transfers and updates metadata |
|
79
|
+
| rucio-daemons.conveyorPoller.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
|
80
|
+
| rucio-daemons.conveyorPoller.olderThan | int | `600` | Filters transfers that are older than the specified time (in seconds) before polling |
|
81
|
+
| rucio-daemons.conveyorPoller.resources.limits.cpu | string | `"3000m"` | |
|
82
|
+
| rucio-daemons.conveyorPoller.resources.limits.memory | string | `"4Gi"` | |
|
83
|
+
| rucio-daemons.conveyorPoller.resources.requests.cpu | string | `"700m"` | |
|
84
|
+
| rucio-daemons.conveyorPoller.resources.requests.memory | string | `"200Mi"` | |
|
85
|
+
| rucio-daemons.conveyorPoller.sleepTime | int | `60` | Defines how often (in seconds) the daemon polls for transfer status updates |
|
86
|
+
| rucio-daemons.conveyorPollerCount | int | `1` | Polls FTS to check the status of ongoing transfers |
|
87
|
+
| rucio-daemons.conveyorReceiverCount | int | `1` | Listens to messages from ActiveMQ, which FTS uses to publish transfer status updates. This ensures Rucio is notified of completed or failed transfers in real time |
|
88
|
+
| rucio-daemons.conveyorTransferSubmitter.activities | string | `"'User Subscriptions'"` | Specifies which Rucio activities to be handled. Some of the activities for data movements are 'User Subscriptions' and 'Production Transfers' |
|
89
|
+
| rucio-daemons.conveyorTransferSubmitter.archiveTimeout | string | `""` | Sets the timeout if required for archiving completed transfers |
|
90
|
+
| rucio-daemons.conveyorTransferSubmitter.resources.limits.cpu | string | `"3000m"` | |
|
91
|
+
| rucio-daemons.conveyorTransferSubmitter.resources.limits.memory | string | `"4Gi"` | |
|
92
|
+
| rucio-daemons.conveyorTransferSubmitter.resources.requests.cpu | string | `"700m"` | |
|
93
|
+
| rucio-daemons.conveyorTransferSubmitter.resources.requests.memory | string | `"200Mi"` | |
|
94
|
+
| rucio-daemons.conveyorTransferSubmitter.sleepTime | int | `5` | Defines the interval (in seconds) the daemon waits before checking for new transfers |
|
95
|
+
| rucio-daemons.conveyorTransferSubmitterCount | int | `1` | Number of container instances to deploy for each Rucio daemon, this daemon submits new transfer requests to the FTS |
|
96
|
+
| rucio-daemons.image.pullPolicy | string | `"Always"` | It defines when kubernetes should pull the container image, the options available are: Always, IfNotPresent, and Never |
|
97
|
+
| rucio-daemons.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-daemons"` | Specifies the container image repository for Rucio daemons |
|
98
|
+
| rucio-daemons.image.tag | string | `"35.4.1-v0.1.0"` | Specific image tag to use for deployment |
|
99
|
+
| rucio-daemons.judgeEvaluatorCount | int | `1` | Evaluates Rucio replication rules and triggers transfers |
|
100
|
+
| rucio-daemons.useDeprecatedImplicitSecrets | bool | `true` | Enables the use of deprecated implicit secrets for authentication |
|
101
|
+
| rucio-server.authRucioHost | string | `"rucio-server.local"` | The hostname of the Rucio authentication server. |
|
102
|
+
| rucio-server.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The database connection URI for Rucio |
|
103
|
+
| rucio-server.config.persistence.accessMode | string | `"ReadWriteOnce"` | Defines how storage can be accessed, the options available are: ReadWriteOnce, ReadOnlyMany, ReadWriteMany |
|
104
|
+
| rucio-server.config.persistence.enabled | bool | `true` | Enables persistent storage for Rucio server, setting it to false means using temporary storage ('ephemeral' in the context of kubernetes) that gets wiped out and lost when the container is stopped or restarted |
|
105
|
+
| rucio-server.config.persistence.size | string | `"1Gi"` | Defines the size of persistent volume claim (PVC) or the amount of the storage capacity provisioned to the Rucio server |
|
106
|
+
| rucio-server.config.persistence.storageClass | string | `""` | Specifies the kubernetes storage class for persistent volume, default storage class is used when its value is left empty |
|
107
|
+
| rucio-server.ftsRenewal.enabled | bool | `false` | Enables automatic renewal of FTS credentials using X.509 certificates and proxy |
|
108
|
+
| rucio-server.httpd_config.encoded_slashes | string | `"True"` | Allows for custom LFNs with slashes in request URLs so that Rucio server (Apache) can decode and handle such requests properly |
|
109
|
+
| rucio-server.httpd_config.grid_site_enabled | string | `"True"` | Enables Rucio server to support and interact with grid middleware (storages) for X509 authentication with proxies |
|
110
|
+
| rucio-server.image.pullPolicy | string | `"Always"` | It defines when kubernetes should pull the container image, the options available are: Always, IfNotPresent, and Never |
|
111
|
+
| rucio-server.image.repository | string | `"harbor.cta-observatory.org/dpps/bdms-rucio-server"` | The container image repository for Rucio server with the CTAO Rucio policy package installed |
|
112
|
+
| rucio-server.image.tag | string | `"35.4.1-v0.1.0"` | The specific image tag to deploy |
|
113
|
+
| rucio-server.ingress.enabled | bool | `true` | Enables an ingress resource (controller) for exposing the Rucio server externally to allow clients connect to the Rucio server. It needs one of the ingress controllers (NGINX, Traefik) to be installed |
|
114
|
+
| rucio-server.ingress.hosts | list | `["rucio-server-manual-tc.local"]` | Defines the hostname to be used to access the Rucio server. It should match DNS configuration and TLS certificates |
|
115
|
+
| rucio-server.replicaCount | int | `1` | Number of replicas of the Rucio server to deploy. We can increase it to meet higher availability goals |
|
116
|
+
| rucio-server.service.name | string | `"https"` | The name of the service port |
|
117
|
+
| rucio-server.service.port | int | `443` | The port exposed by the kubernetes service, making the Rucio server accessible within the cluster |
|
118
|
+
| rucio-server.service.protocol | string | `"TCP"` | The network protocol used for HTTPS based communication |
|
119
|
+
| rucio-server.service.targetPort | int | `443` | The port inside the Rucio server container that listens for incoming traffic |
|
120
|
+
| rucio-server.service.type | string | `"ClusterIP"` | Specifies the kubernetes service type for making the Rucio server accessible within or outside the kubernetes cluster, available options include clusterIP (internal access only, default), NodePort (exposes the service on port across all cluster nodes), and LoadBalancer (Uses an external load balancer) |
|
121
|
+
| rucio-server.useSSL | bool | `true` | Enables the Rucio server to use SSL/TLS for secure communication, requiring valid certificates to be configured |
|
122
|
+
| rucio.password | string | `"secret"` | |
|
123
|
+
| rucio.username | string | `"dpps"` | Specifies the username for Rucio operations as part of Rucio configuration |
|
124
|
+
| rucio.version | string | `"35.4.1"` | The version of Rucio being deployed |
|
125
|
+
| rucio_db.connection | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@bdms-postgresql:5432/rucio"` | The database connection URI for Rucio. It is of the format: `postgresql://<user>:<password>@<host>:<port>/<database>`, this field in use only if 'existingSecret.enabled' is set to 'false', otherwise ignored |
|
126
|
+
| rucio_db.deploy | bool | `true` | If true, deploys a postgresql instance for the Rucio database, otherwise use an external database |
|
127
|
+
| rucio_db.existingSecret.enabled | bool | `false` | If true, the database connection URI is obtained from a kubernetes secret in |
|
128
|
+
| rucio_db.existingSecret.key | string | `"connection"` | The key inside the kubernetes secret that holds the database connection URI |
|
129
|
+
| rucio_db.existingSecret.secretName | string | `"rucio-db"` | The name of the kubernetes secret storing the database connection URI. Its in use only if 'existingSecret.enabled: true' |
|
130
|
+
| safe_to_bootstrap_rucio | bool | `true` | This is a destructive operation, it will delete all data in the database |
|
131
|
+
| server.certificate.existingSecret.cert | string | `"tls.crt"` | The key inside the kubernetes secret that stores the TLS certificate |
|
132
|
+
| server.certificate.existingSecret.enabled | bool | `true` | Use an existing kubernetes (K8s) secret for certificates instead of creating new ones |
|
133
|
+
| server.certificate.existingSecret.key | string | `"tls.key"` | The key inside the kubernetes secret that stores the private key |
|
134
|
+
| server.certificate.existingSecret.secretName | string | `"rucio-server-manual-tc.local"` | The name of the kubernetes secret containing the TLS certificate and key |
|
135
|
+
| server.certificate.letsencrypt.email | string | `""` | |
|
136
|
+
| server.certificate.letsencrypt.enabled | bool | `false` | Enables SSL/TLS certificate provisioning using Let's encrypt |
|
137
|
+
| server.rucioHost | string | `"rucio-server-manual-tc.local"` | The hostname of the Rucio server. It is used by clients and services to communicate with Rucio |
|
138
|
+
| storages | list | `["rucio-storage-1","rucio-storage-2","rucio-storage-3"]` | a list of storage element (RSE) hostnames names, for each RSE, one deployment and service are configured, two configmaps xrdconfig and xrd-entrypoint for those three storages |
|
139
|
+
| suffix_namespace | string | `"default"` | Specifies the Namespace suffix used for managing deployments in kubernetes |
|
140
|
+
|