ctao-dpps-clients 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ctao_dpps_clients-0.0.0/.gitignore +142 -0
- ctao_dpps_clients-0.0.0/.gitlab-ci.yml +60 -0
- ctao_dpps_clients-0.0.0/.gitmodules +9 -0
- ctao_dpps_clients-0.0.0/.pre-commit-config.yaml +47 -0
- ctao_dpps_clients-0.0.0/CHANGES.rst +11 -0
- ctao_dpps_clients-0.0.0/Dockerfile +61 -0
- ctao_dpps_clients-0.0.0/Makefile +11 -0
- ctao_dpps_clients-0.0.0/PKG-INFO +36 -0
- ctao_dpps_clients-0.0.0/README.md +9 -0
- ctao_dpps_clients-0.0.0/aiv-config-dependencies.yml +6 -0
- ctao_dpps_clients-0.0.0/aiv-config.yml +25 -0
- ctao_dpps_clients-0.0.0/chart/Chart.yaml +33 -0
- ctao_dpps_clients-0.0.0/chart/README.md +79 -0
- ctao_dpps_clients-0.0.0/chart/templates/tests/test-pytest.yaml +125 -0
- ctao_dpps_clients-0.0.0/chart/values.yaml +119 -0
- ctao_dpps_clients-0.0.0/docs/Makefile +25 -0
- ctao_dpps_clients-0.0.0/docs/_static/ctao.png +0 -0
- ctao_dpps_clients-0.0.0/docs/_static/ctao_dark.png +0 -0
- ctao_dpps_clients-0.0.0/docs/changelog.rst +6 -0
- ctao_dpps_clients-0.0.0/docs/changes/template.rst +43 -0
- ctao_dpps_clients-0.0.0/docs/chart.rst +6 -0
- ctao_dpps_clients-0.0.0/docs/conf.py +70 -0
- ctao_dpps_clients-0.0.0/docs/developer-guide.rst +42 -0
- ctao_dpps_clients-0.0.0/docs/index.rst +48 -0
- ctao_dpps_clients-0.0.0/pyproject.toml +144 -0
- ctao_dpps_clients-0.0.0/report/inspection.tex +0 -0
- ctao_dpps_clients-0.0.0/report/performance_verification.tex +1 -0
- ctao_dpps_clients-0.0.0/report/preamble.tex +19 -0
- ctao_dpps_clients-0.0.0/setup.cfg +4 -0
- ctao_dpps_clients-0.0.0/sonar-project.properties +12 -0
- ctao_dpps_clients-0.0.0/src/ctao_dpps_clients.egg-info/PKG-INFO +36 -0
- ctao_dpps_clients-0.0.0/src/ctao_dpps_clients.egg-info/SOURCES.txt +39 -0
- ctao_dpps_clients-0.0.0/src/ctao_dpps_clients.egg-info/dependency_links.txt +1 -0
- ctao_dpps_clients-0.0.0/src/ctao_dpps_clients.egg-info/requires.txt +18 -0
- ctao_dpps_clients-0.0.0/src/ctao_dpps_clients.egg-info/top_level.txt +1 -0
- ctao_dpps_clients-0.0.0/src/dpps/__init__.py +23 -0
- ctao_dpps_clients-0.0.0/src/dpps/_version.py +21 -0
- ctao_dpps_clients-0.0.0/test_utils_dirac.py +35 -0
- ctao_dpps_clients-0.0.0/tests/conftest.py +79 -0
- ctao_dpps_clients-0.0.0/tests/test_dirac_rucio_catalog.py +145 -0
- ctao_dpps_clients-0.0.0/tests/test_dpps_package.py +8 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# Files downloaded by the dpps-aiv-toolkit
|
|
2
|
+
helm
|
|
3
|
+
helm.tar.gz
|
|
4
|
+
kind
|
|
5
|
+
kubectl
|
|
6
|
+
linux-amd64
|
|
7
|
+
# generated file
|
|
8
|
+
chart/**/*.tgz
|
|
9
|
+
report.xml
|
|
10
|
+
# setuptools_scm generated version file
|
|
11
|
+
_version.py
|
|
12
|
+
|
|
13
|
+
# Byte-compiled / optimized / DLL files
|
|
14
|
+
__pycache__/
|
|
15
|
+
*.py[cod]
|
|
16
|
+
*$py.class
|
|
17
|
+
|
|
18
|
+
# C extensions
|
|
19
|
+
*.so
|
|
20
|
+
|
|
21
|
+
# Distribution / packaging
|
|
22
|
+
.Python
|
|
23
|
+
build/
|
|
24
|
+
develop-eggs/
|
|
25
|
+
dist/
|
|
26
|
+
downloads/
|
|
27
|
+
eggs/
|
|
28
|
+
.eggs/
|
|
29
|
+
lib/
|
|
30
|
+
lib64/
|
|
31
|
+
parts/
|
|
32
|
+
sdist/
|
|
33
|
+
var/
|
|
34
|
+
wheels/
|
|
35
|
+
pip-wheel-metadata/
|
|
36
|
+
share/python-wheels/
|
|
37
|
+
*.egg-info/
|
|
38
|
+
.installed.cfg
|
|
39
|
+
*.egg
|
|
40
|
+
MANIFEST
|
|
41
|
+
|
|
42
|
+
# PyInstaller
|
|
43
|
+
# Usually these files are written by a python script from a template
|
|
44
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
45
|
+
*.manifest
|
|
46
|
+
*.spec
|
|
47
|
+
|
|
48
|
+
# Installer logs
|
|
49
|
+
pip-log.txt
|
|
50
|
+
pip-delete-this-directory.txt
|
|
51
|
+
|
|
52
|
+
# Unit test / coverage reports
|
|
53
|
+
htmlcov/
|
|
54
|
+
.tox/
|
|
55
|
+
.nox/
|
|
56
|
+
.coverage
|
|
57
|
+
.coverage.*
|
|
58
|
+
.cache
|
|
59
|
+
nosetests.xml
|
|
60
|
+
coverage.xml
|
|
61
|
+
*.cover
|
|
62
|
+
*.py,cover
|
|
63
|
+
.hypothesis/
|
|
64
|
+
.pytest_cache/
|
|
65
|
+
|
|
66
|
+
# Translations
|
|
67
|
+
*.mo
|
|
68
|
+
*.pot
|
|
69
|
+
|
|
70
|
+
# Django stuff:
|
|
71
|
+
*.log
|
|
72
|
+
local_settings.py
|
|
73
|
+
db.sqlite3
|
|
74
|
+
db.sqlite3-journal
|
|
75
|
+
|
|
76
|
+
# Flask stuff:
|
|
77
|
+
instance/
|
|
78
|
+
.webassets-cache
|
|
79
|
+
|
|
80
|
+
# Scrapy stuff:
|
|
81
|
+
.scrapy
|
|
82
|
+
|
|
83
|
+
# Sphinx documentation
|
|
84
|
+
docs/_build/
|
|
85
|
+
docs/api
|
|
86
|
+
|
|
87
|
+
# PyBuilder
|
|
88
|
+
target/
|
|
89
|
+
|
|
90
|
+
# Jupyter Notebook
|
|
91
|
+
.ipynb_checkpoints
|
|
92
|
+
|
|
93
|
+
# IPython
|
|
94
|
+
profile_default/
|
|
95
|
+
ipython_config.py
|
|
96
|
+
|
|
97
|
+
# pyenv
|
|
98
|
+
.python-version
|
|
99
|
+
|
|
100
|
+
# pipenv
|
|
101
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
102
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
103
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
104
|
+
# install all needed dependencies.
|
|
105
|
+
#Pipfile.lock
|
|
106
|
+
|
|
107
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
108
|
+
__pypackages__/
|
|
109
|
+
|
|
110
|
+
# Celery stuff
|
|
111
|
+
celerybeat-schedule
|
|
112
|
+
celerybeat.pid
|
|
113
|
+
|
|
114
|
+
# SageMath parsed files
|
|
115
|
+
*.sage.py
|
|
116
|
+
|
|
117
|
+
# Environments
|
|
118
|
+
.env
|
|
119
|
+
.venv
|
|
120
|
+
env/
|
|
121
|
+
venv/
|
|
122
|
+
ENV/
|
|
123
|
+
env.bak/
|
|
124
|
+
venv.bak/
|
|
125
|
+
|
|
126
|
+
# Spyder project settings
|
|
127
|
+
.spyderproject
|
|
128
|
+
.spyproject
|
|
129
|
+
|
|
130
|
+
# Rope project settings
|
|
131
|
+
.ropeproject
|
|
132
|
+
|
|
133
|
+
# mkdocs documentation
|
|
134
|
+
/site
|
|
135
|
+
|
|
136
|
+
# mypy
|
|
137
|
+
.mypy_cache/
|
|
138
|
+
.dmypy.json
|
|
139
|
+
dmypy.json
|
|
140
|
+
|
|
141
|
+
# Pyre type checker
|
|
142
|
+
.pyre/
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
#
|
|
2
|
+
include:
|
|
3
|
+
- project: 'cta-computing/dpps/aiv/dpps-aiv-toolkit'
|
|
4
|
+
ref: 34ef394f4f707dacea789778c718300fa521b10d
|
|
5
|
+
file: 'ci-functions.yml'
|
|
6
|
+
- "aiv-config.yml"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
variables:
|
|
10
|
+
CHART_LOCATION: chart
|
|
11
|
+
CHART_NAME: dpps
|
|
12
|
+
CHART_EXTRA_VALUES: "--set dev.client_image_tag=${DOCKER_TAG}"
|
|
13
|
+
DPPS_AIV_TOOLKIT_DIR: dpps-aiv-toolkit
|
|
14
|
+
DOCKER_IMAGE_CONTEXT: '${CI_PROJECT_DIR}'
|
|
15
|
+
CONDA_IMAGE: "quay.io/condaforge/miniforge3:24.11.3-0"
|
|
16
|
+
|
|
17
|
+
stages:
|
|
18
|
+
- prepare
|
|
19
|
+
- lint
|
|
20
|
+
- build
|
|
21
|
+
- sign
|
|
22
|
+
- tests
|
|
23
|
+
- sonarqube
|
|
24
|
+
- publish
|
|
25
|
+
- report
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
build:
|
|
29
|
+
variables:
|
|
30
|
+
CI_HARBOR_REGISTRY_IMAGE: '${HARBOR_HOST}/dpps/dpps-clients:${DOCKER_TAG}'
|
|
31
|
+
rules:
|
|
32
|
+
- when: always
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# wms depends on gfal2 via dirac, which is sessentially impossible to install with pip
|
|
36
|
+
build-docs:
|
|
37
|
+
image: "${CONDA_IMAGE}"
|
|
38
|
+
|
|
39
|
+
before_script:
|
|
40
|
+
- source /opt/conda/etc/profile.d/conda.sh
|
|
41
|
+
- mamba create -y -n ci python=3.12 gfal2 m2crypto dirac-grid make
|
|
42
|
+
- conda activate ci
|
|
43
|
+
- python --version
|
|
44
|
+
- pip install .[doc]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
pylint:
|
|
48
|
+
image: "${CONDA_IMAGE}"
|
|
49
|
+
|
|
50
|
+
before_script:
|
|
51
|
+
- apt update && apt install -y --no-install-recommends git
|
|
52
|
+
- source /opt/conda/etc/profile.d/conda.sh
|
|
53
|
+
- mamba create -y -n ci python=3.12 gfal2 m2crypto dirac-grid pylint
|
|
54
|
+
- conda activate ci
|
|
55
|
+
- python --version
|
|
56
|
+
- pip install .[all]
|
|
57
|
+
|
|
58
|
+
sign:
|
|
59
|
+
rules:
|
|
60
|
+
- when: never
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
[submodule "dpps-aiv-toolkit"]
|
|
2
|
+
path = dpps-aiv-toolkit
|
|
3
|
+
url = ../aiv/dpps-aiv-toolkit.git
|
|
4
|
+
[submodule "dpps-release-plan"]
|
|
5
|
+
path = dpps-release-plan
|
|
6
|
+
url = ../dpps-project-management/dpps-release-plan.git
|
|
7
|
+
[submodule "requirements-traceability"]
|
|
8
|
+
path = requirements-traceability
|
|
9
|
+
url = ../dpps-project-management/requirements-traceability.git
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: v4.5.0 # Use the ref you want to point at
|
|
4
|
+
hooks:
|
|
5
|
+
- id: trailing-whitespace
|
|
6
|
+
exclude: ".*\\.fits(\\.fz)?"
|
|
7
|
+
- id: check-added-large-files
|
|
8
|
+
- id: check-case-conflict
|
|
9
|
+
- id: check-merge-conflict
|
|
10
|
+
exclude: ".*\\.fits(\\.fz)?"
|
|
11
|
+
- id: end-of-file-fixer
|
|
12
|
+
exclude: '(.*\.fits(\.fz)?)|(.*.patch.*)|(chart\/.*README.md)$'
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
- repo: https://github.com/codespell-project/codespell
|
|
16
|
+
rev: v2.2.4
|
|
17
|
+
hooks:
|
|
18
|
+
- id: codespell
|
|
19
|
+
additional_dependencies:
|
|
20
|
+
- tomli
|
|
21
|
+
|
|
22
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
23
|
+
rev: v0.2.1
|
|
24
|
+
hooks:
|
|
25
|
+
- id: ruff
|
|
26
|
+
args: [ --fix, --show-fixes ]
|
|
27
|
+
- id: ruff-format
|
|
28
|
+
|
|
29
|
+
- repo: local
|
|
30
|
+
hooks:
|
|
31
|
+
- id: CI yaml check
|
|
32
|
+
args: [ dpps-aiv-toolkit ]
|
|
33
|
+
name: CI yaml check
|
|
34
|
+
description: CI yaml check
|
|
35
|
+
language: script
|
|
36
|
+
entry: dpps-aiv-toolkit/cicheck.sh
|
|
37
|
+
pass_filenames: true
|
|
38
|
+
files: ^.gitlab-ci.yml$
|
|
39
|
+
always_run: true
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
- repo: https://github.com/norwoodj/helm-docs
|
|
43
|
+
rev: "v1.14.2"
|
|
44
|
+
hooks:
|
|
45
|
+
- id: helm-docs-built
|
|
46
|
+
args:
|
|
47
|
+
- "--chart-search-root=chart"
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
DPPS v0.0.0 (2025-02-26)
|
|
2
|
+
------------------------
|
|
3
|
+
|
|
4
|
+
Initial Release of the Data Processing and Preservation System.
|
|
5
|
+
|
|
6
|
+
This release integrates
|
|
7
|
+
|
|
8
|
+
- `Workload Management System v0.1.0 <http://cta-computing.gitlab-pages.cta-observatory.org/dpps/workload/wms/v0.1.0/>`_
|
|
9
|
+
- `Bulk Data Management System v0.1.0 <http://cta-computing.gitlab-pages.cta-observatory.org/dpps/bdms/bdms/v0.1.0/>`_
|
|
10
|
+
|
|
11
|
+
Deployment is facilitated by docker images and helm charts.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
FROM harbor.cta-observatory.org/proxy_cache/python:3.12 AS builder
|
|
2
|
+
|
|
3
|
+
COPY pyproject.toml /tmp/dpps/
|
|
4
|
+
COPY .git /tmp/dpps/.git/
|
|
5
|
+
COPY src /tmp/dpps/src/
|
|
6
|
+
|
|
7
|
+
RUN python -m pip install build \
|
|
8
|
+
&& python -m build /tmp/dpps --wheel -o /tmp/dist
|
|
9
|
+
|
|
10
|
+
FROM harbor.cta-observatory.org/proxy_cache/almalinux:9
|
|
11
|
+
|
|
12
|
+
# TODO: these can be derived from pyproject.toml
|
|
13
|
+
ARG DIRACOS_VERSION="2.50"
|
|
14
|
+
ARG USERID=1000
|
|
15
|
+
ARG GROUPID=1000
|
|
16
|
+
|
|
17
|
+
ENV DIRAC_DIR="/home/dpps"
|
|
18
|
+
ENV DIRACOS="${DIRAC_DIR}/diracos"
|
|
19
|
+
|
|
20
|
+
RUN \
|
|
21
|
+
if getent group ${GROUPID}; then \
|
|
22
|
+
groupmod --new-name dpps $(getent group ${GROUPID} | cut -d: -f1); \
|
|
23
|
+
else \
|
|
24
|
+
groupadd --gid ${GROUPID} dpps; \
|
|
25
|
+
fi\
|
|
26
|
+
&& adduser --uid ${USERID} --gid dpps -s /bin/bash -d /home/dpps dpps
|
|
27
|
+
|
|
28
|
+
RUN mkdir -p ${DIRAC_DIR} \
|
|
29
|
+
&& chown -R dpps:dpps ${DIRAC_DIR}
|
|
30
|
+
|
|
31
|
+
WORKDIR ${DIRAC_DIR}
|
|
32
|
+
|
|
33
|
+
# Install DIRAC client
|
|
34
|
+
USER dpps
|
|
35
|
+
|
|
36
|
+
RUN curl -LO https://github.com/DIRACGrid/DIRACOS2/releases/download/${DIRACOS_VERSION}/DIRACOS-Linux-x86_64.sh \
|
|
37
|
+
&& bash DIRACOS-Linux-x86_64.sh -p ${DIRAC_DIR}/diracos \
|
|
38
|
+
&& rm -f DIRACOS-Linux-x86_64.sh
|
|
39
|
+
|
|
40
|
+
RUN mkdir -p ${DIRAC_DIR}/.globus \
|
|
41
|
+
&& mkdir -p ${DIRACOS}/etc/grid-security/certificates
|
|
42
|
+
|
|
43
|
+
COPY --from=builder /tmp/dist/ /tmp/dist/
|
|
44
|
+
RUN source ${DIRAC_DIR}/diracos/diracosrc && pip install --no-cache-dir /tmp/dist/ctao*
|
|
45
|
+
|
|
46
|
+
ENV \
|
|
47
|
+
CONDA_PREFIX="${DIRACOS}" \
|
|
48
|
+
MAMBA_ROOT_PREFIX="${DIRACOS}" \
|
|
49
|
+
CONDOR_CONFIG="${DIRACOS}/etc/condor/condor_config" \
|
|
50
|
+
DAVIX_DISABLE_REDIRECT_CACHING=1 \
|
|
51
|
+
DAVIX_USE_LIBCURL=1 \
|
|
52
|
+
GSETTINGS_SCHEMA_DIR="${DIRACOS}/share/glib-2.0/schemas" \
|
|
53
|
+
PATH="${DIRACOS}/bin:${DIRACOS}/condabin:${PATH}" \
|
|
54
|
+
X509_CERT_DIR="${DIRACOS}/etc/grid-security/certificates" \
|
|
55
|
+
X509_VOMSES="${DIRACOS}/etc/grid-security/vomses" \
|
|
56
|
+
X509_VOMS_DIR="${DIRACOS}/etc/grid-security/vomsdir" \
|
|
57
|
+
XML_CATALOG_FILES="file://${DIRACOS}/etc/xml/catalog file:///etc/xml/catalog" \
|
|
58
|
+
RUCIO_POLICY_PACKAGE="bdms_rucio_policy" \
|
|
59
|
+
RUCIO_CLIENT_MODE="1"
|
|
60
|
+
|
|
61
|
+
USER dpps
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
###
|
|
2
|
+
export CHART_NAME=dpps
|
|
3
|
+
export CHART_LOCATION=chart
|
|
4
|
+
include dpps-aiv-toolkit/Makefile
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
export TEST_ARTIFACTS_PATH ?= $(PWD)
|
|
8
|
+
export TEST_REPORT_CONFIG ?= $(PWD)/config.yml
|
|
9
|
+
|
|
10
|
+
# for some reason, here the default is different
|
|
11
|
+
export DEV_IMAGE_SUFFIX=clients
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: ctao-dpps-clients
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: DPPS python clients package
|
|
5
|
+
Author: The CTAO DPPS team
|
|
6
|
+
Maintainer-email: Maximilian Linhoff <maximilian.linhoff@cta-observatory.org>
|
|
7
|
+
License: BSD-3-Clause
|
|
8
|
+
Project-URL: repository, https://gitlab.cta-observatory.org/cta-computing/dpps/dpps
|
|
9
|
+
Project-URL: documentation, http://cta-computing.gitlab-pages.cta-observatory.org/dpps/dpps/
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
Requires-Dist: ctao-wms-clients~=0.1.0rc2
|
|
13
|
+
Requires-Dist: ctao-bdms-clients~=0.1.0rc3
|
|
14
|
+
Provides-Extra: test
|
|
15
|
+
Requires-Dist: pytest; extra == "test"
|
|
16
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
17
|
+
Requires-Dist: pytest-requirements; extra == "test"
|
|
18
|
+
Provides-Extra: doc
|
|
19
|
+
Requires-Dist: sphinx; extra == "doc"
|
|
20
|
+
Requires-Dist: numpydoc; extra == "doc"
|
|
21
|
+
Requires-Dist: ctao-sphinx-theme; extra == "doc"
|
|
22
|
+
Requires-Dist: myst-parser; extra == "doc"
|
|
23
|
+
Requires-Dist: sphinx-changelog; extra == "doc"
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: setuptools_scm; extra == "dev"
|
|
26
|
+
Requires-Dist: sphinx-autobuild; extra == "dev"
|
|
27
|
+
|
|
28
|
+
# Data Processing and Preservation System (DPPS)
|
|
29
|
+
|
|
30
|
+
The Data Processing and Preservation Systems is one of the major components of CTAO Computing.
|
|
31
|
+
|
|
32
|
+
It is responsible for transferring and preserving the DL0 data produced by
|
|
33
|
+
[ACADA](https://gitlab.cta-observatory.org/cta-computing/acada/array-control-and-data-acquisition)
|
|
34
|
+
and then processing it to data level DL3.
|
|
35
|
+
|
|
36
|
+
See [the documentation](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/dpps) for more details.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Data Processing and Preservation System (DPPS)
|
|
2
|
+
|
|
3
|
+
The Data Processing and Preservation Systems is one of the major components of CTAO Computing.
|
|
4
|
+
|
|
5
|
+
It is responsible for transferring and preserving the DL0 data produced by
|
|
6
|
+
[ACADA](https://gitlab.cta-observatory.org/cta-computing/acada/array-control-and-data-acquisition)
|
|
7
|
+
and then processing it to data level DL3.
|
|
8
|
+
|
|
9
|
+
See [the documentation](http://cta-computing.gitlab-pages.cta-observatory.org/dpps/dpps) for more details.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# This is the configuration used both in CI and local Makefile
|
|
2
|
+
|
|
3
|
+
variables:
|
|
4
|
+
# used to select requirements
|
|
5
|
+
DPPS_UC_GROUPS: ALL
|
|
6
|
+
DPPS_RELEASE: v0.0
|
|
7
|
+
|
|
8
|
+
CHART_LOCATION: chart
|
|
9
|
+
CHART_NAME: dpps
|
|
10
|
+
|
|
11
|
+
RELEASE_PLAN_FN: dpps-release-plan/release_development_document/dpps-release-plan.tex
|
|
12
|
+
TRACEABILITY_CSV_FN: requirements-traceability/DPPS_Requirements_Traceability_Matrix.csv
|
|
13
|
+
|
|
14
|
+
APPLICATION_NAME: DPPS
|
|
15
|
+
APPLICATION_AUTHOR: DPPS Team
|
|
16
|
+
APPLICATION_AUTHOR_ORGANIZATION: CTAO
|
|
17
|
+
APPLICATION_VERSION: 0.0.0a0
|
|
18
|
+
|
|
19
|
+
DEFAULT_HELM_TEST: name=dpps-pytest
|
|
20
|
+
|
|
21
|
+
EXTRA_CONFIG_FILES: aiv-config-dependencies.yml
|
|
22
|
+
|
|
23
|
+
# these UCs are executed during project deployment and hence verified by the very completion of the pipeline
|
|
24
|
+
# TODO: separate in bdms,wms
|
|
25
|
+
DEPLOYMENT_UCS: DPPS-UC-170-1.2.1,DPPS-UC-170-1.2.2,DPPS-UC-170-1.2.3,DPPS-UC-170-1.2.4,DPPS-UC-170-1.6
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# this is a parent helm chart for the DPPS project
|
|
2
|
+
|
|
3
|
+
apiVersion: v2
|
|
4
|
+
name: dpps
|
|
5
|
+
version: "0.0.0-dev"
|
|
6
|
+
appVersion: "0.0.0-dev"
|
|
7
|
+
description: A Helm chart for the DPPS project
|
|
8
|
+
type: application
|
|
9
|
+
keywords:
|
|
10
|
+
- dpps
|
|
11
|
+
|
|
12
|
+
maintainers:
|
|
13
|
+
- name: The DPPS Authors
|
|
14
|
+
email:
|
|
15
|
+
|
|
16
|
+
dependencies:
|
|
17
|
+
- name: bdms
|
|
18
|
+
version: v0.1.0
|
|
19
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
|
20
|
+
|
|
21
|
+
- name: wms
|
|
22
|
+
version: v0.1.0
|
|
23
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
|
24
|
+
|
|
25
|
+
- name: cvmfs
|
|
26
|
+
version: v0.1.0
|
|
27
|
+
condition: cvmfs.enabled
|
|
28
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
|
29
|
+
|
|
30
|
+
- name: cert-generator-grid
|
|
31
|
+
condition: cert-generator-grid.enabled
|
|
32
|
+
version: v0.2.3
|
|
33
|
+
repository: oci://harbor.cta-observatory.org/dpps
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# dpps
|
|
2
|
+
|
|
3
|
+
  
|
|
4
|
+
|
|
5
|
+
A Helm chart for the DPPS project
|
|
6
|
+
|
|
7
|
+
## Maintainers
|
|
8
|
+
|
|
9
|
+
| Name | Email | Url |
|
|
10
|
+
| ---- | ------ | --- |
|
|
11
|
+
| The DPPS Authors | | |
|
|
12
|
+
|
|
13
|
+
## Requirements
|
|
14
|
+
|
|
15
|
+
| Repository | Name | Version |
|
|
16
|
+
|------------|------|---------|
|
|
17
|
+
| oci://harbor.cta-observatory.org/dpps | bdms | v0.1.0 |
|
|
18
|
+
| oci://harbor.cta-observatory.org/dpps | cert-generator-grid | v0.2.3 |
|
|
19
|
+
| oci://harbor.cta-observatory.org/dpps | cvmfs | v0.1.0 |
|
|
20
|
+
| oci://harbor.cta-observatory.org/dpps | wms | v0.1.0 |
|
|
21
|
+
|
|
22
|
+
## Values
|
|
23
|
+
|
|
24
|
+
| Key | Type | Default | Description |
|
|
25
|
+
|-----|------|---------|-------------|
|
|
26
|
+
| bdms.cert-generator-grid.enabled | bool | `true` | |
|
|
27
|
+
| bdms.configure_test_setup | bool | `true` | |
|
|
28
|
+
| bdms.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@dpps-postgresql:5432/rucio"` | |
|
|
29
|
+
| bdms.postgresql.global.postgresql.auth.database | string | `"rucio"` | |
|
|
30
|
+
| bdms.postgresql.global.postgresql.auth.password | string | `"XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM"` | |
|
|
31
|
+
| bdms.postgresql.global.postgresql.auth.username | string | `"rucio"` | |
|
|
32
|
+
| bdms.rucio-daemons.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@dpps-postgresql:5432/rucio"` | |
|
|
33
|
+
| bdms.rucio-daemons.conveyorTransferSubmitterCount | int | `1` | |
|
|
34
|
+
| bdms.rucio-server.authRucioHost | string | `"rucio-server.local"` | |
|
|
35
|
+
| bdms.rucio-server.config.database.default | string | `"postgresql://rucio:XcL0xT9FgFgJEc4i3OcQf2DMVKpjIWDGezqcIPmXlM@dpps-postgresql:5432/rucio"` | |
|
|
36
|
+
| bdms.rucio-server.exposeErrorLogs | bool | `false` | |
|
|
37
|
+
| bdms.rucio-server.ftsRenewal.enabled | bool | `false` | |
|
|
38
|
+
| bdms.rucio-server.httpd_config.grid_site_enabled | string | `"True"` | |
|
|
39
|
+
| bdms.rucio-server.ingress.enabled | bool | `true` | |
|
|
40
|
+
| bdms.rucio-server.ingress.hosts[0] | string | `"rucio-server.local"` | |
|
|
41
|
+
| bdms.rucio-server.livenessProbe.initialDelaySeconds | int | `40` | |
|
|
42
|
+
| bdms.rucio-server.livenessProbe.periodSeconds | int | `10` | |
|
|
43
|
+
| bdms.rucio-server.livenessProbe.successThreshold | int | `1` | |
|
|
44
|
+
| bdms.rucio-server.livenessProbe.timeoutSeconds | int | `15` | |
|
|
45
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_BOOTSTRAP_USERPASS_IDENTITY | string | `"dpps"` | |
|
|
46
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_BOOTSTRAP_USERPASS_PWD | string | `"secret"` | |
|
|
47
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_BOOTSTRAP_X509_EMAIL | string | `"dpps-test@cta-observatory.org"` | |
|
|
48
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_BOOTSTRAP_X509_IDENTITY | string | `"CN=DPPS User"` | |
|
|
49
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_COMMON_EXTRACT_SCOPE | string | `"ctao_bdms"` | |
|
|
50
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_POLICY_LFN2PFN_ALGORITHM_DEFAULT | string | `"ctao_bdms"` | |
|
|
51
|
+
| bdms.rucio-server.optional_config.RUCIO_CFG_POLICY_PACKAGE | string | `"bdms_rucio_policy"` | |
|
|
52
|
+
| bdms.rucio-server.readinessProbe.initialDelaySeconds | int | `40` | |
|
|
53
|
+
| bdms.rucio-server.readinessProbe.periodSeconds | int | `10` | |
|
|
54
|
+
| bdms.rucio-server.readinessProbe.successThreshold | int | `1` | |
|
|
55
|
+
| bdms.rucio-server.readinessProbe.timeoutSeconds | int | `15` | |
|
|
56
|
+
| bdms.rucio-server.replicaCount | int | `1` | |
|
|
57
|
+
| bdms.rucio-server.service.name | string | `"https"` | |
|
|
58
|
+
| bdms.rucio-server.service.port | int | `443` | |
|
|
59
|
+
| bdms.rucio-server.service.protocol | string | `"TCP"` | |
|
|
60
|
+
| bdms.rucio-server.service.targetPort | int | `443` | |
|
|
61
|
+
| bdms.rucio-server.service.type | string | `"ClusterIP"` | |
|
|
62
|
+
| bdms.rucio-server.useSSL | bool | `true` | |
|
|
63
|
+
| bdms.safe_to_bootstrap_rucio | bool | `true` | |
|
|
64
|
+
| cert-generator-grid.enabled | bool | `false` | |
|
|
65
|
+
| cert-generator-grid.generatePreHooks | bool | `true` | |
|
|
66
|
+
| cvmfs.enabled | bool | `false` | |
|
|
67
|
+
| dev.client_image_tag | string | `nil` | |
|
|
68
|
+
| dev.mount_repo | bool | `true` | |
|
|
69
|
+
| dev.runAsGroup | int | `1000` | |
|
|
70
|
+
| dev.runAsUser | int | `1000` | |
|
|
71
|
+
| dev.run_tests | bool | `true` | |
|
|
72
|
+
| dev.sleep | bool | `false` | |
|
|
73
|
+
| dev.start_long_running_client | bool | `false` | |
|
|
74
|
+
| image.pullPolicy | string | `"IfNotPresent"` | |
|
|
75
|
+
| image.repository_prefix | string | `"harbor.cta-observatory.org/dpps/dpps"` | |
|
|
76
|
+
| wms.cert-generator-grid.enabled | bool | `false` | |
|
|
77
|
+
| wms.rucio.enabled | bool | `true` | |
|
|
78
|
+
| wms.rucio.rucioConfig | string | `"dpps-bdms-rucio-config"` | |
|
|
79
|
+
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
apiVersion: v1
|
|
2
|
+
kind: Pod
|
|
3
|
+
metadata:
|
|
4
|
+
name: "{{ .Release.Name }}-pytest"
|
|
5
|
+
labels:
|
|
6
|
+
{{- include "wms.labels" . | nindent 4 }}
|
|
7
|
+
{{ if .Values.dev.start_long_running_client }}
|
|
8
|
+
{{ else }}
|
|
9
|
+
annotations:
|
|
10
|
+
"helm.sh/hook": test
|
|
11
|
+
{{ end }}
|
|
12
|
+
spec:
|
|
13
|
+
securityContext:
|
|
14
|
+
runAsUser: {{ .Values.dev.runAsUser }}
|
|
15
|
+
runAsGroup: {{ .Values.dev.runAsGroup }}
|
|
16
|
+
fsGroup: {{ .Values.dev.runAsGroup }}
|
|
17
|
+
|
|
18
|
+
containers:
|
|
19
|
+
- name: test
|
|
20
|
+
image: "{{ .Values.image.repository_prefix }}-clients:{{ .Values.dev.client_image_tag | default .Chart.AppVersion }}"
|
|
21
|
+
workingDir: /home/dpps/dpps
|
|
22
|
+
|
|
23
|
+
command:
|
|
24
|
+
- bash
|
|
25
|
+
- -c
|
|
26
|
+
- |
|
|
27
|
+
set -xe
|
|
28
|
+
|
|
29
|
+
source ${DIRACOS}/diracosrc
|
|
30
|
+
|
|
31
|
+
{{ if .Values.dev.mount_repo }}
|
|
32
|
+
pip install -e .[test]
|
|
33
|
+
{{ end }}
|
|
34
|
+
|
|
35
|
+
mkdir -pv ${DIRACOS}/etc/grid-security/certificates/
|
|
36
|
+
cp -fv /etc/grid-security/certificates/* ${DIRACOS}/etc/grid-security/certificates/
|
|
37
|
+
install -D -m 644 -v /opt/user/globus/usercert.pem $HOME/.globus/usercert.pem
|
|
38
|
+
install -D -m 600 -v /opt/user/globus/userkey.pem $HOME/.globus/userkey.pem
|
|
39
|
+
install -D -m 600 /opt/user/ssh/diracuser_sshkey $HOME/.ssh/diracuser_sshkey
|
|
40
|
+
ls -lR ${DIRACOS}/etc/grid-security
|
|
41
|
+
ls -lR ${HOME}/.globus/
|
|
42
|
+
|
|
43
|
+
dirac-proxy-init -g dpps_group --nocs
|
|
44
|
+
dirac-configure -C dips://dirac-server:9135/Configuration/Server -S DPPS-Tests
|
|
45
|
+
|
|
46
|
+
{{ if .Values.dev.run_tests }}
|
|
47
|
+
python3 -m pytest \
|
|
48
|
+
-svv \
|
|
49
|
+
-o cache_dir=$PWD/.pytest-cache \
|
|
50
|
+
--cov \
|
|
51
|
+
--cov-report=xml:/tmp/coverage.xml \
|
|
52
|
+
--junitxml=/tmp/report.xml \
|
|
53
|
+
--color=yes \
|
|
54
|
+
--log-cli-level=INFO
|
|
55
|
+
|
|
56
|
+
curl -T /tmp/coverage.xml http://testkit:8000/coverage.xml
|
|
57
|
+
curl -T /tmp/report.xml http://testkit:8000/report.xml
|
|
58
|
+
{{ end }}
|
|
59
|
+
|
|
60
|
+
{{ if .Values.dev.sleep }}
|
|
61
|
+
sleep infinity
|
|
62
|
+
{{ end }}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
volumeMounts:
|
|
66
|
+
{{ if .Values.dev.mount_repo }}
|
|
67
|
+
- name: repo
|
|
68
|
+
mountPath: /home/dpps/dpps
|
|
69
|
+
{{ end }}
|
|
70
|
+
- name: cafile
|
|
71
|
+
subPath: ca.pem
|
|
72
|
+
mountPath: /etc/grid-security/certificates/dpps_test_ca.pem
|
|
73
|
+
- name: cafile
|
|
74
|
+
subPath: ca.pem
|
|
75
|
+
mountPath: /etc/grid-security/certificates/74df993b.0
|
|
76
|
+
- name: cafile
|
|
77
|
+
subPath: dpps_test_ca.crl.r0
|
|
78
|
+
mountPath: /etc/grid-security/certificates/74df993b.r0
|
|
79
|
+
- name: cafile
|
|
80
|
+
subPath: dpps_test_ca.crl.r0
|
|
81
|
+
mountPath: /etc/grid-security/certificates/dpps_test_ca.crl.r0
|
|
82
|
+
- name: cafile
|
|
83
|
+
subPath: ca.pem
|
|
84
|
+
mountPath: /etc/grid-security/ca.pem
|
|
85
|
+
# this unusual location is needed not to causee somewhat unpredictable behavior due to conflict with mounted home
|
|
86
|
+
- name: dppsuser-certkey-400
|
|
87
|
+
mountPath: /opt/user/globus/userkey.pem
|
|
88
|
+
subPath: dppsuser.key.pem
|
|
89
|
+
- name: dppsuser-certkey-600
|
|
90
|
+
mountPath: /opt/user/globus/usercert.pem
|
|
91
|
+
subPath: dppsuser.pem
|
|
92
|
+
- name: diracuser-sshkey
|
|
93
|
+
mountPath: /opt/user/ssh/diracuser_sshkey
|
|
94
|
+
subPath: ssh-diracuser_sshkey
|
|
95
|
+
- name: rucio-config
|
|
96
|
+
mountPath: /opt/rucio/etc
|
|
97
|
+
|
|
98
|
+
volumes:
|
|
99
|
+
{{ if .Values.dev.mount_repo }}
|
|
100
|
+
- name: repo
|
|
101
|
+
hostPath:
|
|
102
|
+
path: /repo
|
|
103
|
+
type: Directory
|
|
104
|
+
{{ end }}
|
|
105
|
+
- name: cafile
|
|
106
|
+
secret:
|
|
107
|
+
defaultMode: 420
|
|
108
|
+
secretName: {{ template "certprefix" . }}-server-cafile
|
|
109
|
+
- name: dppsuser-certkey-600
|
|
110
|
+
secret:
|
|
111
|
+
defaultMode: 0600
|
|
112
|
+
secretName: {{ template "certprefix" . }}-dppsuser-certkey
|
|
113
|
+
- name: dppsuser-certkey-400
|
|
114
|
+
secret:
|
|
115
|
+
defaultMode: 0400
|
|
116
|
+
secretName: {{ template "certprefix" . }}-dppsuser-certkey
|
|
117
|
+
- name: diracuser-sshkey
|
|
118
|
+
secret:
|
|
119
|
+
secretName: {{ include "wms.fullname" . }}-diracuser-sshkey
|
|
120
|
+
defaultMode: 420
|
|
121
|
+
- name: rucio-config
|
|
122
|
+
configMap:
|
|
123
|
+
name: {{ .Release.Name }}-bdms-rucio-config
|
|
124
|
+
|
|
125
|
+
restartPolicy: Never
|