exordos-s3 0.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exordos_s3-0.0.3/.github/workflows/build.yml +125 -0
- exordos_s3-0.0.3/.github/workflows/publish-to-pypi.yml +97 -0
- exordos_s3-0.0.3/.github/workflows/tests.yaml +24 -0
- exordos_s3-0.0.3/.gitignore +9 -0
- exordos_s3-0.0.3/Makefile +46 -0
- exordos_s3-0.0.3/PKG-INFO +399 -0
- exordos_s3-0.0.3/README.md +366 -0
- exordos_s3-0.0.3/etc/exordos_metapaas/logging.yaml +29 -0
- exordos_s3-0.0.3/etc/exordos_metapaas/metapaas_s3_agent.conf +10 -0
- exordos_s3-0.0.3/etc/systemd/exordos-metapaas-rustfs.service +35 -0
- exordos_s3-0.0.3/etc/systemd/exordos-metapaas-s3-agent.service +14 -0
- exordos_s3-0.0.3/exordos/exordos.yaml +41 -0
- exordos_s3-0.0.3/exordos/images/dp_bootstrap.sh +36 -0
- exordos_s3-0.0.3/exordos/images/dp_install.sh +87 -0
- exordos_s3-0.0.3/exordos/manifests/communal_s3.yaml.j2 +42 -0
- exordos_s3-0.0.3/exordos/manifests/example_s3.yaml.j2 +174 -0
- exordos_s3-0.0.3/exordos/manifests/s3aas.yaml.j2 +169 -0
- exordos_s3-0.0.3/exordos_s3/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/constants.py +18 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/api/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/api/controllers.py +189 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/api/routes.py +69 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/dm/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/dm/models.py +304 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/infra/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/infra/dm/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/infra/dm/models.py +122 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/infra/services/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/infra/services/builder.py +189 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/paas/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/paas/dm/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/paas/dm/models.py +99 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/paas/services/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/controlplane/paas/services/builder.py +158 -0
- exordos_s3-0.0.3/exordos_s3/dataplane/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/dataplane/driver.py +718 -0
- exordos_s3-0.0.3/exordos_s3/definition.py +77 -0
- exordos_s3-0.0.3/exordos_s3/migrations/0000-init-s3.py +185 -0
- exordos_s3-0.0.3/exordos_s3/migrations/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/tests/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/__init__.py +0 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/conftest.py +538 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/prepare_env.py +487 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/test_access_key_lifecycle.py +313 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/test_basic_crud.py +341 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/test_data_integrity.py +175 -0
- exordos_s3-0.0.3/exordos_s3/tests/functional/test_policy_enforcement.py +473 -0
- exordos_s3-0.0.3/exordos_s3/utils.py +31 -0
- exordos_s3-0.0.3/exordos_s3.egg-info/PKG-INFO +399 -0
- exordos_s3-0.0.3/exordos_s3.egg-info/SOURCES.txt +57 -0
- exordos_s3-0.0.3/exordos_s3.egg-info/dependency_links.txt +1 -0
- exordos_s3-0.0.3/exordos_s3.egg-info/entry_points.txt +5 -0
- exordos_s3-0.0.3/exordos_s3.egg-info/requires.txt +27 -0
- exordos_s3-0.0.3/exordos_s3.egg-info/top_level.txt +1 -0
- exordos_s3-0.0.3/pyproject.toml +105 -0
- exordos_s3-0.0.3/setup.cfg +4 -0
- exordos_s3-0.0.3/tox.ini +66 -0
- exordos_s3-0.0.3/uv.lock +2377 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
name: build
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
types: [opened]
|
|
7
|
+
|
|
8
|
+
env:
|
|
9
|
+
PACKER_VERSION: "1.9.2"
|
|
10
|
+
PYTHON_VERSION: "3.12"
|
|
11
|
+
CORE_ENDPOINT_URL: http://10.20.0.2:80/api/core
|
|
12
|
+
CORE_USERNAME: "admin"
|
|
13
|
+
CORE_PASSWORD: "admin"
|
|
14
|
+
ELEMENT_NAME: s3aas
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
Build:
|
|
18
|
+
runs-on: ubuntu-24.04
|
|
19
|
+
if: ${{ github.actor != 'dependabot[bot]' }}
|
|
20
|
+
steps:
|
|
21
|
+
- name: Setup packer
|
|
22
|
+
uses: hashicorp/setup-packer@main
|
|
23
|
+
with:
|
|
24
|
+
version: ${{ env.PACKER_VERSION }}
|
|
25
|
+
|
|
26
|
+
- name: Install exordos CLI
|
|
27
|
+
run: curl -fsSL https://repo.exordos.com/install.sh | sudo sh
|
|
28
|
+
|
|
29
|
+
- name: Install hypervisor
|
|
30
|
+
run: |
|
|
31
|
+
sudo exordos compute hypervisors init
|
|
32
|
+
sudo chmod 666 /dev/kvm
|
|
33
|
+
|
|
34
|
+
- uses: actions/checkout@v6
|
|
35
|
+
with:
|
|
36
|
+
fetch-depth: 0
|
|
37
|
+
|
|
38
|
+
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
|
39
|
+
uses: actions/setup-python@v6
|
|
40
|
+
with:
|
|
41
|
+
python-version: ${{ env.PYTHON_VERSION }}
|
|
42
|
+
|
|
43
|
+
- name: Install uv + tox + build
|
|
44
|
+
uses: astral-sh/setup-uv@v7
|
|
45
|
+
- run: |
|
|
46
|
+
uv tool install tox --with tox-uv
|
|
47
|
+
pip install build
|
|
48
|
+
|
|
49
|
+
- name: Build and push exordos_s3
|
|
50
|
+
timeout-minutes: 20
|
|
51
|
+
env:
|
|
52
|
+
PUSH_EXORDOS_CFG: ${{ secrets.PUSH_EXORDOS_CFG }}
|
|
53
|
+
run: |
|
|
54
|
+
set -eux
|
|
55
|
+
|
|
56
|
+
VERSION="$(exordos get-version .)"
|
|
57
|
+
|
|
58
|
+
exordos build $(pwd)
|
|
59
|
+
|
|
60
|
+
echo "$PUSH_EXORDOS_CFG" | base64 -d > exordos/exordos.push.yaml
|
|
61
|
+
exordos push -c exordos/exordos.push.yaml -t exordos_repo -f --latest
|
|
62
|
+
|
|
63
|
+
echo "S3AAS_VERSION=${VERSION}" >> $GITHUB_ENV
|
|
64
|
+
|
|
65
|
+
- name: Configure exordos settings
|
|
66
|
+
run: |
|
|
67
|
+
exordos settings set-realm default --endpoint $CORE_ENDPOINT_URL
|
|
68
|
+
exordos settings set-context default --name default --user $CORE_USERNAME --password $CORE_PASSWORD
|
|
69
|
+
|
|
70
|
+
- name: Bootstrap exordos_core
|
|
71
|
+
run: exordos bootstrap -i latest -f -m core --admin-password $CORE_PASSWORD --cidr 10.20.0.0/22
|
|
72
|
+
|
|
73
|
+
- name: Wait for Exordos Core API
|
|
74
|
+
run: |
|
|
75
|
+
for i in {1..60}; do
|
|
76
|
+
if exordos ready_api; then
|
|
77
|
+
echo "Exordos Core API is ready"
|
|
78
|
+
break
|
|
79
|
+
fi
|
|
80
|
+
echo "Waiting for Exordos Core API... ($i/60)"
|
|
81
|
+
sleep 5
|
|
82
|
+
done
|
|
83
|
+
|
|
84
|
+
- name: Set hypervisor with overbooking
|
|
85
|
+
run: |
|
|
86
|
+
HYPER_UUID="$(exordos c h l -o json | jq -r .[0]."UUID")"
|
|
87
|
+
exordos compute hypervisors update $HYPER_UUID --cores-ratio 10.0 --ram-ratio 10.0
|
|
88
|
+
|
|
89
|
+
- name: Build + serve + install (prepare_env.py)
|
|
90
|
+
run: |
|
|
91
|
+
tox -e develop
|
|
92
|
+
|
|
93
|
+
.tox/develop/bin/python \
|
|
94
|
+
exordos_s3/tests/functional/prepare_env.py \
|
|
95
|
+
--project-dir . \
|
|
96
|
+
--output-dir /tmp/metapaas-s3-build \
|
|
97
|
+
--endpoint $CORE_ENDPOINT_URL \
|
|
98
|
+
--username $CORE_USERNAME \
|
|
99
|
+
--password "${CORE_PASSWORD}" \
|
|
100
|
+
--wait-timeout 600 \
|
|
101
|
+
2>&1 | tee /tmp/prepare_env.log
|
|
102
|
+
|
|
103
|
+
grep "^ export " /tmp/prepare_env.log | sed 's/^ export //' >> $GITHUB_ENV
|
|
104
|
+
|
|
105
|
+
- name: Wait for S3 API
|
|
106
|
+
run: |
|
|
107
|
+
CP_URL=$(grep "EXORDOS_S3_CP_URL" $GITHUB_ENV | cut -d= -f2)
|
|
108
|
+
for i in {1..30}; do
|
|
109
|
+
CODE=$(curl -s -o /dev/null -w "%{http_code}" "${CP_URL}/v1/")
|
|
110
|
+
if echo "$CODE" | grep -qE "200|404"; then
|
|
111
|
+
echo "MetaPaaS user-api ready (HTTP $CODE)"; exit 0
|
|
112
|
+
fi
|
|
113
|
+
echo "Waiting ($i/30) for user-api…"; sleep 10
|
|
114
|
+
done
|
|
115
|
+
exit 1
|
|
116
|
+
|
|
117
|
+
- name: Run functional tests
|
|
118
|
+
run: tox -e ${{ env.PYTHON_VERSION }}-functional
|
|
119
|
+
|
|
120
|
+
- name: Debug session on failure
|
|
121
|
+
uses: owenthereal/action-upterm@v1
|
|
122
|
+
if: failure()
|
|
123
|
+
with:
|
|
124
|
+
limit-access-to-actor: true
|
|
125
|
+
wait-timeout-minutes: 30
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
name: Publish Python 🐍 distribution 📦 to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- '*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build:
|
|
10
|
+
name: Build distribution 📦
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v6
|
|
15
|
+
with:
|
|
16
|
+
persist-credentials: false
|
|
17
|
+
- name: Set up Python
|
|
18
|
+
uses: actions/setup-python@v6
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
- name: Install pypa/build
|
|
22
|
+
run: >-
|
|
23
|
+
python3 -m
|
|
24
|
+
pip install
|
|
25
|
+
build
|
|
26
|
+
--user
|
|
27
|
+
- name: Build a binary wheel and a source tarball
|
|
28
|
+
run: python3 -m build
|
|
29
|
+
- name: Store the distribution packages
|
|
30
|
+
uses: actions/upload-artifact@v7
|
|
31
|
+
with:
|
|
32
|
+
name: python-package-distributions
|
|
33
|
+
path: dist/
|
|
34
|
+
|
|
35
|
+
publish-to-pypi:
|
|
36
|
+
name: >-
|
|
37
|
+
Publish Python 🐍 distribution 📦 to PyPI
|
|
38
|
+
needs:
|
|
39
|
+
- build
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
environment:
|
|
42
|
+
name: pypi
|
|
43
|
+
url: https://pypi.org/p/exordos_s3
|
|
44
|
+
permissions:
|
|
45
|
+
id-token: write # IMPORTANT: mandatory for trusted publishing
|
|
46
|
+
|
|
47
|
+
steps:
|
|
48
|
+
- name: Download all the dists
|
|
49
|
+
uses: actions/download-artifact@v8
|
|
50
|
+
with:
|
|
51
|
+
name: python-package-distributions
|
|
52
|
+
path: dist/
|
|
53
|
+
- name: Publish distribution 📦 to PyPI
|
|
54
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
55
|
+
|
|
56
|
+
github-release:
|
|
57
|
+
name: >-
|
|
58
|
+
Sign the Python 🐍 distribution 📦 with Sigstore
|
|
59
|
+
and upload them to GitHub Release
|
|
60
|
+
needs:
|
|
61
|
+
- publish-to-pypi
|
|
62
|
+
runs-on: ubuntu-latest
|
|
63
|
+
|
|
64
|
+
permissions:
|
|
65
|
+
contents: write # IMPORTANT: mandatory for making GitHub Releases
|
|
66
|
+
id-token: write # IMPORTANT: mandatory for sigstore
|
|
67
|
+
|
|
68
|
+
steps:
|
|
69
|
+
- name: Download all the dists
|
|
70
|
+
uses: actions/download-artifact@v8
|
|
71
|
+
with:
|
|
72
|
+
name: python-package-distributions
|
|
73
|
+
path: dist/
|
|
74
|
+
- name: Sign the dists with Sigstore
|
|
75
|
+
uses: sigstore/gh-action-sigstore-python@v3.3.0
|
|
76
|
+
with:
|
|
77
|
+
inputs: >-
|
|
78
|
+
./dist/*.tar.gz
|
|
79
|
+
./dist/*.whl
|
|
80
|
+
- name: Create GitHub Release
|
|
81
|
+
env:
|
|
82
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
83
|
+
run: >-
|
|
84
|
+
gh release create
|
|
85
|
+
"$GITHUB_REF_NAME"
|
|
86
|
+
--repo "$GITHUB_REPOSITORY"
|
|
87
|
+
--notes ""
|
|
88
|
+
- name: Upload artifact signatures to GitHub Release
|
|
89
|
+
env:
|
|
90
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
91
|
+
# Upload to GitHub Release using the `gh` CLI.
|
|
92
|
+
# `dist/` contains the built packages, and the
|
|
93
|
+
# sigstore-produced signatures and certificates.
|
|
94
|
+
run: >-
|
|
95
|
+
gh release upload
|
|
96
|
+
"$GITHUB_REF_NAME" dist/**
|
|
97
|
+
--repo "$GITHUB_REPOSITORY"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
types: [opened]
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
Lint:
|
|
10
|
+
runs-on: ubuntu-24.04
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v6
|
|
13
|
+
- name: Set up Python 3.12
|
|
14
|
+
uses: actions/setup-python@v6
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.12"
|
|
17
|
+
- name: Install requirements
|
|
18
|
+
run: sudo apt update && sudo apt install --yes libev-dev libvirt-dev
|
|
19
|
+
- name: Install uv
|
|
20
|
+
uses: astral-sh/setup-uv@v7
|
|
21
|
+
- name: Install tox-uv
|
|
22
|
+
run: uv tool install tox --with tox-uv
|
|
23
|
+
- name: Ruff check
|
|
24
|
+
run: tox -e ruff-check
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
SHELL := bash
|
|
2
|
+
SSH_KEY ?= ~/.ssh/id_ed25519.pub
|
|
3
|
+
REPOSITORY ?= http://10.20.0.1:8080/exordos-elements
|
|
4
|
+
INDEX_URL ?= http://10.20.0.1:8080/simple/
|
|
5
|
+
|
|
6
|
+
all: help
|
|
7
|
+
|
|
8
|
+
help:
|
|
9
|
+
@echo "build - build the s3aas element manifest + DP image"
|
|
10
|
+
@echo "install - install s3aas element into Core"
|
|
11
|
+
@echo "wheel - build Python wheel for exordos_paas_s3"
|
|
12
|
+
@echo "publish-wheel - copy wheel to local pip index"
|
|
13
|
+
@echo "lint - run ruff check"
|
|
14
|
+
@echo "format - run ruff format"
|
|
15
|
+
@echo "test - run unit tests via tox"
|
|
16
|
+
@echo "functional - run functional tests (needs live stand)"
|
|
17
|
+
@echo "typecheck - run mypy"
|
|
18
|
+
|
|
19
|
+
build:
|
|
20
|
+
exordos build -c exordos/exordos.yaml -i $(SSH_KEY) -f \
|
|
21
|
+
--manifest-var repository=$(REPOSITORY) \
|
|
22
|
+
--manifest-var index_url=$(INDEX_URL)
|
|
23
|
+
|
|
24
|
+
install:
|
|
25
|
+
exordos em elements install output/manifests/s3aas.yaml
|
|
26
|
+
|
|
27
|
+
wheel:
|
|
28
|
+
python -m build --wheel
|
|
29
|
+
|
|
30
|
+
publish-wheel: wheel
|
|
31
|
+
cp dist/exordos_paas_s3-*.whl /srv/exordos-local-repo/simple/
|
|
32
|
+
|
|
33
|
+
lint:
|
|
34
|
+
tox -e ruff-check
|
|
35
|
+
|
|
36
|
+
format:
|
|
37
|
+
tox -e ruff
|
|
38
|
+
|
|
39
|
+
test:
|
|
40
|
+
tox -e py312
|
|
41
|
+
|
|
42
|
+
functional:
|
|
43
|
+
tox -e py312-functional
|
|
44
|
+
|
|
45
|
+
typecheck:
|
|
46
|
+
tox -e mypy
|
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: exordos_s3
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: S3 PaaS plugin for the Exordos MetaPaaS runtime
|
|
5
|
+
Author-email: Genesis Corporation <mail@gmelikov.ru>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: homepage, https://github.com/exordos/exordos_s3/
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: oslo.config<10.0.0,>=3.22.2
|
|
11
|
+
Requires-Dist: restalchemy<16.0.0,>=15.0.0
|
|
12
|
+
Requires-Dist: gcl_iam<2.0.0,>=1.0.3
|
|
13
|
+
Requires-Dist: gcl_looper<2.0.0,>=1.2.3
|
|
14
|
+
Requires-Dist: gcl_sdk<4.0.0,>=3.0.0
|
|
15
|
+
Requires-Dist: exordos_metapaas<1.0.0,>=0.0.3
|
|
16
|
+
Requires-Dist: minio<8.0.0,>=7.2.0
|
|
17
|
+
Requires-Dist: requests<3.0.0,>=2.28.0
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: tox>=4.0.0; extra == "dev"
|
|
20
|
+
Requires-Dist: tox-uv; extra == "dev"
|
|
21
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: coverage>=4.0; extra == "test"
|
|
24
|
+
Requires-Dist: mock<4.0.0,>=3.0.5; extra == "test"
|
|
25
|
+
Requires-Dist: pytest<9.0.0,>=8.0.0; extra == "test"
|
|
26
|
+
Requires-Dist: pytest-timer<2.0.0,>=1.0.0; extra == "test"
|
|
27
|
+
Requires-Dist: boto3<2.0.0,>=1.26.0; extra == "test"
|
|
28
|
+
Requires-Dist: exordos<3.0.0,>=2.0.0; extra == "test"
|
|
29
|
+
Provides-Extra: ruff
|
|
30
|
+
Requires-Dist: ruff; extra == "ruff"
|
|
31
|
+
Provides-Extra: mypy
|
|
32
|
+
Requires-Dist: mypy; extra == "mypy"
|
|
33
|
+
|
|
34
|
+
# S3 Storage PaaS Plugin for MetaPaaS
|
|
35
|
+
|
|
36
|
+
A production-ready S3-compatible object storage service (RustFS + MinIO) packaged as a MetaPaaS plugin.
|
|
37
|
+
|
|
38
|
+
## What is This?
|
|
39
|
+
|
|
40
|
+
This repository provides the **s3aas** plugin: a complete, installable PaaS service for Exordos MetaPaaS. Instead of deploying separate control-plane infrastructure for S3 storage, this plugin integrates with a shared MetaPaaS runtime.
|
|
41
|
+
|
|
42
|
+
**Key components:**
|
|
43
|
+
- **CP (Control Plane):** REST API for managing S3 instances, buckets, users, policies, access keys (Python)
|
|
44
|
+
- **DP (Data Plane):** RustFS-based S3-compatible object storage VM (Packer-built)
|
|
45
|
+
- **Integration:** Automatic installation + scaling via MetaPaaS orchestration
|
|
46
|
+
|
|
47
|
+
## Quick Start
|
|
48
|
+
|
|
49
|
+
### Prerequisites
|
|
50
|
+
|
|
51
|
+
- Running `exordos_core` deployment
|
|
52
|
+
- Running `exordos_metapaas` deployment (shares one control-plane)
|
|
53
|
+
- Local build tools: `exordos` CLI, `packer`, Python 3.10+
|
|
54
|
+
|
|
55
|
+
### Build
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
make build \
|
|
59
|
+
REPOSITORY=http://10.20.0.1:8080/exordos-elements \
|
|
60
|
+
INDEX_URL=http://10.20.0.1:8080/simple/
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Produces:
|
|
64
|
+
- `output/dist/exordos_paas_s3-*.whl` — Control-plane package
|
|
65
|
+
- `output/exordos-metapaas-s3-dp.raw.zst` — Data-plane VM image
|
|
66
|
+
- `output/manifests/s3aas.yaml` — Element manifest for Exordos Core
|
|
67
|
+
|
|
68
|
+
### Publish to Index
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
make publish-wheel
|
|
72
|
+
# Copies .whl to /srv/exordos-local-repo/simple/
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Install to Exordos Core
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
exordos -e http://10.20.0.2:11010 \
|
|
79
|
+
-u admin -p <admin-password> \
|
|
80
|
+
ee install metapaas --version 0.0.7 --repository http://10.20.0.1:8080/exordos-elements
|
|
81
|
+
|
|
82
|
+
exordos -e http://10.20.0.2:11010 \
|
|
83
|
+
-u admin -p <admin-password> \
|
|
84
|
+
ee install s3aas --version 0.0.1 --repository http://10.20.0.1:8080/exordos-elements
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Wait for both elements to become ACTIVE.
|
|
88
|
+
|
|
89
|
+
### Create an S3 Instance
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
curl -X POST http://metapaas-cp:8080/v1/types/s3/instances \
|
|
93
|
+
-H 'Content-Type: application/json' \
|
|
94
|
+
-H 'Authorization: Bearer <token>' \
|
|
95
|
+
-d '{
|
|
96
|
+
"name": "s3-data",
|
|
97
|
+
"version": "0.0.1",
|
|
98
|
+
"bucket_name": "data-bucket",
|
|
99
|
+
"encryption": "aes256",
|
|
100
|
+
"versioning": true
|
|
101
|
+
}'
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Response (example):
|
|
105
|
+
```json
|
|
106
|
+
{
|
|
107
|
+
"id": "s3-instance-uuid",
|
|
108
|
+
"name": "s3-data",
|
|
109
|
+
"status": "CREATING",
|
|
110
|
+
"bucket_name": "data-bucket",
|
|
111
|
+
"encryption": "aes256",
|
|
112
|
+
"versioning": true,
|
|
113
|
+
"nodes": [
|
|
114
|
+
{
|
|
115
|
+
"id": "node-uuid",
|
|
116
|
+
"ip": "10.20.0.21",
|
|
117
|
+
"port": 9000
|
|
118
|
+
}
|
|
119
|
+
]
|
|
120
|
+
}
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Wait for `status` to become `ACTIVE`, then access via boto3:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
import boto3
|
|
127
|
+
|
|
128
|
+
s3 = boto3.client(
|
|
129
|
+
's3',
|
|
130
|
+
endpoint_url='http://10.20.0.21:9000',
|
|
131
|
+
aws_access_key_id='<access-key>',
|
|
132
|
+
aws_secret_access_key='<secret-key>',
|
|
133
|
+
region_name='us-east-1'
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
s3.put_object(Bucket='data-bucket', Key='test.txt', Body=b'hello')
|
|
137
|
+
obj = s3.get_object(Bucket='data-bucket', Key='test.txt')
|
|
138
|
+
print(obj['Body'].read()) # b'hello'
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## API Reference
|
|
142
|
+
|
|
143
|
+
### Instances
|
|
144
|
+
|
|
145
|
+
| Method | Endpoint | Description |
|
|
146
|
+
|--------|----------|-------------|
|
|
147
|
+
| POST | `/v1/types/s3/instances` | Create instance |
|
|
148
|
+
| GET | `/v1/types/s3/instances` | List instances |
|
|
149
|
+
| GET | `/v1/types/s3/instances/{id}` | Get instance details |
|
|
150
|
+
| PATCH | `/v1/types/s3/instances/{id}` | Update configuration |
|
|
151
|
+
| DELETE | `/v1/types/s3/instances/{id}` | Delete instance |
|
|
152
|
+
|
|
153
|
+
### Instance Fields
|
|
154
|
+
|
|
155
|
+
| Field | Type | Mutable | Description |
|
|
156
|
+
|-------|------|--------|-------------|
|
|
157
|
+
| `name` | string | ✓ | Instance name (unique per project) |
|
|
158
|
+
| `version` | string | ✗ | PaaS version (e.g., "0.0.1") |
|
|
159
|
+
| `bucket_name` | string | ✗ | Default bucket name |
|
|
160
|
+
| `encryption` | string | ✓ | Encryption mode: "none" or "aes256" |
|
|
161
|
+
| `versioning` | boolean | ✓ | Enable S3 object versioning |
|
|
162
|
+
| `mfa_delete` | boolean | ✓ | Require MFA to delete objects |
|
|
163
|
+
| `status` | string | ✗ | Instance status: PENDING, CREATING, ACTIVE, ERROR |
|
|
164
|
+
| `nodes` | array | ✗ | Data-plane nodes with IP, port |
|
|
165
|
+
|
|
166
|
+
### Buckets
|
|
167
|
+
|
|
168
|
+
| Method | Endpoint | Description |
|
|
169
|
+
|--------|----------|-------------|
|
|
170
|
+
| POST | `/v1/types/s3/instances/{id}/buckets` | Create bucket |
|
|
171
|
+
| GET | `/v1/types/s3/instances/{id}/buckets` | List buckets |
|
|
172
|
+
| DELETE | `/v1/types/s3/instances/{id}/buckets/{name}` | Delete bucket |
|
|
173
|
+
|
|
174
|
+
### Users
|
|
175
|
+
|
|
176
|
+
| Method | Endpoint | Description |
|
|
177
|
+
|--------|----------|-------------|
|
|
178
|
+
| POST | `/v1/types/s3/instances/{id}/users` | Create IAM user |
|
|
179
|
+
| GET | `/v1/types/s3/instances/{id}/users` | List users |
|
|
180
|
+
| DELETE | `/v1/types/s3/instances/{id}/users/{name}` | Delete user |
|
|
181
|
+
|
|
182
|
+
### Access Keys
|
|
183
|
+
|
|
184
|
+
| Method | Endpoint | Description |
|
|
185
|
+
|--------|----------|-------------|
|
|
186
|
+
| POST | `/v1/types/s3/instances/{id}/users/{user}/access-keys` | Create key pair |
|
|
187
|
+
| GET | `/v1/types/s3/instances/{id}/users/{user}/access-keys` | List keys |
|
|
188
|
+
| DELETE | `/v1/types/s3/instances/{id}/users/{user}/access-keys/{key}` | Delete key |
|
|
189
|
+
|
|
190
|
+
## Testing
|
|
191
|
+
|
|
192
|
+
### Unit Tests
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
make test
|
|
196
|
+
# or: tox -e py312
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### Lint & Type Checking
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
make lint # ruff check
|
|
203
|
+
make format # ruff format
|
|
204
|
+
make typecheck # mypy
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
### Functional Tests
|
|
208
|
+
|
|
209
|
+
Requires live exordos_core + exordos_metapaas deployment:
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
EXORDOS_ENDPOINT=http://10.20.0.2:11010 \
|
|
213
|
+
EXORDOS_USERNAME=admin \
|
|
214
|
+
EXORDOS_PASSWORD=<pass> \
|
|
215
|
+
METAPAAS_USERNAME=metapaas \
|
|
216
|
+
METAPAAS_PASSWORD=<pass> \
|
|
217
|
+
EXORDOS_S3_CP_URL=http://10.20.0.X:8080 \
|
|
218
|
+
make functional
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Or use the E2E preparation script:
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
python exordos_paas_s3/tests/functional/prepare_env.py \
|
|
225
|
+
--metapaas-dir ../exordos_metapaas \
|
|
226
|
+
--project-dir . \
|
|
227
|
+
--output-dir /tmp/s3-build \
|
|
228
|
+
--endpoint http://10.20.0.2:11010 \
|
|
229
|
+
--username admin \
|
|
230
|
+
--password <admin-pass> \
|
|
231
|
+
--wait-timeout 600
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
This script:
|
|
235
|
+
1. Builds s3aas + metapaas elements
|
|
236
|
+
2. Serves them via HTTP
|
|
237
|
+
3. Installs both to running exordos_core
|
|
238
|
+
4. Outputs environment variables for tests
|
|
239
|
+
5. Waits for all nodes ACTIVE
|
|
240
|
+
|
|
241
|
+
## Architecture
|
|
242
|
+
|
|
243
|
+
### Control Plane (exordos_paas_s3/)
|
|
244
|
+
|
|
245
|
+
- **models.py** — SQLAlchemy: S3Instance, Bucket, User, Policy, AccessKey, Version
|
|
246
|
+
- **controllers.py** — REST endpoints with field-level IAM permissions
|
|
247
|
+
- **iam_config.py** — IAM roles: owner, operator, viewer
|
|
248
|
+
- **migrations/** — Database schema migrations (per-paas independent)
|
|
249
|
+
- **tests/** — Unit + functional test suites
|
|
250
|
+
|
|
251
|
+
### Data Plane (exordos/s3-dp/)
|
|
252
|
+
|
|
253
|
+
- **packer.pkr.hcl** — Builds RustFS-based S3 VM image
|
|
254
|
+
- **conf/** — Configuration templates (rendered per-instance)
|
|
255
|
+
- **systemd/** — Service units (rustfs, minio-gateway, etc.)
|
|
256
|
+
|
|
257
|
+
### Manifest & Build
|
|
258
|
+
|
|
259
|
+
- **exordos/exordos.yaml** — Build config (orchestrates CP wheel + DP image + manifest)
|
|
260
|
+
- **exordos/manifests/s3aas.yaml.j2** — Jinja2 element manifest (deployed to exordos_core)
|
|
261
|
+
|
|
262
|
+
### CI/CD
|
|
263
|
+
|
|
264
|
+
- **.github/workflows/tests.yaml** — Unit tests on Python 3.10/3.12/3.13
|
|
265
|
+
- **.github/workflows/func_tests.yaml** — Full E2E: bootstrap core → build → install → test
|
|
266
|
+
|
|
267
|
+
## Repository Structure
|
|
268
|
+
|
|
269
|
+
```
|
|
270
|
+
.
|
|
271
|
+
├── exordos_paas_s3/ # Control-plane Python package
|
|
272
|
+
│ ├── models.py # SQLAlchemy ORM models
|
|
273
|
+
│ ├── controllers.py # REST API controllers
|
|
274
|
+
│ ├── iam_config.py # IAM setup
|
|
275
|
+
│ ├── migrations/ # Database migrations
|
|
276
|
+
│ └── tests/
|
|
277
|
+
│ ├── test_*.py # Unit tests
|
|
278
|
+
│ └── functional/ # E2E tests
|
|
279
|
+
├── exordos/
|
|
280
|
+
│ ├── exordos.yaml # Build config
|
|
281
|
+
│ ├── manifests/
|
|
282
|
+
│ │ └── s3aas.yaml.j2 # Element manifest template
|
|
283
|
+
│ └── s3-dp/
|
|
284
|
+
│ └── packer.pkr.hcl # Data-plane image config
|
|
285
|
+
├── pyproject.toml # Project metadata + dependencies
|
|
286
|
+
├── tox.ini # Test automation
|
|
287
|
+
├── Makefile # Build targets
|
|
288
|
+
├── .github/workflows/ # CI/CD pipelines
|
|
289
|
+
├── README.md # This file
|
|
290
|
+
└── output/ # Build artifacts (git-ignored)
|
|
291
|
+
├── dist/ # Python wheel
|
|
292
|
+
├── manifests/ # Rendered manifests
|
|
293
|
+
└── images/ # Packer-built images
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
## Configuration
|
|
297
|
+
|
|
298
|
+
### Build Manifest Variables
|
|
299
|
+
|
|
300
|
+
Control how the element image and manifest are rendered:
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
exordos build \
|
|
304
|
+
--manifest-var repository=http://custom-repo.local/exordos-elements \
|
|
305
|
+
--manifest-var index_url=http://custom-repo.local/simple/
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
These override defaults in `exordos/exordos.yaml`.
|
|
309
|
+
|
|
310
|
+
### Instance Configuration
|
|
311
|
+
|
|
312
|
+
When creating an S3 instance, specify:
|
|
313
|
+
|
|
314
|
+
```json
|
|
315
|
+
{
|
|
316
|
+
"name": "my-s3",
|
|
317
|
+
"version": "0.0.1",
|
|
318
|
+
"bucket_name": "default-bucket",
|
|
319
|
+
"encryption": "aes256",
|
|
320
|
+
"versioning": true,
|
|
321
|
+
"mfa_delete": false
|
|
322
|
+
}
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
Configuration is applied to all data-plane nodes in the instance (currently single-node; clustering planned).
|
|
326
|
+
|
|
327
|
+
## Troubleshooting
|
|
328
|
+
|
|
329
|
+
### Instance stuck in CREATING
|
|
330
|
+
|
|
331
|
+
Check PluginReconciler logs on metapaas-cp:
|
|
332
|
+
|
|
333
|
+
```bash
|
|
334
|
+
exordos -e http://10.20.0.2:11010 -u admin -p <pass> \
|
|
335
|
+
cn exec metapaas-cp -- \
|
|
336
|
+
journalctl -u metapaas-plugin-reconciler -f
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
Also check orchestration logs:
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
exordos -e http://10.20.0.2:11010 -u admin -p <pass> \
|
|
343
|
+
cn exec metapaas-cp -- \
|
|
344
|
+
tail -f /var/log/orch-api.log
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
### Cannot connect to S3 API
|
|
348
|
+
|
|
349
|
+
Verify data-plane node is ACTIVE and reachable:
|
|
350
|
+
|
|
351
|
+
```bash
|
|
352
|
+
# Get node IP from instance
|
|
353
|
+
curl -s http://metapaas-cp:8080/v1/types/s3/instances/<id> \
|
|
354
|
+
-H 'Authorization: Bearer <token>' | jq '.nodes[].ip'
|
|
355
|
+
|
|
356
|
+
# Test connectivity
|
|
357
|
+
telnet <ip> 9000
|
|
358
|
+
|
|
359
|
+
# Check S3 health
|
|
360
|
+
curl -s http://<ip>:9000/health
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
### Permissions error (403 Forbidden)
|
|
364
|
+
|
|
365
|
+
Ensure test user has `owner` role in METAPAAS_PROJECT_ID:
|
|
366
|
+
|
|
367
|
+
```bash
|
|
368
|
+
# Query user's roles
|
|
369
|
+
curl -s http://10.20.0.2:11010/v1/iam/projects/4d657461-0000-0000-0000-000000000002/roles \
|
|
370
|
+
-u admin:<pass> | jq '.items[] | select(.user_id=="<your-uuid>")'
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
## Development Guidelines
|
|
374
|
+
|
|
375
|
+
### Adding New Endpoints
|
|
376
|
+
|
|
377
|
+
1. Add model field to `models.py`
|
|
378
|
+
2. Add controller method + permissions to `controllers.py`
|
|
379
|
+
3. Add migration in `migrations/` with UUID name
|
|
380
|
+
4. Test in `tests/functional/`
|
|
381
|
+
|
|
382
|
+
### Read-Only Fields
|
|
383
|
+
|
|
384
|
+
Fields like `status`, `id`, `created_at`, `nodes`, `kind` are read-only and auto-populated. Omit them from CREATE payloads.
|
|
385
|
+
|
|
386
|
+
### IAM Model
|
|
387
|
+
|
|
388
|
+
Permissions are bound to **METAPAAS_PROJECT_ID**, not instance project. Users must have explicit role grant in metapaas project to access any S3 instance.
|
|
389
|
+
|
|
390
|
+
## References
|
|
391
|
+
|
|
392
|
+
- **MetaPaaS Platform:** [../exordos_metapaas/](../exordos_metapaas/)
|
|
393
|
+
- **How to Build New PaaS:** [../exordos_metapaas/HOW_TO_BUILD_NEW_PAAS.md](../exordos_metapaas/HOW_TO_BUILD_NEW_PAAS.md)
|
|
394
|
+
- **Mail-aas Blueprint:** [../exordos_mail/](../exordos_mail/) (reference plugin, simpler than s3aas)
|
|
395
|
+
- **Exordos Core:** https://github.com/exordos/exordos_core
|
|
396
|
+
|
|
397
|
+
## License
|
|
398
|
+
|
|
399
|
+
Proprietary — Exordos
|