soreplicator 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- soreplicator-0.1.0/PKG-INFO +221 -0
- soreplicator-0.1.0/README.md +199 -0
- soreplicator-0.1.0/pyproject.toml +61 -0
- soreplicator-0.1.0/setup.cfg +4 -0
- soreplicator-0.1.0/soreplicator/__init__.py +3 -0
- soreplicator-0.1.0/soreplicator/client/__init__.py +3 -0
- soreplicator-0.1.0/soreplicator/client/cli.py +154 -0
- soreplicator-0.1.0/soreplicator/client/config.py +89 -0
- soreplicator-0.1.0/soreplicator/client/storage.py +225 -0
- soreplicator-0.1.0/soreplicator/server/__init__.py +5 -0
- soreplicator-0.1.0/soreplicator/server/app.py +247 -0
- soreplicator-0.1.0/soreplicator/server/auth.py +41 -0
- soreplicator-0.1.0/soreplicator/server/config.py +74 -0
- soreplicator-0.1.0/soreplicator/server/gateway.py +278 -0
- soreplicator-0.1.0/soreplicator/server/static/app.css +1 -0
- soreplicator-0.1.0/soreplicator/server/static/app.js +65 -0
- soreplicator-0.1.0/soreplicator/server/static/apple-touch-icon.png +0 -0
- soreplicator-0.1.0/soreplicator/server/static/favicon-32.png +0 -0
- soreplicator-0.1.0/soreplicator/server/static/htmx.min.js +1 -0
- soreplicator-0.1.0/soreplicator/server/static/icon-192.png +0 -0
- soreplicator-0.1.0/soreplicator/server/static/icon-512.png +0 -0
- soreplicator-0.1.0/soreplicator/server/static/site.webmanifest +19 -0
- soreplicator-0.1.0/soreplicator/server/static/sore-logo-64.png +0 -0
- soreplicator-0.1.0/soreplicator/server/static/sore_pixel.png +0 -0
- soreplicator-0.1.0/soreplicator/server/templates/admin.html +12 -0
- soreplicator-0.1.0/soreplicator/server/templates/base.html +30 -0
- soreplicator-0.1.0/soreplicator/server/templates/credentials.html +5 -0
- soreplicator-0.1.0/soreplicator/server/templates/error.html +5 -0
- soreplicator-0.1.0/soreplicator/server/templates/fragments/admin_table.html +19 -0
- soreplicator-0.1.0/soreplicator/server/templates/fragments/credentials_panel.html +10 -0
- soreplicator-0.1.0/soreplicator/server/templates/fragments/provisioning_error.html +3 -0
- soreplicator-0.1.0/soreplicator/server/templates/provisioning_error.html +5 -0
- soreplicator-0.1.0/soreplicator/server/templates/setup.html +24 -0
- soreplicator-0.1.0/soreplicator/server/templates.py +133 -0
- soreplicator-0.1.0/soreplicator.egg-info/PKG-INFO +221 -0
- soreplicator-0.1.0/soreplicator.egg-info/SOURCES.txt +41 -0
- soreplicator-0.1.0/soreplicator.egg-info/dependency_links.txt +1 -0
- soreplicator-0.1.0/soreplicator.egg-info/entry_points.txt +3 -0
- soreplicator-0.1.0/soreplicator.egg-info/requires.txt +15 -0
- soreplicator-0.1.0/soreplicator.egg-info/top_level.txt +1 -0
- soreplicator-0.1.0/tests/test_client.py +252 -0
- soreplicator-0.1.0/tests/test_gateway.py +135 -0
- soreplicator-0.1.0/tests/test_server.py +90 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: soreplicator
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Simons Observatory REplicator server and command-line client
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: boto3<2,>=1.40
|
|
9
|
+
Requires-Dist: defusedxml<1,>=0.7.1
|
|
10
|
+
Requires-Dist: fastapi<1,>=0.116
|
|
11
|
+
Requires-Dist: httpx<1,>=0.28
|
|
12
|
+
Requires-Dist: jinja2<4,>=3.1
|
|
13
|
+
Requires-Dist: pydantic-settings<3,>=2.10
|
|
14
|
+
Requires-Dist: python-multipart<1,>=0.0.20
|
|
15
|
+
Requires-Dist: soauth<1,>=0.8.2
|
|
16
|
+
Requires-Dist: socolors<1,>=0.0.8
|
|
17
|
+
Requires-Dist: uvicorn[standard]<1,>=0.35
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest<9,>=8.4; extra == "dev"
|
|
20
|
+
Requires-Dist: pytest-asyncio<2,>=1.1; extra == "dev"
|
|
21
|
+
Requires-Dist: ruff<1,>=0.12; extra == "dev"
|
|
22
|
+
|
|
23
|
+
SORE: Simons Observatory REplicator
|
|
24
|
+
===================================
|
|
25
|
+
|
|
26
|
+
SORE is a tool to ensure that a copy of your critical data is stored on all of our
|
|
27
|
+
HPC machines. You can upload a file to SORE, and within an hour it should be
|
|
28
|
+
replicated across NERSC, Princeton, and SO:UK. The system works by leveraging
|
|
29
|
+
S3: we use Versity's S3 Gateway to allow you to upload data to the NERSC filesystem
|
|
30
|
+
from anywhere in the world, and then a background Globus job ensures data is
|
|
31
|
+
replicated to the other systems. You can only add and delete data that you have
|
|
32
|
+
uploaded; no 'edit' functionality exists. You should also not attempt to
|
|
33
|
+
directly modify data on disk at any of the sites (and this will be impossible
|
|
34
|
+
anyway).
|
|
35
|
+
|
|
36
|
+
To upload data, all you need to do is use the `sore` command-line utility:
|
|
37
|
+
```bash
|
|
38
|
+
sore -i local_file.fits -o testing/2026-01-01/local_file.fits
|
|
39
|
+
```
|
|
40
|
+
The `-i` input file is the path on your local filesystem. The `-o` output
|
|
41
|
+
filename is the position in the global namespace that your file will take.
|
|
42
|
+
Your file will then be uploaded, over the internet, to NERSC, where it will
|
|
43
|
+
appear at `sore/$GITHUB_USERNAME/testing/2026-01-01/local_file.fits`. It will
|
|
44
|
+
shortly be copied to other sites.
|
|
45
|
+
|
|
46
|
+
Developed with the Codex/OpenAI 5.6 generation models.
|
|
47
|
+
|
|
48
|
+
Getting set up
|
|
49
|
+
--------------
|
|
50
|
+
|
|
51
|
+
To get set up, you will first need to install the `soreplicator` python package:
|
|
52
|
+
```bash
|
|
53
|
+
uv pip install soreplicator
|
|
54
|
+
```
|
|
55
|
+
To authenticate with the system, you will need a username and password. These are
|
|
56
|
+
_secrets_ and should never be shared with anyone else. To get your username
|
|
57
|
+
and password, you will need to log into the web interface at
|
|
58
|
+
https://sore.simonsobservatory.org/setup. Here, you will be able to get your username
|
|
59
|
+
and password for the `sore` server. Your username will always be the same
|
|
60
|
+
as your GitHub username.
|
|
61
|
+
|
|
62
|
+
`sore` includes a utility to store these secrets for re-use in a safe location.
|
|
63
|
+
We recommend using this. To set up `sore` you should call:
|
|
64
|
+
```bash
|
|
65
|
+
sore --setup
|
|
66
|
+
```
|
|
67
|
+
This will allow you to paste in your username and password, which will be saved
|
|
68
|
+
in your config location (usually `~/.config/sore.json`) with `600` permissions.
|
|
69
|
+
|
|
70
|
+
Deleting Files
|
|
71
|
+
--------------
|
|
72
|
+
|
|
73
|
+
Sometimes you will want to remove files from the system, because they are old and
|
|
74
|
+
no longer used. You can do this with the `sore` command-line tool:
|
|
75
|
+
```bash
|
|
76
|
+
sore -d testing/2026-01-01/local-file.fits
|
|
77
|
+
```
|
|
78
|
+
Which will ask you to input `y/n` for whether you are sure you want to delete
|
|
79
|
+
this file in the global namespace. Note that you can only change your own files,
|
|
80
|
+
not anyone elses. If you are sure and want to avoid the prompt, you can use
|
|
81
|
+
```bash
|
|
82
|
+
sore -d testing/2026-01-01/local-file.fits -f
|
|
83
|
+
```
|
|
84
|
+
to force-delete without needing to respond to the prompt.
|
|
85
|
+
|
|
86
|
+
Administrator Information
|
|
87
|
+
-------------------------
|
|
88
|
+
|
|
89
|
+
Setting up `sore` is easy. We provide a containerized version of the server. The most
|
|
90
|
+
important thing to realize is that we provide one bucket per user, with the bucket
|
|
91
|
+
name the same as their GitHub username. This allows us to set bucket-level permissions
|
|
92
|
+
for each user, allowing them only `PUT`, `GET`, and `DELETE` access to these buckets,
|
|
93
|
+
with `GET` permissions for all buckets.
|
|
94
|
+
|
|
95
|
+
Included with this repository is a `nginx` configuration file that allows you to mount:
|
|
96
|
+
- Versity's S3 Gateway on /
|
|
97
|
+
- The provided `sore` server on `/setup`.
|
|
98
|
+
- Sharing of directories for `.well-known` challenges.
|
|
99
|
+
|
|
100
|
+
We authenticate users using the Simons Observatory authentication framework `soauth`.
|
|
101
|
+
|
|
102
|
+
To set up `sore`, you will need an administrator credential (ideally not the root
|
|
103
|
+
credential) that allows the creation of buckets and users with various levels of
|
|
104
|
+
permissions. You should set the following environment variables:
|
|
105
|
+
|
|
106
|
+
- `SORE_GATEWAY_USERNAME`: the username of the account that can create other accounts
|
|
107
|
+
- `SORE_GATEWAY_PASSWORD`: the password of the account that can create other accounts
|
|
108
|
+
- `SORE_GATEWAY_REMOTE_LOCATION`: the location of the gateway on the open internet
|
|
109
|
+
- `SORE_GATEWAY_LOCAL_LOCATION`: the location of the gateway on the local network
|
|
110
|
+
- `SORE_ADMIN_GRANT`: the `soauth` grant for administrator priviliges (`sore:admin`)
|
|
111
|
+
- `SORE_USER_GRANT`: the `soauth` grant required for users to create accounts (`sore:user`)
|
|
112
|
+
|
|
113
|
+
We will only communicate with the gateway on the local network and never over the
|
|
114
|
+
open internet. The `REMOTE_LOCATION` is just for providing information in the UI.
|
|
115
|
+
|
|
116
|
+
### Administrator Panels
|
|
117
|
+
|
|
118
|
+
Within the Web UI, you can see information on all of the user accounts that have
|
|
119
|
+
been created to date, and the buckets along with their sizes. This information is
|
|
120
|
+
pulled dynamically from the S3 gateway and is not stored separately for the `sore`
|
|
121
|
+
server so the `sore` server is effectively stateless. You will require the
|
|
122
|
+
`SORE_ADMIN_GRANT` to view this page.
|
|
123
|
+
|
|
124
|
+
Implementation
|
|
125
|
+
--------------
|
|
126
|
+
|
|
127
|
+
This repository contains the complete gateway edge, account service, and client:
|
|
128
|
+
|
|
129
|
+
- `docker-compose.yml` runs VersityGW with a POSIX directory, sidecar metadata,
|
|
130
|
+
IAM, its WebGUI, the SORE server, and nginx.
|
|
131
|
+
- `soreplicator/server/` is a stateless FastAPI service. It authenticates with `soauth`, creates
|
|
132
|
+
or rotates a VersityGW account, creates the matching bucket, and maintains a
|
|
133
|
+
bucket policy that gives every provisioned account read access while preserving
|
|
134
|
+
owner write/delete access.
|
|
135
|
+
- `soreplicator/client/` supplies the `sore` command. Large files use four concurrent S3
|
|
136
|
+
multipart transfers with a terminal progress bar; smaller files use a regular
|
|
137
|
+
PUT. Both paths write conditionally, so an existing object cannot be
|
|
138
|
+
overwritten. Configuration is written atomically with mode `600` and insecure
|
|
139
|
+
remote HTTP endpoints are rejected.
|
|
140
|
+
- The `/setup/admin` view queries users, buckets, object counts, and sizes directly
|
|
141
|
+
from VersityGW. No credentials or inventory are stored in an application database.
|
|
142
|
+
|
|
143
|
+
The downstream Globus replication job is intentionally outside this repository's
|
|
144
|
+
scope; it can consume the gateway's shared POSIX directory independently.
|
|
145
|
+
|
|
146
|
+
### Local stack
|
|
147
|
+
|
|
148
|
+
The included compose topology is a loopback-only development environment:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
cp .env.example .env
|
|
152
|
+
# Change the secrets and SORE_DEVELOPMENT_USERNAME in .env first.
|
|
153
|
+
docker compose up -d --build
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Open the following local endpoints:
|
|
157
|
+
|
|
158
|
+
- SORE setup: `http://sore.localhost:8080/setup/`
|
|
159
|
+
- VersityGW WebGUI through nginx: `http://gateway.localhost:8080/`
|
|
160
|
+
- S3 through nginx: `http://s3.localhost:8080/`
|
|
161
|
+
|
|
162
|
+
The WebGUI is preconfigured to use `s3.localhost` for S3 and `admin.localhost`
|
|
163
|
+
for its admin API. Log in with the gateway administrator values from `.env`.
|
|
164
|
+
Persistent objects, versions, IAM data, and metadata are stored under `./data`
|
|
165
|
+
by default. Set `SORE_DATA_DIRECTORY` to share another host directory.
|
|
166
|
+
|
|
167
|
+
The compose stack deliberately uses the fixed development identity named by
|
|
168
|
+
`SORE_DEVELOPMENT_USERNAME`; nginx publishes only on `127.0.0.1`. Development
|
|
169
|
+
authentication cannot start unless `SORE_DEBUG=true`. Never expose this mode to a
|
|
170
|
+
network.
|
|
171
|
+
|
|
172
|
+
### Production server configuration
|
|
173
|
+
|
|
174
|
+
The Docker image runs as an unprivileged user with a read-only-compatible root
|
|
175
|
+
filesystem. For production, set `SORE_AUTH_MODE=soauth`, `SORE_DEBUG=false`, and
|
|
176
|
+
provide these settings to the server container:
|
|
177
|
+
|
|
178
|
+
| Setting | Purpose |
|
|
179
|
+
| --- | --- |
|
|
180
|
+
| `SORE_GATEWAY_USERNAME` / `SORE_GATEWAY_PASSWORD` | Versity administrator credential |
|
|
181
|
+
| `SORE_GATEWAY_REMOTE_LOCATION` | Public S3 URL displayed to clients |
|
|
182
|
+
| `SORE_GATEWAY_LOCAL_LOCATION` | Private S3 URL used by the server |
|
|
183
|
+
| `SORE_GATEWAY_ADMIN_LOCATION` | Private Versity admin API URL |
|
|
184
|
+
| `SORE_GATEWAY_REGION` | SigV4 region; defaults to `us-east-1` |
|
|
185
|
+
| `SORE_APP_BASE_URL` | Public application URL, including `/setup` |
|
|
186
|
+
| `SORE_AUTHENTICATION_BASE_URL` | SOAuth service URL |
|
|
187
|
+
| `SORE_APP_ID` / `SORE_CLIENT_SECRET` | SOAuth application credential |
|
|
188
|
+
| `SORE_PUBLIC_KEY_FILE` | Mounted SOAuth public key path |
|
|
189
|
+
| `SORE_KEY_PAIR_TYPE` | SOAuth key type; defaults to `Ed25519` |
|
|
190
|
+
| `SORE_ADMIN_GRANT` / `SORE_USER_GRANT` | Required grants |
|
|
191
|
+
|
|
192
|
+
Terminate TLS at nginx or an upstream load balancer, keep the admin API private,
|
|
193
|
+
replace every development secret, and adapt `nginx/nginx.conf` to the production
|
|
194
|
+
hostnames. The `.well-known` directory is mounted read-only for ACME challenges.
|
|
195
|
+
|
|
196
|
+
### Development and verification
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
uv sync --extra dev
|
|
200
|
+
uv run pytest
|
|
201
|
+
uv run ruff check .
|
|
202
|
+
npm install
|
|
203
|
+
npm run build:css
|
|
204
|
+
docker compose config
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Tailwind output and htmx are vendored under `soreplicator/server/static`, so the setup UI does
|
|
208
|
+
not depend on third-party CDNs. The Tailwind palette is the colorblind-friendly
|
|
209
|
+
SO palette from `socolors`.
|
|
210
|
+
|
|
211
|
+
### Releases
|
|
212
|
+
|
|
213
|
+
GitHub Actions runs Ruff and the test suite on Python 3.11–3.13 for pull requests
|
|
214
|
+
and pushes to `main`, and verifies that the committed Tailwind stylesheet is current.
|
|
215
|
+
To publish a release, update the version in `pyproject.toml`, merge it to `main`,
|
|
216
|
+
and publish a GitHub Release.
|
|
217
|
+
|
|
218
|
+
The publishing workflow uses [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
|
|
219
|
+
instead of a stored API token. Before the first release, configure a GitHub Actions
|
|
220
|
+
Trusted Publisher for this repository and the `pypi` environment in the
|
|
221
|
+
`soreplicator` PyPI project.
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
SORE: Simons Observatory REplicator
|
|
2
|
+
===================================
|
|
3
|
+
|
|
4
|
+
SORE is a tool to ensure that a copy of your critical data is stored on all of our
|
|
5
|
+
HPC machines. You can upload a file to SORE, and within an hour it should be
|
|
6
|
+
replicated across NERSC, Princeton, and SO:UK. The system works by leveraging
|
|
7
|
+
S3: we use Versity's S3 Gateway to allow you to upload data to the NERSC filesystem
|
|
8
|
+
from anywhere in the world, and then a background Globus job ensures data is
|
|
9
|
+
replicated to the other systems. You can only add and delete data that you have
|
|
10
|
+
uploaded; no 'edit' functionality exists. You should also not attempt to
|
|
11
|
+
directly modify data on disk at any of the sites (and this will be impossible
|
|
12
|
+
anyway).
|
|
13
|
+
|
|
14
|
+
To upload data, all you need to do is use the `sore` command-line utility:
|
|
15
|
+
```bash
|
|
16
|
+
sore -i local_file.fits -o testing/2026-01-01/local_file.fits
|
|
17
|
+
```
|
|
18
|
+
The `-i` input file is the path on your local filesystem. The `-o` output
|
|
19
|
+
filename is the position in the global namespace that your file will take.
|
|
20
|
+
Your file will then be uploaded, over the internet, to NERSC, where it will
|
|
21
|
+
appear at `sore/$GITHUB_USERNAME/testing/2026-01-01/local_file.fits`. It will
|
|
22
|
+
shortly be copied to other sites.
|
|
23
|
+
|
|
24
|
+
Developed with the Codex/OpenAI 5.6 generation models.
|
|
25
|
+
|
|
26
|
+
Getting set up
|
|
27
|
+
--------------
|
|
28
|
+
|
|
29
|
+
To get set up, you will first need to install the `soreplicator` python package:
|
|
30
|
+
```bash
|
|
31
|
+
uv pip install soreplicator
|
|
32
|
+
```
|
|
33
|
+
To authenticate with the system, you will need a username and password. These are
|
|
34
|
+
_secrets_ and should never be shared with anyone else. To get your username
|
|
35
|
+
and password, you will need to log into the web interface at
|
|
36
|
+
https://sore.simonsobservatory.org/setup. Here, you will be able to get your username
|
|
37
|
+
and password for the `sore` server. Your username will always be the same
|
|
38
|
+
as your GitHub username.
|
|
39
|
+
|
|
40
|
+
`sore` includes a utility to store these secrets for re-use in a safe location.
|
|
41
|
+
We recommend using this. To set up `sore` you should call:
|
|
42
|
+
```bash
|
|
43
|
+
sore --setup
|
|
44
|
+
```
|
|
45
|
+
This will allow you to paste in your username and password, which will be saved
|
|
46
|
+
in your config location (usually `~/.config/sore.json`) with `600` permissions.
|
|
47
|
+
|
|
48
|
+
Deleting Files
|
|
49
|
+
--------------
|
|
50
|
+
|
|
51
|
+
Sometimes you will want to remove files from the system, because they are old and
|
|
52
|
+
no longer used. You can do this with the `sore` command-line tool:
|
|
53
|
+
```bash
|
|
54
|
+
sore -d testing/2026-01-01/local-file.fits
|
|
55
|
+
```
|
|
56
|
+
Which will ask you to input `y/n` for whether you are sure you want to delete
|
|
57
|
+
this file in the global namespace. Note that you can only change your own files,
|
|
58
|
+
not anyone elses. If you are sure and want to avoid the prompt, you can use
|
|
59
|
+
```bash
|
|
60
|
+
sore -d testing/2026-01-01/local-file.fits -f
|
|
61
|
+
```
|
|
62
|
+
to force-delete without needing to respond to the prompt.
|
|
63
|
+
|
|
64
|
+
Administrator Information
|
|
65
|
+
-------------------------
|
|
66
|
+
|
|
67
|
+
Setting up `sore` is easy. We provide a containerized version of the server. The most
|
|
68
|
+
important thing to realize is that we provide one bucket per user, with the bucket
|
|
69
|
+
name the same as their GitHub username. This allows us to set bucket-level permissions
|
|
70
|
+
for each user, allowing them only `PUT`, `GET`, and `DELETE` access to these buckets,
|
|
71
|
+
with `GET` permissions for all buckets.
|
|
72
|
+
|
|
73
|
+
Included with this repository is a `nginx` configuration file that allows you to mount:
|
|
74
|
+
- Versity's S3 Gateway on /
|
|
75
|
+
- The provided `sore` server on `/setup`.
|
|
76
|
+
- Sharing of directories for `.well-known` challenges.
|
|
77
|
+
|
|
78
|
+
We authenticate users using the Simons Observatory authentication framework `soauth`.
|
|
79
|
+
|
|
80
|
+
To set up `sore`, you will need an administrator credential (ideally not the root
|
|
81
|
+
credential) that allows the creation of buckets and users with various levels of
|
|
82
|
+
permissions. You should set the following environment variables:
|
|
83
|
+
|
|
84
|
+
- `SORE_GATEWAY_USERNAME`: the username of the account that can create other accounts
|
|
85
|
+
- `SORE_GATEWAY_PASSWORD`: the password of the account that can create other accounts
|
|
86
|
+
- `SORE_GATEWAY_REMOTE_LOCATION`: the location of the gateway on the open internet
|
|
87
|
+
- `SORE_GATEWAY_LOCAL_LOCATION`: the location of the gateway on the local network
|
|
88
|
+
- `SORE_ADMIN_GRANT`: the `soauth` grant for administrator priviliges (`sore:admin`)
|
|
89
|
+
- `SORE_USER_GRANT`: the `soauth` grant required for users to create accounts (`sore:user`)
|
|
90
|
+
|
|
91
|
+
We will only communicate with the gateway on the local network and never over the
|
|
92
|
+
open internet. The `REMOTE_LOCATION` is just for providing information in the UI.
|
|
93
|
+
|
|
94
|
+
### Administrator Panels
|
|
95
|
+
|
|
96
|
+
Within the Web UI, you can see information on all of the user accounts that have
|
|
97
|
+
been created to date, and the buckets along with their sizes. This information is
|
|
98
|
+
pulled dynamically from the S3 gateway and is not stored separately for the `sore`
|
|
99
|
+
server so the `sore` server is effectively stateless. You will require the
|
|
100
|
+
`SORE_ADMIN_GRANT` to view this page.
|
|
101
|
+
|
|
102
|
+
Implementation
|
|
103
|
+
--------------
|
|
104
|
+
|
|
105
|
+
This repository contains the complete gateway edge, account service, and client:
|
|
106
|
+
|
|
107
|
+
- `docker-compose.yml` runs VersityGW with a POSIX directory, sidecar metadata,
|
|
108
|
+
IAM, its WebGUI, the SORE server, and nginx.
|
|
109
|
+
- `soreplicator/server/` is a stateless FastAPI service. It authenticates with `soauth`, creates
|
|
110
|
+
or rotates a VersityGW account, creates the matching bucket, and maintains a
|
|
111
|
+
bucket policy that gives every provisioned account read access while preserving
|
|
112
|
+
owner write/delete access.
|
|
113
|
+
- `soreplicator/client/` supplies the `sore` command. Large files use four concurrent S3
|
|
114
|
+
multipart transfers with a terminal progress bar; smaller files use a regular
|
|
115
|
+
PUT. Both paths write conditionally, so an existing object cannot be
|
|
116
|
+
overwritten. Configuration is written atomically with mode `600` and insecure
|
|
117
|
+
remote HTTP endpoints are rejected.
|
|
118
|
+
- The `/setup/admin` view queries users, buckets, object counts, and sizes directly
|
|
119
|
+
from VersityGW. No credentials or inventory are stored in an application database.
|
|
120
|
+
|
|
121
|
+
The downstream Globus replication job is intentionally outside this repository's
|
|
122
|
+
scope; it can consume the gateway's shared POSIX directory independently.
|
|
123
|
+
|
|
124
|
+
### Local stack
|
|
125
|
+
|
|
126
|
+
The included compose topology is a loopback-only development environment:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
cp .env.example .env
|
|
130
|
+
# Change the secrets and SORE_DEVELOPMENT_USERNAME in .env first.
|
|
131
|
+
docker compose up -d --build
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Open the following local endpoints:
|
|
135
|
+
|
|
136
|
+
- SORE setup: `http://sore.localhost:8080/setup/`
|
|
137
|
+
- VersityGW WebGUI through nginx: `http://gateway.localhost:8080/`
|
|
138
|
+
- S3 through nginx: `http://s3.localhost:8080/`
|
|
139
|
+
|
|
140
|
+
The WebGUI is preconfigured to use `s3.localhost` for S3 and `admin.localhost`
|
|
141
|
+
for its admin API. Log in with the gateway administrator values from `.env`.
|
|
142
|
+
Persistent objects, versions, IAM data, and metadata are stored under `./data`
|
|
143
|
+
by default. Set `SORE_DATA_DIRECTORY` to share another host directory.
|
|
144
|
+
|
|
145
|
+
The compose stack deliberately uses the fixed development identity named by
|
|
146
|
+
`SORE_DEVELOPMENT_USERNAME`; nginx publishes only on `127.0.0.1`. Development
|
|
147
|
+
authentication cannot start unless `SORE_DEBUG=true`. Never expose this mode to a
|
|
148
|
+
network.
|
|
149
|
+
|
|
150
|
+
### Production server configuration
|
|
151
|
+
|
|
152
|
+
The Docker image runs as an unprivileged user with a read-only-compatible root
|
|
153
|
+
filesystem. For production, set `SORE_AUTH_MODE=soauth`, `SORE_DEBUG=false`, and
|
|
154
|
+
provide these settings to the server container:
|
|
155
|
+
|
|
156
|
+
| Setting | Purpose |
|
|
157
|
+
| --- | --- |
|
|
158
|
+
| `SORE_GATEWAY_USERNAME` / `SORE_GATEWAY_PASSWORD` | Versity administrator credential |
|
|
159
|
+
| `SORE_GATEWAY_REMOTE_LOCATION` | Public S3 URL displayed to clients |
|
|
160
|
+
| `SORE_GATEWAY_LOCAL_LOCATION` | Private S3 URL used by the server |
|
|
161
|
+
| `SORE_GATEWAY_ADMIN_LOCATION` | Private Versity admin API URL |
|
|
162
|
+
| `SORE_GATEWAY_REGION` | SigV4 region; defaults to `us-east-1` |
|
|
163
|
+
| `SORE_APP_BASE_URL` | Public application URL, including `/setup` |
|
|
164
|
+
| `SORE_AUTHENTICATION_BASE_URL` | SOAuth service URL |
|
|
165
|
+
| `SORE_APP_ID` / `SORE_CLIENT_SECRET` | SOAuth application credential |
|
|
166
|
+
| `SORE_PUBLIC_KEY_FILE` | Mounted SOAuth public key path |
|
|
167
|
+
| `SORE_KEY_PAIR_TYPE` | SOAuth key type; defaults to `Ed25519` |
|
|
168
|
+
| `SORE_ADMIN_GRANT` / `SORE_USER_GRANT` | Required grants |
|
|
169
|
+
|
|
170
|
+
Terminate TLS at nginx or an upstream load balancer, keep the admin API private,
|
|
171
|
+
replace every development secret, and adapt `nginx/nginx.conf` to the production
|
|
172
|
+
hostnames. The `.well-known` directory is mounted read-only for ACME challenges.
|
|
173
|
+
|
|
174
|
+
### Development and verification
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
uv sync --extra dev
|
|
178
|
+
uv run pytest
|
|
179
|
+
uv run ruff check .
|
|
180
|
+
npm install
|
|
181
|
+
npm run build:css
|
|
182
|
+
docker compose config
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Tailwind output and htmx are vendored under `soreplicator/server/static`, so the setup UI does
|
|
186
|
+
not depend on third-party CDNs. The Tailwind palette is the colorblind-friendly
|
|
187
|
+
SO palette from `socolors`.
|
|
188
|
+
|
|
189
|
+
### Releases
|
|
190
|
+
|
|
191
|
+
GitHub Actions runs Ruff and the test suite on Python 3.11–3.13 for pull requests
|
|
192
|
+
and pushes to `main`, and verifies that the committed Tailwind stylesheet is current.
|
|
193
|
+
To publish a release, update the version in `pyproject.toml`, merge it to `main`,
|
|
194
|
+
and publish a GitHub Release.
|
|
195
|
+
|
|
196
|
+
The publishing workflow uses [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
|
|
197
|
+
instead of a stored API token. Before the first release, configure a GitHub Actions
|
|
198
|
+
Trusted Publisher for this repository and the `pypi` environment in the
|
|
199
|
+
`soreplicator` PyPI project.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "soreplicator"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Simons Observatory REplicator server and command-line client"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"boto3>=1.40,<2",
|
|
14
|
+
"defusedxml>=0.7.1,<1",
|
|
15
|
+
"fastapi>=0.116,<1",
|
|
16
|
+
"httpx>=0.28,<1",
|
|
17
|
+
"jinja2>=3.1,<4",
|
|
18
|
+
"pydantic-settings>=2.10,<3",
|
|
19
|
+
"python-multipart>=0.0.20,<1",
|
|
20
|
+
"soauth>=0.8.2,<1",
|
|
21
|
+
"socolors>=0.0.8,<1",
|
|
22
|
+
"uvicorn[standard]>=0.35,<1",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = [
|
|
27
|
+
"pytest>=8.4,<9",
|
|
28
|
+
"pytest-asyncio>=1.1,<2",
|
|
29
|
+
"ruff>=0.12,<1",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
sore = "soreplicator.client.cli:main"
|
|
34
|
+
sore-server = "soreplicator.server.app:run"
|
|
35
|
+
|
|
36
|
+
[tool.setuptools]
|
|
37
|
+
packages = ["soreplicator", "soreplicator.client", "soreplicator.server"]
|
|
38
|
+
include-package-data = false
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.package-data]
|
|
41
|
+
"soreplicator.server" = [
|
|
42
|
+
"static/app.css",
|
|
43
|
+
"static/app.js",
|
|
44
|
+
"static/htmx.min.js",
|
|
45
|
+
"static/*.png",
|
|
46
|
+
"static/*.webmanifest",
|
|
47
|
+
"templates/*.html",
|
|
48
|
+
"templates/fragments/*.html",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
|
53
|
+
addopts = "-q"
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 88
|
|
57
|
+
target-version = "py311"
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint]
|
|
60
|
+
extend-select = ["I", "B", "S", "UP"]
|
|
61
|
+
ignore = ["S101"]
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""The ``sore`` command-line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import getpass
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TextIO
|
|
10
|
+
from urllib.parse import urlsplit
|
|
11
|
+
|
|
12
|
+
from pydantic import ValidationError
|
|
13
|
+
|
|
14
|
+
from soreplicator.client.config import (
|
|
15
|
+
ClientConfig,
|
|
16
|
+
config_path,
|
|
17
|
+
load_config,
|
|
18
|
+
save_config,
|
|
19
|
+
)
|
|
20
|
+
from soreplicator.client.storage import StorageError, delete, upload
|
|
21
|
+
|
|
22
|
+
DEFAULT_ENDPOINT = "https://sore.simonsobservatory.org"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _human_bytes(value: int) -> str:
|
|
26
|
+
size = float(value)
|
|
27
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
28
|
+
if size < 1000 or unit == "TB":
|
|
29
|
+
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
|
|
30
|
+
size /= 1000
|
|
31
|
+
return f"{size:.1f} TB"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ProgressBar:
|
|
35
|
+
def __init__(self, total: int, stream: TextIO = sys.stderr, width: int = 28):
|
|
36
|
+
self.total = total
|
|
37
|
+
self.stream = stream
|
|
38
|
+
self.width = width
|
|
39
|
+
self.enabled = stream.isatty()
|
|
40
|
+
self.rendered = False
|
|
41
|
+
if self.enabled:
|
|
42
|
+
self.update(0, total)
|
|
43
|
+
|
|
44
|
+
def update(self, uploaded: int, total: int) -> None:
|
|
45
|
+
if not self.enabled:
|
|
46
|
+
return
|
|
47
|
+
fraction = 1.0 if total == 0 else min(1.0, uploaded / total)
|
|
48
|
+
filled = round(self.width * fraction)
|
|
49
|
+
bar = "#" * filled + "-" * (self.width - filled)
|
|
50
|
+
self.stream.write(
|
|
51
|
+
f"\rUploading [{bar}] {fraction:6.1%} "
|
|
52
|
+
f"{_human_bytes(uploaded)} / {_human_bytes(total)}"
|
|
53
|
+
)
|
|
54
|
+
self.stream.flush()
|
|
55
|
+
self.rendered = True
|
|
56
|
+
|
|
57
|
+
def close(self) -> None:
|
|
58
|
+
if self.rendered:
|
|
59
|
+
self.stream.write("\n")
|
|
60
|
+
self.stream.flush()
|
|
61
|
+
self.rendered = False
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
65
|
+
parser = argparse.ArgumentParser(
|
|
66
|
+
prog="sore", description="Upload immutable files to the SORE replica service."
|
|
67
|
+
)
|
|
68
|
+
actions = parser.add_mutually_exclusive_group(required=True)
|
|
69
|
+
actions.add_argument(
|
|
70
|
+
"--setup", action="store_true", help="store gateway credentials"
|
|
71
|
+
)
|
|
72
|
+
actions.add_argument("-i", "--input", type=Path, help="local file to upload")
|
|
73
|
+
actions.add_argument(
|
|
74
|
+
"-d", "--delete", metavar="REMOTE_PATH", help="delete an object"
|
|
75
|
+
)
|
|
76
|
+
parser.add_argument("-o", "--output", help="remote path for an upload")
|
|
77
|
+
parser.add_argument(
|
|
78
|
+
"-f", "--force", action="store_true", help="skip delete confirmation"
|
|
79
|
+
)
|
|
80
|
+
parser.add_argument(
|
|
81
|
+
"--config", type=Path, default=config_path(), help=argparse.SUPPRESS
|
|
82
|
+
)
|
|
83
|
+
return parser
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _setup(path: Path) -> int:
|
|
87
|
+
endpoint = (
|
|
88
|
+
input(f"Gateway endpoint [{DEFAULT_ENDPOINT}]: ").strip() or DEFAULT_ENDPOINT
|
|
89
|
+
)
|
|
90
|
+
parsed = urlsplit(endpoint)
|
|
91
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
92
|
+
raise ValueError("endpoint must be an absolute HTTP(S) URL")
|
|
93
|
+
hostname = parsed.hostname or ""
|
|
94
|
+
is_loopback = hostname in {"localhost", "127.0.0.1", "::1"} or hostname.endswith(
|
|
95
|
+
".localhost"
|
|
96
|
+
)
|
|
97
|
+
if parsed.scheme != "https" and not is_loopback:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
"credentials may only be sent over HTTPS (except on localhost)"
|
|
100
|
+
)
|
|
101
|
+
username = input("Username: ").strip()
|
|
102
|
+
password = getpass.getpass("Password: ")
|
|
103
|
+
if not password:
|
|
104
|
+
raise ValueError("password cannot be empty")
|
|
105
|
+
destination = save_config(
|
|
106
|
+
ClientConfig(endpoint=endpoint, username=username, password=password), path
|
|
107
|
+
)
|
|
108
|
+
print(f"Credentials saved to {destination} with mode 600.")
|
|
109
|
+
return 0
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _confirm_delete(remote_path: str) -> bool:
|
|
113
|
+
answer = input(f"Delete {remote_path!r} permanently? [y/N] ").strip().lower()
|
|
114
|
+
return answer in {"y", "yes"}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def main(argv: list[str] | None = None) -> int:
|
|
118
|
+
parser = build_parser()
|
|
119
|
+
args = parser.parse_args(argv)
|
|
120
|
+
try:
|
|
121
|
+
if args.setup:
|
|
122
|
+
if args.output or args.force:
|
|
123
|
+
parser.error("--setup cannot be combined with -o or -f")
|
|
124
|
+
return _setup(args.config)
|
|
125
|
+
|
|
126
|
+
config = load_config(args.config)
|
|
127
|
+
if args.input:
|
|
128
|
+
if not args.output:
|
|
129
|
+
parser.error("-o/--output is required with -i/--input")
|
|
130
|
+
if args.force:
|
|
131
|
+
parser.error("-f/--force is only valid with -d/--delete")
|
|
132
|
+
progress = ProgressBar(args.input.stat().st_size)
|
|
133
|
+
try:
|
|
134
|
+
upload(config, args.input, args.output, progress.update)
|
|
135
|
+
finally:
|
|
136
|
+
progress.close()
|
|
137
|
+
print(f"Uploaded {args.input} to sore/{config.username}/{args.output}")
|
|
138
|
+
return 0
|
|
139
|
+
|
|
140
|
+
if args.output:
|
|
141
|
+
parser.error("-o/--output is only valid with -i/--input")
|
|
142
|
+
if not args.force and not _confirm_delete(args.delete):
|
|
143
|
+
print("Delete cancelled.")
|
|
144
|
+
return 0
|
|
145
|
+
delete(config, args.delete)
|
|
146
|
+
print(f"Deleted sore/{config.username}/{args.delete}")
|
|
147
|
+
return 0
|
|
148
|
+
except (OSError, PermissionError, ValueError, ValidationError, StorageError) as exc:
|
|
149
|
+
print(f"sore: {exc}", file=sys.stderr)
|
|
150
|
+
return 1
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
if __name__ == "__main__":
|
|
154
|
+
raise SystemExit(main())
|