soreplicator 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. soreplicator-0.1.0/PKG-INFO +221 -0
  2. soreplicator-0.1.0/README.md +199 -0
  3. soreplicator-0.1.0/pyproject.toml +61 -0
  4. soreplicator-0.1.0/setup.cfg +4 -0
  5. soreplicator-0.1.0/soreplicator/__init__.py +3 -0
  6. soreplicator-0.1.0/soreplicator/client/__init__.py +3 -0
  7. soreplicator-0.1.0/soreplicator/client/cli.py +154 -0
  8. soreplicator-0.1.0/soreplicator/client/config.py +89 -0
  9. soreplicator-0.1.0/soreplicator/client/storage.py +225 -0
  10. soreplicator-0.1.0/soreplicator/server/__init__.py +5 -0
  11. soreplicator-0.1.0/soreplicator/server/app.py +247 -0
  12. soreplicator-0.1.0/soreplicator/server/auth.py +41 -0
  13. soreplicator-0.1.0/soreplicator/server/config.py +74 -0
  14. soreplicator-0.1.0/soreplicator/server/gateway.py +278 -0
  15. soreplicator-0.1.0/soreplicator/server/static/app.css +1 -0
  16. soreplicator-0.1.0/soreplicator/server/static/app.js +65 -0
  17. soreplicator-0.1.0/soreplicator/server/static/apple-touch-icon.png +0 -0
  18. soreplicator-0.1.0/soreplicator/server/static/favicon-32.png +0 -0
  19. soreplicator-0.1.0/soreplicator/server/static/htmx.min.js +1 -0
  20. soreplicator-0.1.0/soreplicator/server/static/icon-192.png +0 -0
  21. soreplicator-0.1.0/soreplicator/server/static/icon-512.png +0 -0
  22. soreplicator-0.1.0/soreplicator/server/static/site.webmanifest +19 -0
  23. soreplicator-0.1.0/soreplicator/server/static/sore-logo-64.png +0 -0
  24. soreplicator-0.1.0/soreplicator/server/static/sore_pixel.png +0 -0
  25. soreplicator-0.1.0/soreplicator/server/templates/admin.html +12 -0
  26. soreplicator-0.1.0/soreplicator/server/templates/base.html +30 -0
  27. soreplicator-0.1.0/soreplicator/server/templates/credentials.html +5 -0
  28. soreplicator-0.1.0/soreplicator/server/templates/error.html +5 -0
  29. soreplicator-0.1.0/soreplicator/server/templates/fragments/admin_table.html +19 -0
  30. soreplicator-0.1.0/soreplicator/server/templates/fragments/credentials_panel.html +10 -0
  31. soreplicator-0.1.0/soreplicator/server/templates/fragments/provisioning_error.html +3 -0
  32. soreplicator-0.1.0/soreplicator/server/templates/provisioning_error.html +5 -0
  33. soreplicator-0.1.0/soreplicator/server/templates/setup.html +24 -0
  34. soreplicator-0.1.0/soreplicator/server/templates.py +133 -0
  35. soreplicator-0.1.0/soreplicator.egg-info/PKG-INFO +221 -0
  36. soreplicator-0.1.0/soreplicator.egg-info/SOURCES.txt +41 -0
  37. soreplicator-0.1.0/soreplicator.egg-info/dependency_links.txt +1 -0
  38. soreplicator-0.1.0/soreplicator.egg-info/entry_points.txt +3 -0
  39. soreplicator-0.1.0/soreplicator.egg-info/requires.txt +15 -0
  40. soreplicator-0.1.0/soreplicator.egg-info/top_level.txt +1 -0
  41. soreplicator-0.1.0/tests/test_client.py +252 -0
  42. soreplicator-0.1.0/tests/test_gateway.py +135 -0
  43. soreplicator-0.1.0/tests/test_server.py +90 -0
@@ -0,0 +1,221 @@
1
+ Metadata-Version: 2.4
2
+ Name: soreplicator
3
+ Version: 0.1.0
4
+ Summary: Simons Observatory REplicator server and command-line client
5
+ License-Expression: MIT
6
+ Requires-Python: >=3.11
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: boto3<2,>=1.40
9
+ Requires-Dist: defusedxml<1,>=0.7.1
10
+ Requires-Dist: fastapi<1,>=0.116
11
+ Requires-Dist: httpx<1,>=0.28
12
+ Requires-Dist: jinja2<4,>=3.1
13
+ Requires-Dist: pydantic-settings<3,>=2.10
14
+ Requires-Dist: python-multipart<1,>=0.0.20
15
+ Requires-Dist: soauth<1,>=0.8.2
16
+ Requires-Dist: socolors<1,>=0.0.8
17
+ Requires-Dist: uvicorn[standard]<1,>=0.35
18
+ Provides-Extra: dev
19
+ Requires-Dist: pytest<9,>=8.4; extra == "dev"
20
+ Requires-Dist: pytest-asyncio<2,>=1.1; extra == "dev"
21
+ Requires-Dist: ruff<1,>=0.12; extra == "dev"
22
+
23
+ SORE: Simons Observatory REplicator
24
+ ===================================
25
+
26
+ SORE is a tool to ensure that a copy of your critical data is stored on all of our
27
+ HPC machines. You can upload a file to SORE, and within an hour it should be
28
+ replicated across NERSC, Princeton, and SO:UK. The system works by leveraging
29
+ S3: we use Versity's S3 Gateway to allow you to upload data to the NERSC filesystem
30
+ from anywhere in the world, and then a background Globus job ensures data is
31
+ replicated to the other systems. You can only add and delete data that you have
32
+ uploaded; no 'edit' functionality exists. You should also not attempt to
33
+ directly modify data on disk at any of the sites (and this will be impossible
34
+ anyway).
35
+
36
+ To upload data, all you need to do is use the `sore` command-line utility:
37
+ ```bash
38
+ sore -i local_file.fits -o testing/2026-01-01/local_file.fits
39
+ ```
40
+ The `-i` input file is the path on your local filesystem. The `-o` output
41
+ filename is the position in the global namespace that your file will take.
42
+ Your file will then be uploaded, over the internet, to NERSC, where it will
43
+ appear at `sore/$GITHUB_USERNAME/testing/2026-01-01/local_file.fits`. It will
44
+ shortly be copied to other sites.
45
+
46
+ Developed with the Codex/OpenAI 5.6 generation models.
47
+
48
+ Getting set up
49
+ --------------
50
+
51
+ To get set up, you will first need to install the `soreplicator` python package:
52
+ ```bash
53
+ uv pip install soreplicator
54
+ ```
55
+ To authenticate with the system, you will need a username and password. These are
56
+ _secrets_ and should never be shared with anyone else. To get your username
57
+ and password, you will need to log into the web interface at
58
+ https://sore.simonsobservatory.org/setup. Here, you will be able to get your username
59
+ and password for the `sore` server. Your username will always be the same
60
+ as your GitHub username.
61
+
62
+ `sore` includes a utility to store these secrets for re-use in a safe location.
63
+ We recommend using this. To set up `sore` you should call:
64
+ ```bash
65
+ sore --setup
66
+ ```
67
+ This will allow you to paste in your username and password, which will be saved
68
+ in your config location (usually `~/.config/sore.json`) with `600` permissions.
69
+
70
+ Deleting Files
71
+ --------------
72
+
73
+ Sometimes you will want to remove files from the system, because they are old and
74
+ no longer used. You can do this with the `sore` command-line tool:
75
+ ```bash
76
+ sore -d testing/2026-01-01/local-file.fits
77
+ ```
78
+ Which will ask you to input `y/n` for whether you are sure you want to delete
79
+ this file in the global namespace. Note that you can only change your own files,
80
+ not anyone elses. If you are sure and want to avoid the prompt, you can use
81
+ ```bash
82
+ sore -d testing/2026-01-01/local-file.fits -f
83
+ ```
84
+ to force-delete without needing to respond to the prompt.
85
+
86
+ Administrator Information
87
+ -------------------------
88
+
89
+ Setting up `sore` is easy. We provide a containerized version of the server. The most
90
+ important thing to realize is that we provide one bucket per user, with the bucket
91
+ name the same as their GitHub username. This allows us to set bucket-level permissions
92
+ for each user, allowing them only `PUT`, `GET`, and `DELETE` access to these buckets,
93
+ with `GET` permissions for all buckets.
94
+
95
+ Included with this repository is a `nginx` configuration file that allows you to mount:
96
+ - Versity's S3 Gateway on /
97
+ - The provided `sore` server on `/setup`.
98
+ - Sharing of directories for `.well-known` challenges.
99
+
100
+ We authenticate users using the Simons Observatory authentication framework `soauth`.
101
+
102
+ To set up `sore`, you will need an administrator credential (ideally not the root
103
+ credential) that allows the creation of buckets and users with various levels of
104
+ permissions. You should set the following environment variables:
105
+
106
+ - `SORE_GATEWAY_USERNAME`: the username of the account that can create other accounts
107
+ - `SORE_GATEWAY_PASSWORD`: the password of the account that can create other accounts
108
+ - `SORE_GATEWAY_REMOTE_LOCATION`: the location of the gateway on the open internet
109
+ - `SORE_GATEWAY_LOCAL_LOCATION`: the location of the gateway on the local network
110
+ - `SORE_ADMIN_GRANT`: the `soauth` grant for administrator priviliges (`sore:admin`)
111
+ - `SORE_USER_GRANT`: the `soauth` grant required for users to create accounts (`sore:user`)
112
+
113
+ We will only communicate with the gateway on the local network and never over the
114
+ open internet. The `REMOTE_LOCATION` is just for providing information in the UI.
115
+
116
+ ### Administrator Panels
117
+
118
+ Within the Web UI, you can see information on all of the user accounts that have
119
+ been created to date, and the buckets along with their sizes. This information is
120
+ pulled dynamically from the S3 gateway and is not stored separately for the `sore`
121
+ server so the `sore` server is effectively stateless. You will require the
122
+ `SORE_ADMIN_GRANT` to view this page.
123
+
124
+ Implementation
125
+ --------------
126
+
127
+ This repository contains the complete gateway edge, account service, and client:
128
+
129
+ - `docker-compose.yml` runs VersityGW with a POSIX directory, sidecar metadata,
130
+ IAM, its WebGUI, the SORE server, and nginx.
131
+ - `soreplicator/server/` is a stateless FastAPI service. It authenticates with `soauth`, creates
132
+ or rotates a VersityGW account, creates the matching bucket, and maintains a
133
+ bucket policy that gives every provisioned account read access while preserving
134
+ owner write/delete access.
135
+ - `soreplicator/client/` supplies the `sore` command. Large files use four concurrent S3
136
+ multipart transfers with a terminal progress bar; smaller files use a regular
137
+ PUT. Both paths write conditionally, so an existing object cannot be
138
+ overwritten. Configuration is written atomically with mode `600` and insecure
139
+ remote HTTP endpoints are rejected.
140
+ - The `/setup/admin` view queries users, buckets, object counts, and sizes directly
141
+ from VersityGW. No credentials or inventory are stored in an application database.
142
+
143
+ The downstream Globus replication job is intentionally outside this repository's
144
+ scope; it can consume the gateway's shared POSIX directory independently.
145
+
146
+ ### Local stack
147
+
148
+ The included compose topology is a loopback-only development environment:
149
+
150
+ ```bash
151
+ cp .env.example .env
152
+ # Change the secrets and SORE_DEVELOPMENT_USERNAME in .env first.
153
+ docker compose up -d --build
154
+ ```
155
+
156
+ Open the following local endpoints:
157
+
158
+ - SORE setup: `http://sore.localhost:8080/setup/`
159
+ - VersityGW WebGUI through nginx: `http://gateway.localhost:8080/`
160
+ - S3 through nginx: `http://s3.localhost:8080/`
161
+
162
+ The WebGUI is preconfigured to use `s3.localhost` for S3 and `admin.localhost`
163
+ for its admin API. Log in with the gateway administrator values from `.env`.
164
+ Persistent objects, versions, IAM data, and metadata are stored under `./data`
165
+ by default. Set `SORE_DATA_DIRECTORY` to share another host directory.
166
+
167
+ The compose stack deliberately uses the fixed development identity named by
168
+ `SORE_DEVELOPMENT_USERNAME`; nginx publishes only on `127.0.0.1`. Development
169
+ authentication cannot start unless `SORE_DEBUG=true`. Never expose this mode to a
170
+ network.
171
+
172
+ ### Production server configuration
173
+
174
+ The Docker image runs as an unprivileged user with a read-only-compatible root
175
+ filesystem. For production, set `SORE_AUTH_MODE=soauth`, `SORE_DEBUG=false`, and
176
+ provide these settings to the server container:
177
+
178
+ | Setting | Purpose |
179
+ | --- | --- |
180
+ | `SORE_GATEWAY_USERNAME` / `SORE_GATEWAY_PASSWORD` | Versity administrator credential |
181
+ | `SORE_GATEWAY_REMOTE_LOCATION` | Public S3 URL displayed to clients |
182
+ | `SORE_GATEWAY_LOCAL_LOCATION` | Private S3 URL used by the server |
183
+ | `SORE_GATEWAY_ADMIN_LOCATION` | Private Versity admin API URL |
184
+ | `SORE_GATEWAY_REGION` | SigV4 region; defaults to `us-east-1` |
185
+ | `SORE_APP_BASE_URL` | Public application URL, including `/setup` |
186
+ | `SORE_AUTHENTICATION_BASE_URL` | SOAuth service URL |
187
+ | `SORE_APP_ID` / `SORE_CLIENT_SECRET` | SOAuth application credential |
188
+ | `SORE_PUBLIC_KEY_FILE` | Mounted SOAuth public key path |
189
+ | `SORE_KEY_PAIR_TYPE` | SOAuth key type; defaults to `Ed25519` |
190
+ | `SORE_ADMIN_GRANT` / `SORE_USER_GRANT` | Required grants |
191
+
192
+ Terminate TLS at nginx or an upstream load balancer, keep the admin API private,
193
+ replace every development secret, and adapt `nginx/nginx.conf` to the production
194
+ hostnames. The `.well-known` directory is mounted read-only for ACME challenges.
195
+
196
+ ### Development and verification
197
+
198
+ ```bash
199
+ uv sync --extra dev
200
+ uv run pytest
201
+ uv run ruff check .
202
+ npm install
203
+ npm run build:css
204
+ docker compose config
205
+ ```
206
+
207
+ Tailwind output and htmx are vendored under `soreplicator/server/static`, so the setup UI does
208
+ not depend on third-party CDNs. The Tailwind palette is the colorblind-friendly
209
+ SO palette from `socolors`.
210
+
211
+ ### Releases
212
+
213
+ GitHub Actions runs Ruff and the test suite on Python 3.11–3.13 for pull requests
214
+ and pushes to `main`, and verifies that the committed Tailwind stylesheet is current.
215
+ To publish a release, update the version in `pyproject.toml`, merge it to `main`,
216
+ and publish a GitHub Release.
217
+
218
+ The publishing workflow uses [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
219
+ instead of a stored API token. Before the first release, configure a GitHub Actions
220
+ Trusted Publisher for this repository and the `pypi` environment in the
221
+ `soreplicator` PyPI project.
@@ -0,0 +1,199 @@
1
+ SORE: Simons Observatory REplicator
2
+ ===================================
3
+
4
+ SORE is a tool to ensure that a copy of your critical data is stored on all of our
5
+ HPC machines. You can upload a file to SORE, and within an hour it should be
6
+ replicated across NERSC, Princeton, and SO:UK. The system works by leveraging
7
+ S3: we use Versity's S3 Gateway to allow you to upload data to the NERSC filesystem
8
+ from anywhere in the world, and then a background Globus job ensures data is
9
+ replicated to the other systems. You can only add and delete data that you have
10
+ uploaded; no 'edit' functionality exists. You should also not attempt to
11
+ directly modify data on disk at any of the sites (and this will be impossible
12
+ anyway).
13
+
14
+ To upload data, all you need to do is use the `sore` command-line utility:
15
+ ```bash
16
+ sore -i local_file.fits -o testing/2026-01-01/local_file.fits
17
+ ```
18
+ The `-i` input file is the path on your local filesystem. The `-o` output
19
+ filename is the position in the global namespace that your file will take.
20
+ Your file will then be uploaded, over the internet, to NERSC, where it will
21
+ appear at `sore/$GITHUB_USERNAME/testing/2026-01-01/local_file.fits`. It will
22
+ shortly be copied to other sites.
23
+
24
+ Developed with the Codex/OpenAI 5.6 generation models.
25
+
26
+ Getting set up
27
+ --------------
28
+
29
+ To get set up, you will first need to install the `soreplicator` python package:
30
+ ```bash
31
+ uv pip install soreplicator
32
+ ```
33
+ To authenticate with the system, you will need a username and password. These are
34
+ _secrets_ and should never be shared with anyone else. To get your username
35
+ and password, you will need to log into the web interface at
36
+ https://sore.simonsobservatory.org/setup. Here, you will be able to get your username
37
+ and password for the `sore` server. Your username will always be the same
38
+ as your GitHub username.
39
+
40
+ `sore` includes a utility to store these secrets for re-use in a safe location.
41
+ We recommend using this. To set up `sore` you should call:
42
+ ```bash
43
+ sore --setup
44
+ ```
45
+ This will allow you to paste in your username and password, which will be saved
46
+ in your config location (usually `~/.config/sore.json`) with `600` permissions.
47
+
48
+ Deleting Files
49
+ --------------
50
+
51
+ Sometimes you will want to remove files from the system, because they are old and
52
+ no longer used. You can do this with the `sore` command-line tool:
53
+ ```bash
54
+ sore -d testing/2026-01-01/local-file.fits
55
+ ```
56
+ Which will ask you to input `y/n` for whether you are sure you want to delete
57
+ this file in the global namespace. Note that you can only change your own files,
58
+ not anyone elses. If you are sure and want to avoid the prompt, you can use
59
+ ```bash
60
+ sore -d testing/2026-01-01/local-file.fits -f
61
+ ```
62
+ to force-delete without needing to respond to the prompt.
63
+
64
+ Administrator Information
65
+ -------------------------
66
+
67
+ Setting up `sore` is easy. We provide a containerized version of the server. The most
68
+ important thing to realize is that we provide one bucket per user, with the bucket
69
+ name the same as their GitHub username. This allows us to set bucket-level permissions
70
+ for each user, allowing them only `PUT`, `GET`, and `DELETE` access to these buckets,
71
+ with `GET` permissions for all buckets.
72
+
73
+ Included with this repository is a `nginx` configuration file that allows you to mount:
74
+ - Versity's S3 Gateway on /
75
+ - The provided `sore` server on `/setup`.
76
+ - Sharing of directories for `.well-known` challenges.
77
+
78
+ We authenticate users using the Simons Observatory authentication framework `soauth`.
79
+
80
+ To set up `sore`, you will need an administrator credential (ideally not the root
81
+ credential) that allows the creation of buckets and users with various levels of
82
+ permissions. You should set the following environment variables:
83
+
84
+ - `SORE_GATEWAY_USERNAME`: the username of the account that can create other accounts
85
+ - `SORE_GATEWAY_PASSWORD`: the password of the account that can create other accounts
86
+ - `SORE_GATEWAY_REMOTE_LOCATION`: the location of the gateway on the open internet
87
+ - `SORE_GATEWAY_LOCAL_LOCATION`: the location of the gateway on the local network
88
+ - `SORE_ADMIN_GRANT`: the `soauth` grant for administrator priviliges (`sore:admin`)
89
+ - `SORE_USER_GRANT`: the `soauth` grant required for users to create accounts (`sore:user`)
90
+
91
+ We will only communicate with the gateway on the local network and never over the
92
+ open internet. The `REMOTE_LOCATION` is just for providing information in the UI.
93
+
94
+ ### Administrator Panels
95
+
96
+ Within the Web UI, you can see information on all of the user accounts that have
97
+ been created to date, and the buckets along with their sizes. This information is
98
+ pulled dynamically from the S3 gateway and is not stored separately for the `sore`
99
+ server so the `sore` server is effectively stateless. You will require the
100
+ `SORE_ADMIN_GRANT` to view this page.
101
+
102
+ Implementation
103
+ --------------
104
+
105
+ This repository contains the complete gateway edge, account service, and client:
106
+
107
+ - `docker-compose.yml` runs VersityGW with a POSIX directory, sidecar metadata,
108
+ IAM, its WebGUI, the SORE server, and nginx.
109
+ - `soreplicator/server/` is a stateless FastAPI service. It authenticates with `soauth`, creates
110
+ or rotates a VersityGW account, creates the matching bucket, and maintains a
111
+ bucket policy that gives every provisioned account read access while preserving
112
+ owner write/delete access.
113
+ - `soreplicator/client/` supplies the `sore` command. Large files use four concurrent S3
114
+ multipart transfers with a terminal progress bar; smaller files use a regular
115
+ PUT. Both paths write conditionally, so an existing object cannot be
116
+ overwritten. Configuration is written atomically with mode `600` and insecure
117
+ remote HTTP endpoints are rejected.
118
+ - The `/setup/admin` view queries users, buckets, object counts, and sizes directly
119
+ from VersityGW. No credentials or inventory are stored in an application database.
120
+
121
+ The downstream Globus replication job is intentionally outside this repository's
122
+ scope; it can consume the gateway's shared POSIX directory independently.
123
+
124
+ ### Local stack
125
+
126
+ The included compose topology is a loopback-only development environment:
127
+
128
+ ```bash
129
+ cp .env.example .env
130
+ # Change the secrets and SORE_DEVELOPMENT_USERNAME in .env first.
131
+ docker compose up -d --build
132
+ ```
133
+
134
+ Open the following local endpoints:
135
+
136
+ - SORE setup: `http://sore.localhost:8080/setup/`
137
+ - VersityGW WebGUI through nginx: `http://gateway.localhost:8080/`
138
+ - S3 through nginx: `http://s3.localhost:8080/`
139
+
140
+ The WebGUI is preconfigured to use `s3.localhost` for S3 and `admin.localhost`
141
+ for its admin API. Log in with the gateway administrator values from `.env`.
142
+ Persistent objects, versions, IAM data, and metadata are stored under `./data`
143
+ by default. Set `SORE_DATA_DIRECTORY` to share another host directory.
144
+
145
+ The compose stack deliberately uses the fixed development identity named by
146
+ `SORE_DEVELOPMENT_USERNAME`; nginx publishes only on `127.0.0.1`. Development
147
+ authentication cannot start unless `SORE_DEBUG=true`. Never expose this mode to a
148
+ network.
149
+
150
+ ### Production server configuration
151
+
152
+ The Docker image runs as an unprivileged user with a read-only-compatible root
153
+ filesystem. For production, set `SORE_AUTH_MODE=soauth`, `SORE_DEBUG=false`, and
154
+ provide these settings to the server container:
155
+
156
+ | Setting | Purpose |
157
+ | --- | --- |
158
+ | `SORE_GATEWAY_USERNAME` / `SORE_GATEWAY_PASSWORD` | Versity administrator credential |
159
+ | `SORE_GATEWAY_REMOTE_LOCATION` | Public S3 URL displayed to clients |
160
+ | `SORE_GATEWAY_LOCAL_LOCATION` | Private S3 URL used by the server |
161
+ | `SORE_GATEWAY_ADMIN_LOCATION` | Private Versity admin API URL |
162
+ | `SORE_GATEWAY_REGION` | SigV4 region; defaults to `us-east-1` |
163
+ | `SORE_APP_BASE_URL` | Public application URL, including `/setup` |
164
+ | `SORE_AUTHENTICATION_BASE_URL` | SOAuth service URL |
165
+ | `SORE_APP_ID` / `SORE_CLIENT_SECRET` | SOAuth application credential |
166
+ | `SORE_PUBLIC_KEY_FILE` | Mounted SOAuth public key path |
167
+ | `SORE_KEY_PAIR_TYPE` | SOAuth key type; defaults to `Ed25519` |
168
+ | `SORE_ADMIN_GRANT` / `SORE_USER_GRANT` | Required grants |
169
+
170
+ Terminate TLS at nginx or an upstream load balancer, keep the admin API private,
171
+ replace every development secret, and adapt `nginx/nginx.conf` to the production
172
+ hostnames. The `.well-known` directory is mounted read-only for ACME challenges.
173
+
174
+ ### Development and verification
175
+
176
+ ```bash
177
+ uv sync --extra dev
178
+ uv run pytest
179
+ uv run ruff check .
180
+ npm install
181
+ npm run build:css
182
+ docker compose config
183
+ ```
184
+
185
+ Tailwind output and htmx are vendored under `soreplicator/server/static`, so the setup UI does
186
+ not depend on third-party CDNs. The Tailwind palette is the colorblind-friendly
187
+ SO palette from `socolors`.
188
+
189
+ ### Releases
190
+
191
+ GitHub Actions runs Ruff and the test suite on Python 3.11–3.13 for pull requests
192
+ and pushes to `main`, and verifies that the committed Tailwind stylesheet is current.
193
+ To publish a release, update the version in `pyproject.toml`, merge it to `main`,
194
+ and publish a GitHub Release.
195
+
196
+ The publishing workflow uses [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/)
197
+ instead of a stored API token. Before the first release, configure a GitHub Actions
198
+ Trusted Publisher for this repository and the `pypi` environment in the
199
+ `soreplicator` PyPI project.
@@ -0,0 +1,61 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "soreplicator"
7
+ version = "0.1.0"
8
+ description = "Simons Observatory REplicator server and command-line client"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = "MIT"
12
+ dependencies = [
13
+ "boto3>=1.40,<2",
14
+ "defusedxml>=0.7.1,<1",
15
+ "fastapi>=0.116,<1",
16
+ "httpx>=0.28,<1",
17
+ "jinja2>=3.1,<4",
18
+ "pydantic-settings>=2.10,<3",
19
+ "python-multipart>=0.0.20,<1",
20
+ "soauth>=0.8.2,<1",
21
+ "socolors>=0.0.8,<1",
22
+ "uvicorn[standard]>=0.35,<1",
23
+ ]
24
+
25
+ [project.optional-dependencies]
26
+ dev = [
27
+ "pytest>=8.4,<9",
28
+ "pytest-asyncio>=1.1,<2",
29
+ "ruff>=0.12,<1",
30
+ ]
31
+
32
+ [project.scripts]
33
+ sore = "soreplicator.client.cli:main"
34
+ sore-server = "soreplicator.server.app:run"
35
+
36
+ [tool.setuptools]
37
+ packages = ["soreplicator", "soreplicator.client", "soreplicator.server"]
38
+ include-package-data = false
39
+
40
+ [tool.setuptools.package-data]
41
+ "soreplicator.server" = [
42
+ "static/app.css",
43
+ "static/app.js",
44
+ "static/htmx.min.js",
45
+ "static/*.png",
46
+ "static/*.webmanifest",
47
+ "templates/*.html",
48
+ "templates/fragments/*.html",
49
+ ]
50
+
51
+ [tool.pytest.ini_options]
52
+ testpaths = ["tests"]
53
+ addopts = "-q"
54
+
55
+ [tool.ruff]
56
+ line-length = 88
57
+ target-version = "py311"
58
+
59
+ [tool.ruff.lint]
60
+ extend-select = ["I", "B", "S", "UP"]
61
+ ignore = ["S101"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """Simons Observatory REplicator."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,3 @@
1
+ """SORE command-line client."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,154 @@
1
+ """The ``sore`` command-line interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import getpass
7
+ import sys
8
+ from pathlib import Path
9
+ from typing import TextIO
10
+ from urllib.parse import urlsplit
11
+
12
+ from pydantic import ValidationError
13
+
14
+ from soreplicator.client.config import (
15
+ ClientConfig,
16
+ config_path,
17
+ load_config,
18
+ save_config,
19
+ )
20
+ from soreplicator.client.storage import StorageError, delete, upload
21
+
22
+ DEFAULT_ENDPOINT = "https://sore.simonsobservatory.org"
23
+
24
+
25
+ def _human_bytes(value: int) -> str:
26
+ size = float(value)
27
+ for unit in ("B", "KB", "MB", "GB", "TB"):
28
+ if size < 1000 or unit == "TB":
29
+ return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
30
+ size /= 1000
31
+ return f"{size:.1f} TB"
32
+
33
+
34
+ class ProgressBar:
35
+ def __init__(self, total: int, stream: TextIO = sys.stderr, width: int = 28):
36
+ self.total = total
37
+ self.stream = stream
38
+ self.width = width
39
+ self.enabled = stream.isatty()
40
+ self.rendered = False
41
+ if self.enabled:
42
+ self.update(0, total)
43
+
44
+ def update(self, uploaded: int, total: int) -> None:
45
+ if not self.enabled:
46
+ return
47
+ fraction = 1.0 if total == 0 else min(1.0, uploaded / total)
48
+ filled = round(self.width * fraction)
49
+ bar = "#" * filled + "-" * (self.width - filled)
50
+ self.stream.write(
51
+ f"\rUploading [{bar}] {fraction:6.1%} "
52
+ f"{_human_bytes(uploaded)} / {_human_bytes(total)}"
53
+ )
54
+ self.stream.flush()
55
+ self.rendered = True
56
+
57
+ def close(self) -> None:
58
+ if self.rendered:
59
+ self.stream.write("\n")
60
+ self.stream.flush()
61
+ self.rendered = False
62
+
63
+
64
+ def build_parser() -> argparse.ArgumentParser:
65
+ parser = argparse.ArgumentParser(
66
+ prog="sore", description="Upload immutable files to the SORE replica service."
67
+ )
68
+ actions = parser.add_mutually_exclusive_group(required=True)
69
+ actions.add_argument(
70
+ "--setup", action="store_true", help="store gateway credentials"
71
+ )
72
+ actions.add_argument("-i", "--input", type=Path, help="local file to upload")
73
+ actions.add_argument(
74
+ "-d", "--delete", metavar="REMOTE_PATH", help="delete an object"
75
+ )
76
+ parser.add_argument("-o", "--output", help="remote path for an upload")
77
+ parser.add_argument(
78
+ "-f", "--force", action="store_true", help="skip delete confirmation"
79
+ )
80
+ parser.add_argument(
81
+ "--config", type=Path, default=config_path(), help=argparse.SUPPRESS
82
+ )
83
+ return parser
84
+
85
+
86
+ def _setup(path: Path) -> int:
87
+ endpoint = (
88
+ input(f"Gateway endpoint [{DEFAULT_ENDPOINT}]: ").strip() or DEFAULT_ENDPOINT
89
+ )
90
+ parsed = urlsplit(endpoint)
91
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
92
+ raise ValueError("endpoint must be an absolute HTTP(S) URL")
93
+ hostname = parsed.hostname or ""
94
+ is_loopback = hostname in {"localhost", "127.0.0.1", "::1"} or hostname.endswith(
95
+ ".localhost"
96
+ )
97
+ if parsed.scheme != "https" and not is_loopback:
98
+ raise ValueError(
99
+ "credentials may only be sent over HTTPS (except on localhost)"
100
+ )
101
+ username = input("Username: ").strip()
102
+ password = getpass.getpass("Password: ")
103
+ if not password:
104
+ raise ValueError("password cannot be empty")
105
+ destination = save_config(
106
+ ClientConfig(endpoint=endpoint, username=username, password=password), path
107
+ )
108
+ print(f"Credentials saved to {destination} with mode 600.")
109
+ return 0
110
+
111
+
112
+ def _confirm_delete(remote_path: str) -> bool:
113
+ answer = input(f"Delete {remote_path!r} permanently? [y/N] ").strip().lower()
114
+ return answer in {"y", "yes"}
115
+
116
+
117
+ def main(argv: list[str] | None = None) -> int:
118
+ parser = build_parser()
119
+ args = parser.parse_args(argv)
120
+ try:
121
+ if args.setup:
122
+ if args.output or args.force:
123
+ parser.error("--setup cannot be combined with -o or -f")
124
+ return _setup(args.config)
125
+
126
+ config = load_config(args.config)
127
+ if args.input:
128
+ if not args.output:
129
+ parser.error("-o/--output is required with -i/--input")
130
+ if args.force:
131
+ parser.error("-f/--force is only valid with -d/--delete")
132
+ progress = ProgressBar(args.input.stat().st_size)
133
+ try:
134
+ upload(config, args.input, args.output, progress.update)
135
+ finally:
136
+ progress.close()
137
+ print(f"Uploaded {args.input} to sore/{config.username}/{args.output}")
138
+ return 0
139
+
140
+ if args.output:
141
+ parser.error("-o/--output is only valid with -i/--input")
142
+ if not args.force and not _confirm_delete(args.delete):
143
+ print("Delete cancelled.")
144
+ return 0
145
+ delete(config, args.delete)
146
+ print(f"Deleted sore/{config.username}/{args.delete}")
147
+ return 0
148
+ except (OSError, PermissionError, ValueError, ValidationError, StorageError) as exc:
149
+ print(f"sore: {exc}", file=sys.stderr)
150
+ return 1
151
+
152
+
153
+ if __name__ == "__main__":
154
+ raise SystemExit(main())