bucket-helper 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bucket_helper-0.2.2/LICENSE +29 -0
- bucket_helper-0.2.2/PKG-INFO +256 -0
- bucket_helper-0.2.2/README.md +208 -0
- bucket_helper-0.2.2/bucket_helper/__init__.py +60 -0
- bucket_helper-0.2.2/bucket_helper/api.py +407 -0
- bucket_helper-0.2.2/bucket_helper/cli_argparse.py +369 -0
- bucket_helper-0.2.2/bucket_helper/cli_click.py +230 -0
- bucket_helper-0.2.2/bucket_helper/main.py +444 -0
- bucket_helper-0.2.2/bucket_helper/mcp.py +92 -0
- bucket_helper-0.2.2/bucket_helper.egg-info/PKG-INFO +256 -0
- bucket_helper-0.2.2/bucket_helper.egg-info/SOURCES.txt +19 -0
- bucket_helper-0.2.2/bucket_helper.egg-info/dependency_links.txt +1 -0
- bucket_helper-0.2.2/bucket_helper.egg-info/entry_points.txt +4 -0
- bucket_helper-0.2.2/bucket_helper.egg-info/requires.txt +27 -0
- bucket_helper-0.2.2/bucket_helper.egg-info/top_level.txt +1 -0
- bucket_helper-0.2.2/pyproject.toml +107 -0
- bucket_helper-0.2.2/setup.cfg +4 -0
- bucket_helper-0.2.2/tests/test_api.py +81 -0
- bucket_helper-0.2.2/tests/test_bucket_helper.py +214 -0
- bucket_helper-0.2.2/tests/test_cli.py +101 -0
- bucket_helper-0.2.2/tests/test_mcp.py +39 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Warith HARCHAOUI
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bucket-helper
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Bucket Helper โ utility functions for AWS S3 and any S3-compatible object storage (MinIO, Backblaze B2, DigitalOcean Spaces, Cloudflare R2, Wasabi, etc.) via boto3, exposed as library, argparse CLI, click CLI, FastAPI HTTP surface and MCP tools. credentials() loader, upload / download / list / delete / exists / make_bucket / remote_tempfile.
|
|
5
|
+
Author-email: Warith HARCHAOUI <warithmetics@deraison.ai>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/warith-harchaoui/bucket-helper
|
|
8
|
+
Project-URL: Issues, https://github.com/warith-harchaoui/bucket-helper/issues
|
|
9
|
+
Keywords: s3,minio,boto3,object-storage,aws,r2,b2,spaces,bucket,cli,fastapi,mcp
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Internet :: File Transfer Protocol (FTP)
|
|
20
|
+
Classifier: Topic :: System :: Archiving
|
|
21
|
+
Requires-Python: <3.14,>=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: os-helper>=1.4.2
|
|
25
|
+
Requires-Dist: boto3<2,>=1.34
|
|
26
|
+
Requires-Dist: botocore<2,>=1.34
|
|
27
|
+
Provides-Extra: cli
|
|
28
|
+
Requires-Dist: click<9,>=8.1; extra == "cli"
|
|
29
|
+
Provides-Extra: api
|
|
30
|
+
Requires-Dist: fastapi<1,>=0.111; extra == "api"
|
|
31
|
+
Requires-Dist: uvicorn<1,>=0.30; extra == "api"
|
|
32
|
+
Requires-Dist: python-multipart<1,>=0.0.9; extra == "api"
|
|
33
|
+
Provides-Extra: mcp
|
|
34
|
+
Requires-Dist: fastapi-mcp<1,>=0.3; extra == "mcp"
|
|
35
|
+
Requires-Dist: fastapi<1,>=0.111; extra == "mcp"
|
|
36
|
+
Requires-Dist: uvicorn<1,>=0.30; extra == "mcp"
|
|
37
|
+
Requires-Dist: python-multipart<1,>=0.0.9; extra == "mcp"
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
40
|
+
Requires-Dist: pyyaml>=6; extra == "dev"
|
|
41
|
+
Requires-Dist: moto[s3]>=5; extra == "dev"
|
|
42
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
43
|
+
Requires-Dist: click<9,>=8.1; extra == "dev"
|
|
44
|
+
Requires-Dist: fastapi<1,>=0.111; extra == "dev"
|
|
45
|
+
Requires-Dist: python-multipart<1,>=0.0.9; extra == "dev"
|
|
46
|
+
Requires-Dist: fastapi-mcp<1,>=0.3; extra == "dev"
|
|
47
|
+
Dynamic: license-file
|
|
48
|
+
|
|
49
|
+
# Bucket Helper
|
|
50
|
+
|
|
51
|
+
[๐ซ๐ท](https://github.com/warith-harchaoui/bucket-helper/blob/main/LISEZMOI.md) ยท [๐ฌ๐ง](https://github.com/warith-harchaoui/bucket-helper/blob/main/README.md)
|
|
52
|
+
|
|
53
|
+
[](https://github.com/warith-harchaoui/bucket-helper/actions/workflows/ci.yml) [](https://github.com/warith-harchaoui/bucket-helper/blob/main/LICENSE) [](#)
|
|
54
|
+
|
|
55
|
+
`Bucket Helper` belongs to a collection of libraries called `AI Helpers` developed for building Artificial Intelligence.
|
|
56
|
+
|
|
57
|
+
Utility functions for **AWS S3** and any **S3-compatible object storage** โ MinIO, Backblaze B2 S3 API, DigitalOcean Spaces, Cloudflare R2, Wasabi, and friends. Built on [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html). Same shape as [sftp-helper](https://github.com/warith-harchaoui/sftp-helper): a `credentials()` loader, the usual CRUD (`upload` / `download` / `delete` / `exists` / `list_prefix`), and a `remote_tempfile` context manager for stage-and-share flows.
|
|
58
|
+
|
|
59
|
+
[๐ AI Helpers](https://harchaoui.org/warith/ai-helpers)
|
|
60
|
+
|
|
61
|
+
[](https://harchaoui.org/warith/ai-helpers)
|
|
62
|
+
|
|
63
|
+
# Installation
|
|
64
|
+
|
|
65
|
+
**Prerequisites** โ **Python 3.10โ3.13** and **git**, cross-platform:
|
|
66
|
+
|
|
67
|
+
- ๐ **macOS** ([Homebrew](https://brew.sh)): `brew install python git`
|
|
68
|
+
- ๐ง **Ubuntu/Debian**: `sudo apt update && sudo apt install -y python3 python3-pip git`
|
|
69
|
+
- ๐ช **Windows** (PowerShell): `winget install Python.Python.3.12 Git.Git`
|
|
70
|
+
|
|
71
|
+
Then install the package:
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install --force-reinstall --no-cache-dir git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Optional extras โ pick what you need:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
# argparse CLI is always available. Add the click twin:
|
|
82
|
+
pip install 'bucket-helper[cli] @ git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2'
|
|
83
|
+
|
|
84
|
+
# HTTP server (FastAPI + uvicorn + python-multipart):
|
|
85
|
+
pip install 'bucket-helper[api] @ git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2'
|
|
86
|
+
|
|
87
|
+
# MCP tools (fastapi-mcp) โ requires the [api] plumbing:
|
|
88
|
+
pip install 'bucket-helper[api,mcp] @ git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2'
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
# Configuration
|
|
92
|
+
|
|
93
|
+
A ready-to-fill template is committed at [`s3_config.json.example`](https://github.com/warith-harchaoui/bucket-helper/blob/main/s3_config.json.example). Copy it to `s3_config.json` and edit in place โ real `*config.json` files are gitignored so you cannot accidentally commit secrets:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
cp s3_config.json.example s3_config.json
|
|
97
|
+
# then edit s3_config.json with your AWS / MinIO / R2 / B2 credentials
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
You may also write a `s3_config.yaml`, use a `.env`, or set environment variables โ `bucket-helper` falls back in that order via `os_helper.get_config`. Required keys:
|
|
101
|
+
|
|
102
|
+
```json
|
|
103
|
+
{
|
|
104
|
+
"s3_access_key": "AKIA...",
|
|
105
|
+
"s3_secret_key": "...",
|
|
106
|
+
"s3_bucket": "my-bucket",
|
|
107
|
+
"s3_https": "https://my-bucket.s3.eu-west-3.amazonaws.com"
|
|
108
|
+
}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Optional keys:
|
|
112
|
+
|
|
113
|
+
| Key | Default | Notes |
|
|
114
|
+
|---|---|---|
|
|
115
|
+
| `s3_region` | `"us-east-1"` | AWS region; mostly cosmetic for MinIO / R2 |
|
|
116
|
+
| `s3_endpoint_url` | empty (= AWS S3) | Set this for S3-compatible backends โ see table below |
|
|
117
|
+
| `s3_prefix` | empty | Default key prefix added by `upload(...)` when no destination is given |
|
|
118
|
+
| `s3_use_path_style` | `"false"` | Force path-style addressing (`endpoint/bucket/key` instead of `bucket.endpoint/key`). Typical for MinIO with custom domains. |
|
|
119
|
+
| `s3_verify_ssl` | `"true"` | Disable only for dev MinIO with self-signed certs |
|
|
120
|
+
|
|
121
|
+
## Endpoint URLs for common S3-compatible storage
|
|
122
|
+
|
|
123
|
+
Set `s3_endpoint_url` to:
|
|
124
|
+
|
|
125
|
+
| Provider | Endpoint |
|
|
126
|
+
|---|---|
|
|
127
|
+
| **AWS S3** | leave empty / unset |
|
|
128
|
+
| **MinIO** | `http://minio.example.com:9000` (or `https://...` with TLS) |
|
|
129
|
+
| **DigitalOcean Spaces** | `https://nyc3.digitaloceanspaces.com` (region in subdomain) |
|
|
130
|
+
| **Cloudflare R2** | `https://<account_id>.r2.cloudflarestorage.com` |
|
|
131
|
+
| **Backblaze B2 (S3 API)** | `https://s3.<region>.backblazeb2.com` |
|
|
132
|
+
| **Wasabi** | `https://s3.<region>.wasabisys.com` |
|
|
133
|
+
|
|
134
|
+
# Usage
|
|
135
|
+
|
|
136
|
+
For the full catalog of recipes (uploads / downloads / listings, S3-compatible endpoints โ MinIO / R2 / B2 / Spaces / Wasabi, temporary remote keys with auto-cleanup, mirroring with sftp-helper), see [๐ EXAMPLES.md](https://github.com/warith-harchaoui/bucket-helper/blob/main/EXAMPLES.md).
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import bucket_helper as bh
|
|
140
|
+
|
|
141
|
+
# Load creds โ JSON / YAML / env / .env (auto-fallback in that order)
|
|
142
|
+
cred = bh.credentials("path/to/s3_config.json")
|
|
143
|
+
|
|
144
|
+
# Upload a local file
|
|
145
|
+
uri = bh.upload("local.txt", cred, "folder/uploaded.txt")
|
|
146
|
+
# uri == "s3://my-bucket/folder/uploaded.txt"
|
|
147
|
+
|
|
148
|
+
assert bh.exists(uri, cred)
|
|
149
|
+
|
|
150
|
+
# Download
|
|
151
|
+
bh.download(uri, "downloaded.txt", cred)
|
|
152
|
+
|
|
153
|
+
# List
|
|
154
|
+
for key in bh.list_prefix("folder/", cred):
|
|
155
|
+
print(key)
|
|
156
|
+
|
|
157
|
+
# Delete
|
|
158
|
+
bh.delete(uri, cred)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## MinIO example
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
cred = {
|
|
165
|
+
"s3_access_key": "minioadmin",
|
|
166
|
+
"s3_secret_key": "minioadmin",
|
|
167
|
+
"s3_bucket": "uploads",
|
|
168
|
+
"s3_https": "http://minio.example.com:9000/uploads",
|
|
169
|
+
"s3_endpoint_url": "http://minio.example.com:9000",
|
|
170
|
+
"s3_use_path_style": "true",
|
|
171
|
+
"s3_region": "us-east-1", # MinIO accepts any region string
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
bh.make_bucket("uploads", cred)
|
|
175
|
+
bh.upload("file.bin", cred, "file.bin")
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Stage-and-share with `remote_tempfile`
|
|
179
|
+
|
|
180
|
+
Drop a generated file at a unique random key, hand the public URL to a
|
|
181
|
+
downstream worker / webhook, and the object is deleted on block exit
|
|
182
|
+
(even if the body raises):
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
import bucket_helper as bh
|
|
186
|
+
import requests
|
|
187
|
+
|
|
188
|
+
cred = bh.credentials("path/to/s3_config.json")
|
|
189
|
+
|
|
190
|
+
with bh.remote_tempfile(cred, ext="json", prefix="runs") as (s3_addr, public_url):
|
|
191
|
+
bh.upload("payload.json", cred, s3_addr, content_type="application/json")
|
|
192
|
+
# Hand the URL to something that fetches it once.
|
|
193
|
+
requests.post("https://hook.example.com/process", json={"input_url": public_url}).raise_for_status()
|
|
194
|
+
# Object is gone here, no manual cleanup.
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
# Multi-surface exposure
|
|
198
|
+
|
|
199
|
+
Every public function in the library is also exposed as:
|
|
200
|
+
|
|
201
|
+
- **argparse CLI** โ `bucket-helper <subcommand>` (installed by default).
|
|
202
|
+
- **click CLI** โ `bucket-helper-click <subcommand>` (install `[cli]` extra).
|
|
203
|
+
- **FastAPI HTTP** โ `uvicorn bucket_helper.api:app --host 0.0.0.0 --port 8000` (install `[api]` extra).
|
|
204
|
+
- **MCP tools** โ `bucket-helper-mcp` (install `[api,mcp]` extras).
|
|
205
|
+
|
|
206
|
+
Both CLIs share the same subcommand names and flags โ pick your favourite.
|
|
207
|
+
|
|
208
|
+
## CLI examples
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
# argparse CLI (always available)
|
|
212
|
+
bucket-helper upload --config s3_config.json --input local.txt --key folder/uploaded.txt
|
|
213
|
+
bucket-helper exists --config s3_config.json --key folder/uploaded.txt
|
|
214
|
+
bucket-helper download --config s3_config.json --key folder/uploaded.txt --output back.txt
|
|
215
|
+
bucket-helper list --config s3_config.json --prefix folder/
|
|
216
|
+
bucket-helper delete --config s3_config.json --key folder/uploaded.txt
|
|
217
|
+
bucket-helper make-bucket --config s3_config.json --bucket new-bucket
|
|
218
|
+
bucket-helper tempfile --config s3_config.json --ext json --prefix runs
|
|
219
|
+
bucket-helper strip-path --config s3_config.json --address s3://my-bucket/path/to/obj
|
|
220
|
+
|
|
221
|
+
# click CLI โ same verbs, same flags
|
|
222
|
+
bucket-helper-click upload --config s3_config.json --input local.txt --key folder/uploaded.txt
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
## HTTP + MCP server
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
# Serve HTTP + MCP (default credentials picked up from BUCKET_HELPER_CONFIG)
|
|
229
|
+
BUCKET_HELPER_CONFIG=$PWD/s3_config.json bucket-helper-mcp
|
|
230
|
+
|
|
231
|
+
# Or run only FastAPI directly:
|
|
232
|
+
uvicorn bucket_helper.api:app --host 0.0.0.0 --port 8000
|
|
233
|
+
# โ Swagger UI at http://localhost:8000/docs
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Per-request credentials can also be sent as multipart form fields
|
|
237
|
+
(`s3_access_key` / `s3_secret_key` / `s3_bucket` / `s3_https` / โฆ).
|
|
238
|
+
|
|
239
|
+
## Docker
|
|
240
|
+
|
|
241
|
+
```bash
|
|
242
|
+
docker build -t bucket-helper .
|
|
243
|
+
docker run --rm -p 8000:8000 \
|
|
244
|
+
-e BUCKET_HELPER_CONFIG=/config/s3_config.json \
|
|
245
|
+
-v $PWD/s3_config.json:/config/s3_config.json:ro \
|
|
246
|
+
bucket-helper
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
See also: [LANDSCAPE.md](https://github.com/warith-harchaoui/bucket-helper/blob/main/LANDSCAPE.md) (competitive positioning) and
|
|
250
|
+
[GUI.md](https://github.com/warith-harchaoui/bucket-helper/blob/main/GUI.md) (visual product design plan).
|
|
251
|
+
|
|
252
|
+
# Author
|
|
253
|
+
- [Warith HARCHAOUI](https://linkedin.com/in/warith-harchaoui)
|
|
254
|
+
|
|
255
|
+
# Acknowledgements
|
|
256
|
+
Special thanks to [Mohamed Chelali](https://mchelali.github.io) and [Bachir Zerroug](https://www.linkedin.com/in/bachirzerroug) for fruitful discussions.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
# Bucket Helper
|
|
2
|
+
|
|
3
|
+
[๐ซ๐ท](https://github.com/warith-harchaoui/bucket-helper/blob/main/LISEZMOI.md) ยท [๐ฌ๐ง](https://github.com/warith-harchaoui/bucket-helper/blob/main/README.md)
|
|
4
|
+
|
|
5
|
+
[](https://github.com/warith-harchaoui/bucket-helper/actions/workflows/ci.yml) [](https://github.com/warith-harchaoui/bucket-helper/blob/main/LICENSE) [](#)
|
|
6
|
+
|
|
7
|
+
`Bucket Helper` belongs to a collection of libraries called `AI Helpers` developed for building Artificial Intelligence.
|
|
8
|
+
|
|
9
|
+
Utility functions for **AWS S3** and any **S3-compatible object storage** โ MinIO, Backblaze B2 S3 API, DigitalOcean Spaces, Cloudflare R2, Wasabi, and friends. Built on [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html). Same shape as [sftp-helper](https://github.com/warith-harchaoui/sftp-helper): a `credentials()` loader, the usual CRUD (`upload` / `download` / `delete` / `exists` / `list_prefix`), and a `remote_tempfile` context manager for stage-and-share flows.
|
|
10
|
+
|
|
11
|
+
[๐ AI Helpers](https://harchaoui.org/warith/ai-helpers)
|
|
12
|
+
|
|
13
|
+
[](https://harchaoui.org/warith/ai-helpers)
|
|
14
|
+
|
|
15
|
+
# Installation
|
|
16
|
+
|
|
17
|
+
**Prerequisites** โ **Python 3.10โ3.13** and **git**, cross-platform:
|
|
18
|
+
|
|
19
|
+
- ๐ **macOS** ([Homebrew](https://brew.sh)): `brew install python git`
|
|
20
|
+
- ๐ง **Ubuntu/Debian**: `sudo apt update && sudo apt install -y python3 python3-pip git`
|
|
21
|
+
- ๐ช **Windows** (PowerShell): `winget install Python.Python.3.12 Git.Git`
|
|
22
|
+
|
|
23
|
+
Then install the package:
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install --force-reinstall --no-cache-dir git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Optional extras โ pick what you need:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
# argparse CLI is always available. Add the click twin:
|
|
34
|
+
pip install 'bucket-helper[cli] @ git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2'
|
|
35
|
+
|
|
36
|
+
# HTTP server (FastAPI + uvicorn + python-multipart):
|
|
37
|
+
pip install 'bucket-helper[api] @ git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2'
|
|
38
|
+
|
|
39
|
+
# MCP tools (fastapi-mcp) โ requires the [api] plumbing:
|
|
40
|
+
pip install 'bucket-helper[api,mcp] @ git+https://github.com/warith-harchaoui/bucket-helper.git@v0.2.2'
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
# Configuration
|
|
44
|
+
|
|
45
|
+
A ready-to-fill template is committed at [`s3_config.json.example`](https://github.com/warith-harchaoui/bucket-helper/blob/main/s3_config.json.example). Copy it to `s3_config.json` and edit in place โ real `*config.json` files are gitignored so you cannot accidentally commit secrets:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
cp s3_config.json.example s3_config.json
|
|
49
|
+
# then edit s3_config.json with your AWS / MinIO / R2 / B2 credentials
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
You may also write a `s3_config.yaml`, use a `.env`, or set environment variables โ `bucket-helper` falls back in that order via `os_helper.get_config`. Required keys:
|
|
53
|
+
|
|
54
|
+
```json
|
|
55
|
+
{
|
|
56
|
+
"s3_access_key": "AKIA...",
|
|
57
|
+
"s3_secret_key": "...",
|
|
58
|
+
"s3_bucket": "my-bucket",
|
|
59
|
+
"s3_https": "https://my-bucket.s3.eu-west-3.amazonaws.com"
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Optional keys:
|
|
64
|
+
|
|
65
|
+
| Key | Default | Notes |
|
|
66
|
+
|---|---|---|
|
|
67
|
+
| `s3_region` | `"us-east-1"` | AWS region; mostly cosmetic for MinIO / R2 |
|
|
68
|
+
| `s3_endpoint_url` | empty (= AWS S3) | Set this for S3-compatible backends โ see table below |
|
|
69
|
+
| `s3_prefix` | empty | Default key prefix added by `upload(...)` when no destination is given |
|
|
70
|
+
| `s3_use_path_style` | `"false"` | Force path-style addressing (`endpoint/bucket/key` instead of `bucket.endpoint/key`). Typical for MinIO with custom domains. |
|
|
71
|
+
| `s3_verify_ssl` | `"true"` | Disable only for dev MinIO with self-signed certs |
|
|
72
|
+
|
|
73
|
+
## Endpoint URLs for common S3-compatible storage
|
|
74
|
+
|
|
75
|
+
Set `s3_endpoint_url` to:
|
|
76
|
+
|
|
77
|
+
| Provider | Endpoint |
|
|
78
|
+
|---|---|
|
|
79
|
+
| **AWS S3** | leave empty / unset |
|
|
80
|
+
| **MinIO** | `http://minio.example.com:9000` (or `https://...` with TLS) |
|
|
81
|
+
| **DigitalOcean Spaces** | `https://nyc3.digitaloceanspaces.com` (region in subdomain) |
|
|
82
|
+
| **Cloudflare R2** | `https://<account_id>.r2.cloudflarestorage.com` |
|
|
83
|
+
| **Backblaze B2 (S3 API)** | `https://s3.<region>.backblazeb2.com` |
|
|
84
|
+
| **Wasabi** | `https://s3.<region>.wasabisys.com` |
|
|
85
|
+
|
|
86
|
+
# Usage
|
|
87
|
+
|
|
88
|
+
For the full catalog of recipes (uploads / downloads / listings, S3-compatible endpoints โ MinIO / R2 / B2 / Spaces / Wasabi, temporary remote keys with auto-cleanup, mirroring with sftp-helper), see [๐ EXAMPLES.md](https://github.com/warith-harchaoui/bucket-helper/blob/main/EXAMPLES.md).
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
import bucket_helper as bh
|
|
92
|
+
|
|
93
|
+
# Load creds โ JSON / YAML / env / .env (auto-fallback in that order)
|
|
94
|
+
cred = bh.credentials("path/to/s3_config.json")
|
|
95
|
+
|
|
96
|
+
# Upload a local file
|
|
97
|
+
uri = bh.upload("local.txt", cred, "folder/uploaded.txt")
|
|
98
|
+
# uri == "s3://my-bucket/folder/uploaded.txt"
|
|
99
|
+
|
|
100
|
+
assert bh.exists(uri, cred)
|
|
101
|
+
|
|
102
|
+
# Download
|
|
103
|
+
bh.download(uri, "downloaded.txt", cred)
|
|
104
|
+
|
|
105
|
+
# List
|
|
106
|
+
for key in bh.list_prefix("folder/", cred):
|
|
107
|
+
print(key)
|
|
108
|
+
|
|
109
|
+
# Delete
|
|
110
|
+
bh.delete(uri, cred)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## MinIO example
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
cred = {
|
|
117
|
+
"s3_access_key": "minioadmin",
|
|
118
|
+
"s3_secret_key": "minioadmin",
|
|
119
|
+
"s3_bucket": "uploads",
|
|
120
|
+
"s3_https": "http://minio.example.com:9000/uploads",
|
|
121
|
+
"s3_endpoint_url": "http://minio.example.com:9000",
|
|
122
|
+
"s3_use_path_style": "true",
|
|
123
|
+
"s3_region": "us-east-1", # MinIO accepts any region string
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
bh.make_bucket("uploads", cred)
|
|
127
|
+
bh.upload("file.bin", cred, "file.bin")
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Stage-and-share with `remote_tempfile`
|
|
131
|
+
|
|
132
|
+
Drop a generated file at a unique random key, hand the public URL to a
|
|
133
|
+
downstream worker / webhook, and the object is deleted on block exit
|
|
134
|
+
(even if the body raises):
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
import bucket_helper as bh
|
|
138
|
+
import requests
|
|
139
|
+
|
|
140
|
+
cred = bh.credentials("path/to/s3_config.json")
|
|
141
|
+
|
|
142
|
+
with bh.remote_tempfile(cred, ext="json", prefix="runs") as (s3_addr, public_url):
|
|
143
|
+
bh.upload("payload.json", cred, s3_addr, content_type="application/json")
|
|
144
|
+
# Hand the URL to something that fetches it once.
|
|
145
|
+
requests.post("https://hook.example.com/process", json={"input_url": public_url}).raise_for_status()
|
|
146
|
+
# Object is gone here, no manual cleanup.
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
# Multi-surface exposure
|
|
150
|
+
|
|
151
|
+
Every public function in the library is also exposed as:
|
|
152
|
+
|
|
153
|
+
- **argparse CLI** โ `bucket-helper <subcommand>` (installed by default).
|
|
154
|
+
- **click CLI** โ `bucket-helper-click <subcommand>` (install `[cli]` extra).
|
|
155
|
+
- **FastAPI HTTP** โ `uvicorn bucket_helper.api:app --host 0.0.0.0 --port 8000` (install `[api]` extra).
|
|
156
|
+
- **MCP tools** โ `bucket-helper-mcp` (install `[api,mcp]` extras).
|
|
157
|
+
|
|
158
|
+
Both CLIs share the same subcommand names and flags โ pick your favourite.
|
|
159
|
+
|
|
160
|
+
## CLI examples
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
# argparse CLI (always available)
|
|
164
|
+
bucket-helper upload --config s3_config.json --input local.txt --key folder/uploaded.txt
|
|
165
|
+
bucket-helper exists --config s3_config.json --key folder/uploaded.txt
|
|
166
|
+
bucket-helper download --config s3_config.json --key folder/uploaded.txt --output back.txt
|
|
167
|
+
bucket-helper list --config s3_config.json --prefix folder/
|
|
168
|
+
bucket-helper delete --config s3_config.json --key folder/uploaded.txt
|
|
169
|
+
bucket-helper make-bucket --config s3_config.json --bucket new-bucket
|
|
170
|
+
bucket-helper tempfile --config s3_config.json --ext json --prefix runs
|
|
171
|
+
bucket-helper strip-path --config s3_config.json --address s3://my-bucket/path/to/obj
|
|
172
|
+
|
|
173
|
+
# click CLI โ same verbs, same flags
|
|
174
|
+
bucket-helper-click upload --config s3_config.json --input local.txt --key folder/uploaded.txt
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
## HTTP + MCP server
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
# Serve HTTP + MCP (default credentials picked up from BUCKET_HELPER_CONFIG)
|
|
181
|
+
BUCKET_HELPER_CONFIG=$PWD/s3_config.json bucket-helper-mcp
|
|
182
|
+
|
|
183
|
+
# Or run only FastAPI directly:
|
|
184
|
+
uvicorn bucket_helper.api:app --host 0.0.0.0 --port 8000
|
|
185
|
+
# โ Swagger UI at http://localhost:8000/docs
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Per-request credentials can also be sent as multipart form fields
|
|
189
|
+
(`s3_access_key` / `s3_secret_key` / `s3_bucket` / `s3_https` / โฆ).
|
|
190
|
+
|
|
191
|
+
## Docker
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
docker build -t bucket-helper .
|
|
195
|
+
docker run --rm -p 8000:8000 \
|
|
196
|
+
-e BUCKET_HELPER_CONFIG=/config/s3_config.json \
|
|
197
|
+
-v $PWD/s3_config.json:/config/s3_config.json:ro \
|
|
198
|
+
bucket-helper
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
See also: [LANDSCAPE.md](https://github.com/warith-harchaoui/bucket-helper/blob/main/LANDSCAPE.md) (competitive positioning) and
|
|
202
|
+
[GUI.md](https://github.com/warith-harchaoui/bucket-helper/blob/main/GUI.md) (visual product design plan).
|
|
203
|
+
|
|
204
|
+
# Author
|
|
205
|
+
- [Warith HARCHAOUI](https://linkedin.com/in/warith-harchaoui)
|
|
206
|
+
|
|
207
|
+
# Acknowledgements
|
|
208
|
+
Special thanks to [Mohamed Chelali](https://mchelali.github.io) and [Bachir Zerroug](https://www.linkedin.com/in/bachirzerroug) for fruitful discussions.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bucket Helper
|
|
3
|
+
|
|
4
|
+
Utility functions for AWS S3 and any S3-compatible object storage
|
|
5
|
+
(MinIO, Backblaze B2 S3 API, DigitalOcean Spaces, Cloudflare R2,
|
|
6
|
+
Wasabi, โฆ) via :mod:`boto3`. Same shape as ``sftp-helper``:
|
|
7
|
+
``credentials()`` loader + ``upload`` / ``download`` / ``delete`` /
|
|
8
|
+
``exists`` / ``list_prefix`` / ``make_bucket`` + a ``remote_tempfile``
|
|
9
|
+
context manager for stage-and-share flows.
|
|
10
|
+
|
|
11
|
+
Also exposes multi-surface entry points:
|
|
12
|
+
|
|
13
|
+
- Library (this module) โ the pure functions.
|
|
14
|
+
- argparse CLI โ :mod:`bucket_helper.cli_argparse` (``bucket-helper``).
|
|
15
|
+
- click CLI โ :mod:`bucket_helper.cli_click` (``bucket-helper-click``).
|
|
16
|
+
- FastAPI HTTP โ :mod:`bucket_helper.api` (``uvicorn bucket_helper.api:app``).
|
|
17
|
+
- MCP tools โ :mod:`bucket_helper.mcp` (``bucket-helper-mcp``).
|
|
18
|
+
|
|
19
|
+
Usage Example
|
|
20
|
+
-------------
|
|
21
|
+
>>> import bucket_helper as bh
|
|
22
|
+
>>> cred = bh.credentials("path/to/s3_config.json")
|
|
23
|
+
>>> uri = bh.upload("local.txt", cred, "folder/uploaded.txt")
|
|
24
|
+
>>> assert bh.exists(uri, cred)
|
|
25
|
+
>>> bh.download(uri, "back.txt", cred)
|
|
26
|
+
>>> bh.delete(uri, cred)
|
|
27
|
+
|
|
28
|
+
Author
|
|
29
|
+
------
|
|
30
|
+
Warith Harchaoui, Ph.D. โ https://linkedin.com/in/warith-harchaoui/
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
__author__ = "Warith Harchaoui, Ph.D."
|
|
34
|
+
__email__ = "warithmetics@deraison.ai"
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"credentials",
|
|
38
|
+
"get_client_s3",
|
|
39
|
+
"upload",
|
|
40
|
+
"download",
|
|
41
|
+
"delete",
|
|
42
|
+
"exists",
|
|
43
|
+
"list_prefix",
|
|
44
|
+
"make_bucket",
|
|
45
|
+
"remote_tempfile",
|
|
46
|
+
"strip_s3_path",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
from .main import (
|
|
50
|
+
credentials,
|
|
51
|
+
delete,
|
|
52
|
+
download,
|
|
53
|
+
exists,
|
|
54
|
+
get_client_s3,
|
|
55
|
+
list_prefix,
|
|
56
|
+
make_bucket,
|
|
57
|
+
remote_tempfile,
|
|
58
|
+
strip_s3_path,
|
|
59
|
+
upload,
|
|
60
|
+
)
|