genblaze-s3 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- genblaze_s3-0.2.0/.gitignore +76 -0
- genblaze_s3-0.2.0/PKG-INFO +150 -0
- genblaze_s3-0.2.0/README.md +132 -0
- genblaze_s3-0.2.0/genblaze_s3/__init__.py +5 -0
- genblaze_s3-0.2.0/genblaze_s3/backend.py +533 -0
- genblaze_s3-0.2.0/genblaze_s3/py.typed +0 -0
- genblaze_s3-0.2.0/pyproject.toml +36 -0
- genblaze_s3-0.2.0/tests/__init__.py +0 -0
- genblaze_s3-0.2.0/tests/test_s3_backend.py +604 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
|
|
7
|
+
# Distribution / packaging
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
*.egg-info/
|
|
11
|
+
*.egg
|
|
12
|
+
wheels/
|
|
13
|
+
/MANIFEST
|
|
14
|
+
|
|
15
|
+
# Virtual environments
|
|
16
|
+
.venv/
|
|
17
|
+
venv/
|
|
18
|
+
env/
|
|
19
|
+
ENV/
|
|
20
|
+
|
|
21
|
+
# IDE / editors
|
|
22
|
+
.vscode/
|
|
23
|
+
.idea/
|
|
24
|
+
*.swp
|
|
25
|
+
*.swo
|
|
26
|
+
*~
|
|
27
|
+
.project
|
|
28
|
+
.settings/
|
|
29
|
+
|
|
30
|
+
# Testing / coverage
|
|
31
|
+
.pytest_cache/
|
|
32
|
+
.coverage
|
|
33
|
+
.coverage.*
|
|
34
|
+
htmlcov/
|
|
35
|
+
coverage.xml
|
|
36
|
+
*.cover
|
|
37
|
+
|
|
38
|
+
# Type checking
|
|
39
|
+
.mypy_cache/
|
|
40
|
+
.pytype/
|
|
41
|
+
.pyre/
|
|
42
|
+
|
|
43
|
+
# Linting
|
|
44
|
+
.ruff_cache/
|
|
45
|
+
|
|
46
|
+
# OS files
|
|
47
|
+
.DS_Store
|
|
48
|
+
Thumbs.db
|
|
49
|
+
ehthumbs.db
|
|
50
|
+
|
|
51
|
+
# Environment / secrets
|
|
52
|
+
.env
|
|
53
|
+
.env.*
|
|
54
|
+
!.env.example
|
|
55
|
+
*.pem
|
|
56
|
+
*.key
|
|
57
|
+
credentials.json
|
|
58
|
+
|
|
59
|
+
# Jupyter
|
|
60
|
+
.ipynb_checkpoints/
|
|
61
|
+
|
|
62
|
+
# MkDocs build output
|
|
63
|
+
site/
|
|
64
|
+
|
|
65
|
+
# Claude Code — local/ephemeral
|
|
66
|
+
.claude/settings.local.json
|
|
67
|
+
.claude/worktrees/
|
|
68
|
+
CLAUDE.local.md
|
|
69
|
+
|
|
70
|
+
# Hypothesis test cache
|
|
71
|
+
.hypothesis/
|
|
72
|
+
|
|
73
|
+
# Temp files
|
|
74
|
+
*.tmp
|
|
75
|
+
*.bak
|
|
76
|
+
*.log
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: genblaze-s3
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: S3-compatible storage backend for genblaze (B2, R2, MinIO, AWS)
|
|
5
|
+
Project-URL: Documentation, https://github.com/backblaze-labs/genblaze
|
|
6
|
+
Project-URL: Repository, https://github.com/backblaze-labs/genblaze
|
|
7
|
+
Project-URL: Issues, https://github.com/backblaze-labs/genblaze/issues
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Typing :: Typed
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: boto3>=1.28
|
|
14
|
+
Requires-Dist: genblaze-core<0.2,>=0.1.0
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
<!-- last_verified: 2026-04-22 -->
|
|
20
|
+
# genblaze-s3
|
|
21
|
+
|
|
22
|
+
**S3-compatible storage backend for [genblaze](https://github.com/backblaze-labs/genblaze) AI media pipelines — durable, content-addressable, dedup-ready. Works with [Backblaze B2](https://www.backblaze.com/cloud-storage?utm_source=github&utm_medium=referral&utm_campaign=ai_artifacts&utm_content=genblaze) (recommended default), Cloudflare R2, MinIO, and AWS S3.**
|
|
23
|
+
|
|
24
|
+
`genblaze-s3` plugs into the genblaze `ObjectStorageSink` to persist AI-generated video, image, and audio — plus their SHA-256 provenance manifests — onto any S3-compatible object store. It handles streaming downloads from provider CDNs, SHA-256 hashing, multipart uploads with retries, pre-signed URLs for private buckets, and Object Lock retention for tamper-evident manifests on Backblaze B2.
|
|
25
|
+
|
|
26
|
+
## Why genblaze-s3
|
|
27
|
+
|
|
28
|
+
- **Durable by default** — Assets + manifests land in object storage, never stuck in a provider's expiring CDN URL.
|
|
29
|
+
- **Backblaze B2 first-class** — One-line `S3StorageBackend.for_backblaze()` helper, Object Lock support for immutable provenance.
|
|
30
|
+
- **Content-addressable dedup** — `KeyStrategy.CONTENT_ADDRESSABLE` stores each unique asset once by SHA-256.
|
|
31
|
+
- **Works with any S3 API** — AWS S3, Backblaze B2, Cloudflare R2, MinIO, SeaweedFS, Wasabi, Ceph.
|
|
32
|
+
- **Presigned URLs** — private buckets get time-limited URLs; public buckets get permanent `public_url_base` links.
|
|
33
|
+
- **Resilient multipart uploads** — credential-preserving retries, preflight checks, no partial writes.
|
|
34
|
+
|
|
35
|
+
## Backends
|
|
36
|
+
|
|
37
|
+
| Provider | Helper | Notes |
|
|
38
|
+
|---|---|---|
|
|
39
|
+
| **Backblaze B2** | `S3StorageBackend.for_backblaze("bucket")` | Reads `B2_KEY_ID` / `B2_APP_KEY`; Object Lock retention supported |
|
|
40
|
+
| AWS S3 | `S3StorageBackend(bucket="...", region="...")` | Standard AWS credential chain |
|
|
41
|
+
| Cloudflare R2 | `S3StorageBackend(bucket="...", endpoint_url="https://<acct>.r2.cloudflarestorage.com")` | |
|
|
42
|
+
| MinIO / self-hosted | `S3StorageBackend(bucket="...", endpoint_url="https://minio.example.com")` | |
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install genblaze-s3
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Quickstart — Backblaze B2 (recommended)
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
export B2_KEY_ID="..."
|
|
54
|
+
export B2_APP_KEY="..."
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from genblaze_core import KeyStrategy, ObjectStorageSink, Pipeline
|
|
59
|
+
from genblaze_s3 import S3StorageBackend
|
|
60
|
+
from genblaze_replicate import ReplicateProvider
|
|
61
|
+
|
|
62
|
+
backend = S3StorageBackend.for_backblaze(
|
|
63
|
+
"my-genblaze-bucket",
|
|
64
|
+
region="us-west-004",
|
|
65
|
+
# Optional: pass public_url_base for public buckets (get_url returns permanent URLs)
|
|
66
|
+
public_url_base="https://f004.backblazeb2.com/file/my-genblaze-bucket",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
sink = ObjectStorageSink(
|
|
70
|
+
backend,
|
|
71
|
+
prefix="genblaze-assets",
|
|
72
|
+
key_strategy=KeyStrategy.CONTENT_ADDRESSABLE, # dedupe by SHA-256
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
result = (
|
|
76
|
+
Pipeline("b2-demo")
|
|
77
|
+
.step(ReplicateProvider(), model="black-forest-labs/flux-schnell",
|
|
78
|
+
prompt="a photorealistic cat wearing a tiny spacesuit")
|
|
79
|
+
.run(sink=sink, timeout=120)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
for step in result.run.steps:
|
|
83
|
+
for asset in step.assets:
|
|
84
|
+
print(asset.url, asset.sha256)
|
|
85
|
+
|
|
86
|
+
backend.close()
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Resulting bucket layout with `CONTENT_ADDRESSABLE`:
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
genblaze-assets/
|
|
93
|
+
├── assets/{sha[:2]}/{sha[2:4]}/{sha}.ext # one object per unique asset
|
|
94
|
+
└── manifests/{run_id}.json # one manifest per run
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Switch to `KeyStrategy.HIERARCHICAL` for `runs/{date}/{run_id}/…` layout (better for run-grouped browsing, worse for dedup).
|
|
98
|
+
|
|
99
|
+
## Quickstart — AWS S3
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
export AWS_ACCESS_KEY_ID="..."
|
|
103
|
+
export AWS_SECRET_ACCESS_KEY="..."
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from genblaze_s3 import S3StorageBackend
|
|
108
|
+
|
|
109
|
+
backend = S3StorageBackend(bucket="my-genblaze-bucket", region="us-east-1")
|
|
110
|
+
# get_url() returns pre-signed URLs when public_url_base is not set
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Quickstart — Cloudflare R2 / MinIO
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from genblaze_s3 import S3StorageBackend
|
|
117
|
+
|
|
118
|
+
# R2
|
|
119
|
+
backend = S3StorageBackend(
|
|
120
|
+
bucket="my-bucket",
|
|
121
|
+
endpoint_url="https://<account-id>.r2.cloudflarestorage.com",
|
|
122
|
+
access_key_id="...", secret_access_key="...",
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# MinIO
|
|
126
|
+
backend = S3StorageBackend(
|
|
127
|
+
bucket="my-bucket",
|
|
128
|
+
endpoint_url="https://minio.example.com",
|
|
129
|
+
access_key_id="...", secret_access_key="...",
|
|
130
|
+
)
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## Object Lock for immutable manifests (Backblaze B2)
|
|
134
|
+
|
|
135
|
+
Genblaze can apply Object Lock retention to uploaded manifests, producing tamper-evident provenance suitable for compliance, legal, and content-authenticity workflows. See the main repo docs for the [Object Lock guide](https://github.com/backblaze-labs/genblaze/tree/main/docs/features).
|
|
136
|
+
|
|
137
|
+
## Documentation
|
|
138
|
+
|
|
139
|
+
- **Main repo**: https://github.com/backblaze-labs/genblaze
|
|
140
|
+
- **Storage feature doc**: https://github.com/backblaze-labs/genblaze/blob/main/docs/features/object-storage.md
|
|
141
|
+
- **Runnable examples**: [`b2_storage_pipeline.py`](https://github.com/backblaze-labs/genblaze/blob/main/examples/b2_storage_pipeline.py), [`s3_storage_pipeline.py`](https://github.com/backblaze-labs/genblaze/blob/main/examples/s3_storage_pipeline.py)
|
|
142
|
+
|
|
143
|
+
## Related packages
|
|
144
|
+
|
|
145
|
+
- [`genblaze-core`](https://pypi.org/project/genblaze-core/) — the pipeline SDK
|
|
146
|
+
- Provider adapters: [`genblaze-openai`](https://pypi.org/project/genblaze-openai/) · [`genblaze-google`](https://pypi.org/project/genblaze-google/) · [`genblaze-runway`](https://pypi.org/project/genblaze-runway/) · [`genblaze-luma`](https://pypi.org/project/genblaze-luma/) · [`genblaze-replicate`](https://pypi.org/project/genblaze-replicate/)
|
|
147
|
+
|
|
148
|
+
## License
|
|
149
|
+
|
|
150
|
+
MIT
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
<!-- last_verified: 2026-04-22 -->
|
|
2
|
+
# genblaze-s3
|
|
3
|
+
|
|
4
|
+
**S3-compatible storage backend for [genblaze](https://github.com/backblaze-labs/genblaze) AI media pipelines — durable, content-addressable, dedup-ready. Works with [Backblaze B2](https://www.backblaze.com/cloud-storage?utm_source=github&utm_medium=referral&utm_campaign=ai_artifacts&utm_content=genblaze) (recommended default), Cloudflare R2, MinIO, and AWS S3.**
|
|
5
|
+
|
|
6
|
+
`genblaze-s3` plugs into the genblaze `ObjectStorageSink` to persist AI-generated video, image, and audio — plus their SHA-256 provenance manifests — onto any S3-compatible object store. It handles streaming downloads from provider CDNs, SHA-256 hashing, multipart uploads with retries, pre-signed URLs for private buckets, and Object Lock retention for tamper-evident manifests on Backblaze B2.
|
|
7
|
+
|
|
8
|
+
## Why genblaze-s3
|
|
9
|
+
|
|
10
|
+
- **Durable by default** — Assets + manifests land in object storage, never stuck in a provider's expiring CDN URL.
|
|
11
|
+
- **Backblaze B2 first-class** — One-line `S3StorageBackend.for_backblaze()` helper, Object Lock support for immutable provenance.
|
|
12
|
+
- **Content-addressable dedup** — `KeyStrategy.CONTENT_ADDRESSABLE` stores each unique asset once by SHA-256.
|
|
13
|
+
- **Works with any S3 API** — AWS S3, Backblaze B2, Cloudflare R2, MinIO, SeaweedFS, Wasabi, Ceph.
|
|
14
|
+
- **Presigned URLs** — private buckets get time-limited URLs; public buckets get permanent `public_url_base` links.
|
|
15
|
+
- **Resilient multipart uploads** — credential-preserving retries, preflight checks, no partial writes.
|
|
16
|
+
|
|
17
|
+
## Backends
|
|
18
|
+
|
|
19
|
+
| Provider | Helper | Notes |
|
|
20
|
+
|---|---|---|
|
|
21
|
+
| **Backblaze B2** | `S3StorageBackend.for_backblaze("bucket")` | Reads `B2_KEY_ID` / `B2_APP_KEY`; Object Lock retention supported |
|
|
22
|
+
| AWS S3 | `S3StorageBackend(bucket="...", region="...")` | Standard AWS credential chain |
|
|
23
|
+
| Cloudflare R2 | `S3StorageBackend(bucket="...", endpoint_url="https://<acct>.r2.cloudflarestorage.com")` | |
|
|
24
|
+
| MinIO / self-hosted | `S3StorageBackend(bucket="...", endpoint_url="https://minio.example.com")` | |
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install genblaze-s3
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Quickstart — Backblaze B2 (recommended)
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
export B2_KEY_ID="..."
|
|
36
|
+
export B2_APP_KEY="..."
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from genblaze_core import KeyStrategy, ObjectStorageSink, Pipeline
|
|
41
|
+
from genblaze_s3 import S3StorageBackend
|
|
42
|
+
from genblaze_replicate import ReplicateProvider
|
|
43
|
+
|
|
44
|
+
backend = S3StorageBackend.for_backblaze(
|
|
45
|
+
"my-genblaze-bucket",
|
|
46
|
+
region="us-west-004",
|
|
47
|
+
# Optional: pass public_url_base for public buckets (get_url returns permanent URLs)
|
|
48
|
+
public_url_base="https://f004.backblazeb2.com/file/my-genblaze-bucket",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
sink = ObjectStorageSink(
|
|
52
|
+
backend,
|
|
53
|
+
prefix="genblaze-assets",
|
|
54
|
+
key_strategy=KeyStrategy.CONTENT_ADDRESSABLE, # dedupe by SHA-256
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
result = (
|
|
58
|
+
Pipeline("b2-demo")
|
|
59
|
+
.step(ReplicateProvider(), model="black-forest-labs/flux-schnell",
|
|
60
|
+
prompt="a photorealistic cat wearing a tiny spacesuit")
|
|
61
|
+
.run(sink=sink, timeout=120)
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
for step in result.run.steps:
|
|
65
|
+
for asset in step.assets:
|
|
66
|
+
print(asset.url, asset.sha256)
|
|
67
|
+
|
|
68
|
+
backend.close()
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Resulting bucket layout with `CONTENT_ADDRESSABLE`:
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
genblaze-assets/
|
|
75
|
+
├── assets/{sha[:2]}/{sha[2:4]}/{sha}.ext # one object per unique asset
|
|
76
|
+
└── manifests/{run_id}.json # one manifest per run
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Switch to `KeyStrategy.HIERARCHICAL` for `runs/{date}/{run_id}/…` layout (better for run-grouped browsing, worse for dedup).
|
|
80
|
+
|
|
81
|
+
## Quickstart — AWS S3
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
export AWS_ACCESS_KEY_ID="..."
|
|
85
|
+
export AWS_SECRET_ACCESS_KEY="..."
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from genblaze_s3 import S3StorageBackend
|
|
90
|
+
|
|
91
|
+
backend = S3StorageBackend(bucket="my-genblaze-bucket", region="us-east-1")
|
|
92
|
+
# get_url() returns pre-signed URLs when public_url_base is not set
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Quickstart — Cloudflare R2 / MinIO
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from genblaze_s3 import S3StorageBackend
|
|
99
|
+
|
|
100
|
+
# R2
|
|
101
|
+
backend = S3StorageBackend(
|
|
102
|
+
bucket="my-bucket",
|
|
103
|
+
endpoint_url="https://<account-id>.r2.cloudflarestorage.com",
|
|
104
|
+
access_key_id="...", secret_access_key="...",
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# MinIO
|
|
108
|
+
backend = S3StorageBackend(
|
|
109
|
+
bucket="my-bucket",
|
|
110
|
+
endpoint_url="https://minio.example.com",
|
|
111
|
+
access_key_id="...", secret_access_key="...",
|
|
112
|
+
)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Object Lock for immutable manifests (Backblaze B2)
|
|
116
|
+
|
|
117
|
+
Genblaze can apply Object Lock retention to uploaded manifests, producing tamper-evident provenance suitable for compliance, legal, and content-authenticity workflows. See the main repo docs for the [Object Lock guide](https://github.com/backblaze-labs/genblaze/tree/main/docs/features).
|
|
118
|
+
|
|
119
|
+
## Documentation
|
|
120
|
+
|
|
121
|
+
- **Main repo**: https://github.com/backblaze-labs/genblaze
|
|
122
|
+
- **Storage feature doc**: https://github.com/backblaze-labs/genblaze/blob/main/docs/features/object-storage.md
|
|
123
|
+
- **Runnable examples**: [`b2_storage_pipeline.py`](https://github.com/backblaze-labs/genblaze/blob/main/examples/b2_storage_pipeline.py), [`s3_storage_pipeline.py`](https://github.com/backblaze-labs/genblaze/blob/main/examples/s3_storage_pipeline.py)
|
|
124
|
+
|
|
125
|
+
## Related packages
|
|
126
|
+
|
|
127
|
+
- [`genblaze-core`](https://pypi.org/project/genblaze-core/) — the pipeline SDK
|
|
128
|
+
- Provider adapters: [`genblaze-openai`](https://pypi.org/project/genblaze-openai/) · [`genblaze-google`](https://pypi.org/project/genblaze-google/) · [`genblaze-runway`](https://pypi.org/project/genblaze-runway/) · [`genblaze-luma`](https://pypi.org/project/genblaze-luma/) · [`genblaze-replicate`](https://pypi.org/project/genblaze-replicate/)
|
|
129
|
+
|
|
130
|
+
## License
|
|
131
|
+
|
|
132
|
+
MIT
|