beam-network-sdk 0.7.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of beam-network-sdk might be problematic. Click here for more details.
- beam_network_sdk-0.7.3/.gitignore +52 -0
- beam_network_sdk-0.7.3/CHANGELOG.md +18 -0
- beam_network_sdk-0.7.3/PKG-INFO +255 -0
- beam_network_sdk-0.7.3/README.md +215 -0
- beam_network_sdk-0.7.3/beam_network_sdk/__init__.py +99 -0
- beam_network_sdk-0.7.3/beam_network_sdk/_client.py +2113 -0
- beam_network_sdk-0.7.3/beam_network_sdk/_diagnostics.py +50 -0
- beam_network_sdk-0.7.3/beam_network_sdk/_multipart_limits.py +27 -0
- beam_network_sdk-0.7.3/beam_network_sdk/_nats_control.py +768 -0
- beam_network_sdk-0.7.3/beam_network_sdk/_retry.py +67 -0
- beam_network_sdk-0.7.3/beam_network_sdk/_validation.py +13 -0
- beam_network_sdk-0.7.3/beam_network_sdk/cli.py +2038 -0
- beam_network_sdk-0.7.3/beam_network_sdk/exceptions.py +64 -0
- beam_network_sdk-0.7.3/beam_network_sdk/huggingface.py +426 -0
- beam_network_sdk-0.7.3/beam_network_sdk/models.py +561 -0
- beam_network_sdk-0.7.3/beam_network_sdk/provider_signing.py +670 -0
- beam_network_sdk-0.7.3/examples/push_records.py +45 -0
- beam_network_sdk-0.7.3/pyproject.toml +76 -0
- beam_network_sdk-0.7.3/tests/test_functional_client.py +729 -0
- beam_network_sdk-0.7.3/tests/test_huggingface.py +403 -0
- beam_network_sdk-0.7.3/tests/test_nats_auth_refresh.py +98 -0
- beam_network_sdk-0.7.3/tests/test_provider_signing.py +248 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
.eggs/
|
|
8
|
+
*.egg
|
|
9
|
+
.venv/
|
|
10
|
+
venv/
|
|
11
|
+
sdks/python/.venv-wsl/
|
|
12
|
+
|
|
13
|
+
# JavaScript / TypeScript
|
|
14
|
+
node_modules/
|
|
15
|
+
*.tsbuildinfo
|
|
16
|
+
sdks/typescript/dist/
|
|
17
|
+
sdks/typescript/*.tgz
|
|
18
|
+
packages/cli/dist/
|
|
19
|
+
packages/cli/*.tgz
|
|
20
|
+
packages/web-sdk/*.tgz
|
|
21
|
+
packages/web-sdk-server/*.tgz
|
|
22
|
+
coverage/
|
|
23
|
+
docs/api/
|
|
24
|
+
|
|
25
|
+
# Go
|
|
26
|
+
sdks/go/bin/
|
|
27
|
+
|
|
28
|
+
# Rust
|
|
29
|
+
sdks/rust/target/
|
|
30
|
+
|
|
31
|
+
# IDE
|
|
32
|
+
.vscode/
|
|
33
|
+
.idea/
|
|
34
|
+
*.swp
|
|
35
|
+
*.swo
|
|
36
|
+
|
|
37
|
+
# OS
|
|
38
|
+
.DS_Store
|
|
39
|
+
Thumbs.db
|
|
40
|
+
|
|
41
|
+
# Env
|
|
42
|
+
.env
|
|
43
|
+
.env.local
|
|
44
|
+
|
|
45
|
+
# Local transfer artifacts
|
|
46
|
+
sdks/python/test_downloads/
|
|
47
|
+
sdks/python/.local-beam-auth.json
|
|
48
|
+
sdks/python/.local-runtime/
|
|
49
|
+
sdks/python/beam-e2e-source.bin
|
|
50
|
+
sdks/python/moto.log
|
|
51
|
+
sdks/python/LOCAL_CLI_S3_E2E.md
|
|
52
|
+
sdks/python/tools/local_cli_e2e.py
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.7.2
|
|
4
|
+
|
|
5
|
+
- Reuse the same default route generation for idempotent prepare retries.
|
|
6
|
+
|
|
7
|
+
## 0.7.1
|
|
8
|
+
|
|
9
|
+
- Default encoded route messages to 8 MiB while preserving explicit overrides.
|
|
10
|
+
- Expose `BeamRouteRecoveryPendingError` when process-local route replay remains active.
|
|
11
|
+
|
|
12
|
+
## 0.6.0
|
|
13
|
+
|
|
14
|
+
- Simplified the Python SDK to transfer creation and transfer lifecycle management.
|
|
15
|
+
- Kept `beam-send` as the only packaged CLI entry point.
|
|
16
|
+
- Removed local receiver, sender, worker, sink, API wrapper, and MCP modules.
|
|
17
|
+
- Removed stream, destination-manager, network, orchestrator, worker, and security managers from `BeamSDK`.
|
|
18
|
+
- Kept provider-aware transfer preparation for S3, R2, GCS, Azure, and Hippius models.
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: beam-network-sdk
|
|
3
|
+
Version: 0.7.3
|
|
4
|
+
Summary: Client SDK for the BEAM decentralized data pipeline
|
|
5
|
+
Project-URL: Homepage, https://b1m.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.b1m.ai/docs/sdk
|
|
7
|
+
Project-URL: Repository, https://github.com/Beam-Network/beam-sdk
|
|
8
|
+
Project-URL: Issues, https://github.com/Beam-Network/beam-sdk/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/Beam-Network/beam-sdk/blob/prod/sdks/python/CHANGELOG.md
|
|
10
|
+
Author: Beam Network
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
Keywords: bandwidth,beam,data-transfer,s3,storage
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Topic :: System :: Networking
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: click<9.0,>=8.0.0
|
|
23
|
+
Requires-Dist: httpx<1.0,>=0.25.0
|
|
24
|
+
Requires-Dist: msgpack<2.0,>=1.0.0
|
|
25
|
+
Requires-Dist: nats-py<3.0,>=2.7.0
|
|
26
|
+
Requires-Dist: pydantic<3.0,>=2.0.0
|
|
27
|
+
Provides-Extra: all
|
|
28
|
+
Requires-Dist: boto3<2.0,>=1.28.0; extra == 'all'
|
|
29
|
+
Requires-Dist: google-cloud-storage<4.0,>=2.16.0; extra == 'all'
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
33
|
+
Provides-Extra: gcs
|
|
34
|
+
Requires-Dist: google-cloud-storage<4.0,>=2.16.0; extra == 'gcs'
|
|
35
|
+
Provides-Extra: r2
|
|
36
|
+
Requires-Dist: boto3<2.0,>=1.28.0; extra == 'r2'
|
|
37
|
+
Provides-Extra: s3
|
|
38
|
+
Requires-Dist: boto3<2.0,>=1.28.0; extra == 's3'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# BEAM SDK for Python
|
|
42
|
+
|
|
43
|
+
The Python SDK for BEAM 0.6.0 is now transfer-only. It exposes `BeamSDK.transfers` for transfer creation, provider-aware preparation, distribution, cancellation, status reconciliation, and event-driven completion waiting.
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install beam-network-sdk
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Optional extras:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install beam-network-sdk[s3] # S3 provider signing
|
|
55
|
+
pip install beam-network-sdk[r2] # Cloudflare R2 provider signing
|
|
56
|
+
pip install beam-network-sdk[gcs] # GCS provider models
|
|
57
|
+
pip install beam-network-sdk[dev] # lint/typecheck tooling
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Configuration
|
|
61
|
+
|
|
62
|
+
| Setting | Purpose |
|
|
63
|
+
| ------- | ------- |
|
|
64
|
+
| `BEAM_NATS_URL` | Beam NATS lifecycle endpoint when neither `nats_url` nor `environment` is passed. |
|
|
65
|
+
| `BEAM_API_KEY` | API key authentication. |
|
|
66
|
+
|
|
67
|
+
Pass `api_key` explicitly or set `BEAM_API_KEY`. Provide either `nats_url` or `environment`, not both.
|
|
68
|
+
|
|
69
|
+
## Basic Transfer
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
import asyncio
|
|
73
|
+
|
|
74
|
+
from beam_network_sdk import BeamSDK
|
|
75
|
+
from beam_network_sdk.models import DestConfig, SourceConfig
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
async def main() -> None:
|
|
79
|
+
async with BeamSDK(api_key="bk_your_key", environment="dev") as beam:
|
|
80
|
+
transfer = await beam.transfers.create(
|
|
81
|
+
sources=[
|
|
82
|
+
SourceConfig(
|
|
83
|
+
type="http",
|
|
84
|
+
url="https://downloads.example.com/report.parquet",
|
|
85
|
+
)
|
|
86
|
+
],
|
|
87
|
+
destinations=[
|
|
88
|
+
DestConfig(
|
|
89
|
+
type="http",
|
|
90
|
+
url="https://storage.example.com/ingest/report.parquet",
|
|
91
|
+
)
|
|
92
|
+
],
|
|
93
|
+
total_size=104_857_600,
|
|
94
|
+
name="daily-report",
|
|
95
|
+
progressive_mode=True,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
await beam.transfers.distribute(transfer.transfer_id)
|
|
99
|
+
status = await beam.transfers.wait_complete(transfer.transfer_id)
|
|
100
|
+
print(status.status)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
asyncio.run(main())
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Provider-Aware Transfer
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import asyncio
|
|
110
|
+
|
|
111
|
+
from beam_network_sdk import BeamSDK
|
|
112
|
+
from beam_network_sdk.models import S3ProviderDestination, S3ProviderSource
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def main() -> None:
|
|
116
|
+
async with BeamSDK(api_key="bk_your_key", environment="dev") as beam:
|
|
117
|
+
transfer = await beam.transfers.prepare_provider_transfer(
|
|
118
|
+
sources=[
|
|
119
|
+
S3ProviderSource(
|
|
120
|
+
bucket="source-bucket",
|
|
121
|
+
key="exports/report.parquet",
|
|
122
|
+
region="us-east-1",
|
|
123
|
+
access_key_id="...",
|
|
124
|
+
secret_access_key="...",
|
|
125
|
+
)
|
|
126
|
+
],
|
|
127
|
+
destinations=[
|
|
128
|
+
S3ProviderDestination(
|
|
129
|
+
bucket="dest-bucket",
|
|
130
|
+
key="imports/report.parquet",
|
|
131
|
+
region="us-east-1",
|
|
132
|
+
access_key_id="...",
|
|
133
|
+
secret_access_key="...",
|
|
134
|
+
)
|
|
135
|
+
],
|
|
136
|
+
distribute=True,
|
|
137
|
+
)
|
|
138
|
+
print(transfer.transfer_id)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
asyncio.run(main())
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Provider credentials stay in the SDK process. BeamCore receives prepared HTTP source URLs and signed destination routes.
|
|
145
|
+
|
|
146
|
+
## Hugging Face Hub
|
|
147
|
+
|
|
148
|
+
`HuggingFaceProviderSource` and `HuggingFaceProviderDestination` move a file in or
|
|
149
|
+
out of a Hub repo. The token stays in this process: a source resolves to the
|
|
150
|
+
presigned CDN URL the Hub redirects to, and a destination uses the presigned part
|
|
151
|
+
URLs the Hub's LFS batch endpoint issues.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
import asyncio
|
|
155
|
+
|
|
156
|
+
from beam_network_sdk import BeamSDK
|
|
157
|
+
from beam_network_sdk.models import HuggingFaceProviderSource, S3ProviderDestination
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
async def main() -> None:
|
|
161
|
+
async with BeamSDK(api_key="bk_your_key", environment="dev") as beam:
|
|
162
|
+
transfer = await beam.transfers.prepare_provider_transfer(
|
|
163
|
+
sources=[
|
|
164
|
+
HuggingFaceProviderSource(
|
|
165
|
+
repo_id="org/dataset",
|
|
166
|
+
path="data/train.parquet",
|
|
167
|
+
repo_type="dataset",
|
|
168
|
+
token="hf_...",
|
|
169
|
+
)
|
|
170
|
+
],
|
|
171
|
+
destinations=[
|
|
172
|
+
S3ProviderDestination(
|
|
173
|
+
bucket="archive",
|
|
174
|
+
key="datasets/train.parquet",
|
|
175
|
+
access_key_id="...",
|
|
176
|
+
secret_access_key="...",
|
|
177
|
+
)
|
|
178
|
+
],
|
|
179
|
+
distribute=True,
|
|
180
|
+
)
|
|
181
|
+
# wait_complete commits any Hugging Face destination once the parts land.
|
|
182
|
+
await beam.transfers.wait_complete(transfer.transfer_id)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
asyncio.run(main())
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
The CLI covers the same flow with `hf://` URIs:
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
beam-send hf-transfer \
|
|
192
|
+
hf://datasets/org/dataset@main/data/train.parquet \
|
|
193
|
+
hf://datasets/org/mirror@main/data/train.parquet
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Constraints, all of them the Hub's rather than Beam's — see
|
|
197
|
+
[`HUGGINGFACE_PROVIDER.md`](../../HUGGINGFACE_PROVIDER.md) for the full protocol:
|
|
198
|
+
|
|
199
|
+
- **Sources are large-file (LFS or Xet) content.** Only those get the
|
|
200
|
+
credential-free CDN redirect; a small regular file is served inline and is
|
|
201
|
+
rejected with that reason. Hub **buckets** work as sources too, but cannot be
|
|
202
|
+
written to — they expose no LFS endpoint.
|
|
203
|
+
- **A destination needs the object's sha256 up front**, because the LFS batch
|
|
204
|
+
endpoint will not issue upload URLs without it. When the source is not itself a
|
|
205
|
+
Hub file, set `allow_source_rehash=True` to let the SDK read the source once to
|
|
206
|
+
compute it.
|
|
207
|
+
- **One source per Hugging Face destination**, because the Hub dictates the part
|
|
208
|
+
size and a plan carries a single chunk size.
|
|
209
|
+
- If you do not call `wait_complete`, call
|
|
210
|
+
`finalize_huggingface_uploads(transfer_id)` yourself once the transfer completes
|
|
211
|
+
— without it the parts are uploaded but no commit is made and the file does not
|
|
212
|
+
appear in the repo.
|
|
213
|
+
|
|
214
|
+
## Route Streaming And Payload Size
|
|
215
|
+
|
|
216
|
+
Provider transfers use `transfer-client-control/v6`. Every reply carries Runtime and transport epochs; prepare and every route-stream message carry a UUID route generation. S3 and R2 retain direct multipart UploadPart/ListParts/HEAD handling. Hippius and Hugging Face destinations take a plain PUT per chunk instead, so they carry no multipart group manifest.
|
|
217
|
+
|
|
218
|
+
The SDK signs up to 64 routes concurrently by default and emits 2,048-route logical batches, split only when encoded MessagePack requests exceed the 8 MiB default guard. Explicit positive overrides remain supported. A single route larger than the configured limit fails before publication with both sizes in the error. The stream ID derives from the transfer, selected flow, and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Lifecycle mutations retry transient failures up to three times with the same request identity. Completion waits subscribe to the API-key-owned, at-most-once terminal signal before the first status read and reconcile every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. `async with BeamSDK(...)` closes signing workers, terminal waiters, and NATS resources; callers not using the context manager must call `await beam.close()`.
|
|
219
|
+
|
|
220
|
+
Hippius uses canonical non-multipart `signed_url` routes, so its manifest is empty and group-level final HEAD verification is not available from that provider flow. GCS and Azure provider signing remain unimplemented.
|
|
221
|
+
|
|
222
|
+
NATS requires this guard because the broker rejects messages above its configured `max_payload`. This limit applies only to lifecycle/control metadata; transfer file bytes do not flow through NATS.
|
|
223
|
+
|
|
224
|
+
## Restart Recovery
|
|
225
|
+
|
|
226
|
+
The SDK keeps one in-memory recovery lease per active transfer and one `runtime.hello` monitor per active shard. The lease is installed before route-stream begin, and a per-transfer lock coalesces initial streaming with replay. Multipart recovery retains the existing upload IDs and compact group state, then re-signs only the expiring route and commit controls. Runtime epoch changes invalidate cached auth, coalesce `transfer.resume`, and regenerate routes under a fresh generation; transport-only changes replay only when Runtime reports routes missing or expired. Provider inputs are released on terminal status or `close()`. Manual `attach_signed_urls` requires `route_generation_id`, plan fingerprint/checksum, and an async `recovery_factory`. No signed URL, credential, or recovery journal is written locally. Foreground cancellation, deadline expiry, and Runtime state-loss responses keep the background lease and retained multipart upload alive.
|
|
227
|
+
|
|
228
|
+
## CLI
|
|
229
|
+
|
|
230
|
+
`beam-send` creates, distributes, monitors, and waits for transfers:
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
beam-send create \
|
|
234
|
+
--source '{"type":"http","url":"https://downloads.example.com/report.parquet"}' \
|
|
235
|
+
--destination '{"type":"http","url":"https://storage.example.com/ingest/report.parquet"}' \
|
|
236
|
+
--total-size 104857600 \
|
|
237
|
+
--wait
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
Convenience commands for object storage remain under `beam-send`, for example:
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
beam-send s3-transfer s3://source-bucket/file.bin s3://dest-bucket/file.bin \
|
|
244
|
+
--server http://23.92.16.104:8000 \
|
|
245
|
+
--api-key bk_your_key \
|
|
246
|
+
--aws-access-key YOUR_KEY \
|
|
247
|
+
--aws-secret-key YOUR_SECRET \
|
|
248
|
+
--aws-region us-east-1
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
Only `beam-send` is packaged.
|
|
252
|
+
|
|
253
|
+
## Multi-Language Repository
|
|
254
|
+
|
|
255
|
+
The Python package lives alongside TypeScript and Go SDKs under `sdks/`. Shared API contracts live in `../../specs/`, and release instructions live in `../../docs/RELEASE.md`.
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# BEAM SDK for Python
|
|
2
|
+
|
|
3
|
+
The Python SDK for BEAM 0.6.0 is now transfer-only. It exposes `BeamSDK.transfers` for transfer creation, provider-aware preparation, distribution, cancellation, status reconciliation, and event-driven completion waiting.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install beam-network-sdk
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Optional extras:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install beam-network-sdk[s3] # S3 provider signing
|
|
15
|
+
pip install beam-network-sdk[r2] # Cloudflare R2 provider signing
|
|
16
|
+
pip install beam-network-sdk[gcs] # GCS provider models
|
|
17
|
+
pip install beam-network-sdk[dev] # lint/typecheck tooling
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Configuration
|
|
21
|
+
|
|
22
|
+
| Setting | Purpose |
|
|
23
|
+
| ------- | ------- |
|
|
24
|
+
| `BEAM_NATS_URL` | Beam NATS lifecycle endpoint when neither `nats_url` nor `environment` is passed. |
|
|
25
|
+
| `BEAM_API_KEY` | API key authentication. |
|
|
26
|
+
|
|
27
|
+
Pass `api_key` explicitly or set `BEAM_API_KEY`. Provide either `nats_url` or `environment`, not both.
|
|
28
|
+
|
|
29
|
+
## Basic Transfer
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import asyncio
|
|
33
|
+
|
|
34
|
+
from beam_network_sdk import BeamSDK
|
|
35
|
+
from beam_network_sdk.models import DestConfig, SourceConfig
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def main() -> None:
|
|
39
|
+
async with BeamSDK(api_key="bk_your_key", environment="dev") as beam:
|
|
40
|
+
transfer = await beam.transfers.create(
|
|
41
|
+
sources=[
|
|
42
|
+
SourceConfig(
|
|
43
|
+
type="http",
|
|
44
|
+
url="https://downloads.example.com/report.parquet",
|
|
45
|
+
)
|
|
46
|
+
],
|
|
47
|
+
destinations=[
|
|
48
|
+
DestConfig(
|
|
49
|
+
type="http",
|
|
50
|
+
url="https://storage.example.com/ingest/report.parquet",
|
|
51
|
+
)
|
|
52
|
+
],
|
|
53
|
+
total_size=104_857_600,
|
|
54
|
+
name="daily-report",
|
|
55
|
+
progressive_mode=True,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
await beam.transfers.distribute(transfer.transfer_id)
|
|
59
|
+
status = await beam.transfers.wait_complete(transfer.transfer_id)
|
|
60
|
+
print(status.status)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
asyncio.run(main())
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Provider-Aware Transfer
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
import asyncio
|
|
70
|
+
|
|
71
|
+
from beam_network_sdk import BeamSDK
|
|
72
|
+
from beam_network_sdk.models import S3ProviderDestination, S3ProviderSource
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
async def main() -> None:
|
|
76
|
+
async with BeamSDK(api_key="bk_your_key", environment="dev") as beam:
|
|
77
|
+
transfer = await beam.transfers.prepare_provider_transfer(
|
|
78
|
+
sources=[
|
|
79
|
+
S3ProviderSource(
|
|
80
|
+
bucket="source-bucket",
|
|
81
|
+
key="exports/report.parquet",
|
|
82
|
+
region="us-east-1",
|
|
83
|
+
access_key_id="...",
|
|
84
|
+
secret_access_key="...",
|
|
85
|
+
)
|
|
86
|
+
],
|
|
87
|
+
destinations=[
|
|
88
|
+
S3ProviderDestination(
|
|
89
|
+
bucket="dest-bucket",
|
|
90
|
+
key="imports/report.parquet",
|
|
91
|
+
region="us-east-1",
|
|
92
|
+
access_key_id="...",
|
|
93
|
+
secret_access_key="...",
|
|
94
|
+
)
|
|
95
|
+
],
|
|
96
|
+
distribute=True,
|
|
97
|
+
)
|
|
98
|
+
print(transfer.transfer_id)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
asyncio.run(main())
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Provider credentials stay in the SDK process. BeamCore receives prepared HTTP source URLs and signed destination routes.
|
|
105
|
+
|
|
106
|
+
## Hugging Face Hub
|
|
107
|
+
|
|
108
|
+
`HuggingFaceProviderSource` and `HuggingFaceProviderDestination` move a file in or
|
|
109
|
+
out of a Hub repo. The token stays in this process: a source resolves to the
|
|
110
|
+
presigned CDN URL the Hub redirects to, and a destination uses the presigned part
|
|
111
|
+
URLs the Hub's LFS batch endpoint issues.
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
import asyncio
|
|
115
|
+
|
|
116
|
+
from beam_network_sdk import BeamSDK
|
|
117
|
+
from beam_network_sdk.models import HuggingFaceProviderSource, S3ProviderDestination
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
async def main() -> None:
|
|
121
|
+
async with BeamSDK(api_key="bk_your_key", environment="dev") as beam:
|
|
122
|
+
transfer = await beam.transfers.prepare_provider_transfer(
|
|
123
|
+
sources=[
|
|
124
|
+
HuggingFaceProviderSource(
|
|
125
|
+
repo_id="org/dataset",
|
|
126
|
+
path="data/train.parquet",
|
|
127
|
+
repo_type="dataset",
|
|
128
|
+
token="hf_...",
|
|
129
|
+
)
|
|
130
|
+
],
|
|
131
|
+
destinations=[
|
|
132
|
+
S3ProviderDestination(
|
|
133
|
+
bucket="archive",
|
|
134
|
+
key="datasets/train.parquet",
|
|
135
|
+
access_key_id="...",
|
|
136
|
+
secret_access_key="...",
|
|
137
|
+
)
|
|
138
|
+
],
|
|
139
|
+
distribute=True,
|
|
140
|
+
)
|
|
141
|
+
# wait_complete commits any Hugging Face destination once the parts land.
|
|
142
|
+
await beam.transfers.wait_complete(transfer.transfer_id)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
asyncio.run(main())
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
The CLI covers the same flow with `hf://` URIs:
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
beam-send hf-transfer \
|
|
152
|
+
hf://datasets/org/dataset@main/data/train.parquet \
|
|
153
|
+
hf://datasets/org/mirror@main/data/train.parquet
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Constraints, all of them the Hub's rather than Beam's — see
|
|
157
|
+
[`HUGGINGFACE_PROVIDER.md`](../../HUGGINGFACE_PROVIDER.md) for the full protocol:
|
|
158
|
+
|
|
159
|
+
- **Sources are large-file (LFS or Xet) content.** Only those get the
|
|
160
|
+
credential-free CDN redirect; a small regular file is served inline and is
|
|
161
|
+
rejected with that reason. Hub **buckets** work as sources too, but cannot be
|
|
162
|
+
written to — they expose no LFS endpoint.
|
|
163
|
+
- **A destination needs the object's sha256 up front**, because the LFS batch
|
|
164
|
+
endpoint will not issue upload URLs without it. When the source is not itself a
|
|
165
|
+
Hub file, set `allow_source_rehash=True` to let the SDK read the source once to
|
|
166
|
+
compute it.
|
|
167
|
+
- **One source per Hugging Face destination**, because the Hub dictates the part
|
|
168
|
+
size and a plan carries a single chunk size.
|
|
169
|
+
- If you do not call `wait_complete`, call
|
|
170
|
+
`finalize_huggingface_uploads(transfer_id)` yourself once the transfer completes
|
|
171
|
+
— without it the parts are uploaded but no commit is made and the file does not
|
|
172
|
+
appear in the repo.
|
|
173
|
+
|
|
174
|
+
## Route Streaming And Payload Size
|
|
175
|
+
|
|
176
|
+
Provider transfers use `transfer-client-control/v6`. Every reply carries Runtime and transport epochs; prepare and every route-stream message carry a UUID route generation. S3 and R2 retain direct multipart UploadPart/ListParts/HEAD handling. Hippius and Hugging Face destinations take a plain PUT per chunk instead, so they carry no multipart group manifest.
|
|
177
|
+
|
|
178
|
+
The SDK signs up to 64 routes concurrently by default and emits 2,048-route logical batches, split only when encoded MessagePack requests exceed the 8 MiB default guard. Explicit positive overrides remain supported. A single route larger than the configured limit fails before publication with both sizes in the error. The stream ID derives from the transfer, selected flow, and immutable plan identity, and each ordered batch ID includes its route-coordinate checksum. Manual multi-destination attachment requires `delivery_index` on every route. Lifecycle mutations retry transient failures up to three times with the same request identity. Completion waits subscribe to the API-key-owned, at-most-once terminal signal before the first status read and reconcile every signal through authoritative status; subscription failure degrades to the same jittered 15-to-30-second status fallback. `async with BeamSDK(...)` closes signing workers, terminal waiters, and NATS resources; callers not using the context manager must call `await beam.close()`.
|
|
179
|
+
|
|
180
|
+
Hippius uses canonical non-multipart `signed_url` routes, so its manifest is empty and group-level final HEAD verification is not available from that provider flow. GCS and Azure provider signing remain unimplemented.
|
|
181
|
+
|
|
182
|
+
NATS requires this guard because the broker rejects messages above its configured `max_payload`. This limit applies only to lifecycle/control metadata; transfer file bytes do not flow through NATS.
|
|
183
|
+
|
|
184
|
+
## Restart Recovery
|
|
185
|
+
|
|
186
|
+
The SDK keeps one in-memory recovery lease per active transfer and one `runtime.hello` monitor per active shard. The lease is installed before route-stream begin, and a per-transfer lock coalesces initial streaming with replay. Multipart recovery retains the existing upload IDs and compact group state, then re-signs only the expiring route and commit controls. Runtime epoch changes invalidate cached auth, coalesce `transfer.resume`, and regenerate routes under a fresh generation; transport-only changes replay only when Runtime reports routes missing or expired. Provider inputs are released on terminal status or `close()`. Manual `attach_signed_urls` requires `route_generation_id`, plan fingerprint/checksum, and an async `recovery_factory`. No signed URL, credential, or recovery journal is written locally. Foreground cancellation, deadline expiry, and Runtime state-loss responses keep the background lease and retained multipart upload alive.
|
|
187
|
+
|
|
188
|
+
## CLI
|
|
189
|
+
|
|
190
|
+
`beam-send` creates, distributes, monitors, and waits for transfers:
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
beam-send create \
|
|
194
|
+
--source '{"type":"http","url":"https://downloads.example.com/report.parquet"}' \
|
|
195
|
+
--destination '{"type":"http","url":"https://storage.example.com/ingest/report.parquet"}' \
|
|
196
|
+
--total-size 104857600 \
|
|
197
|
+
--wait
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Convenience commands for object storage remain under `beam-send`, for example:
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
beam-send s3-transfer s3://source-bucket/file.bin s3://dest-bucket/file.bin \
|
|
204
|
+
--server http://23.92.16.104:8000 \
|
|
205
|
+
--api-key bk_your_key \
|
|
206
|
+
--aws-access-key YOUR_KEY \
|
|
207
|
+
--aws-secret-key YOUR_SECRET \
|
|
208
|
+
--aws-region us-east-1
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Only `beam-send` is packaged.
|
|
212
|
+
|
|
213
|
+
## Multi-Language Repository
|
|
214
|
+
|
|
215
|
+
The Python package lives alongside TypeScript and Go SDKs under `sdks/`. Shared API contracts live in `../../specs/`, and release instructions live in `../../docs/RELEASE.md`.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Python client for BEAM transfer creation and management."""
|
|
2
|
+
|
|
3
|
+
from beam_network_sdk._client import BEAM_DEV_URL, BEAM_PROD_URL, BeamSDK
|
|
4
|
+
from beam_network_sdk._nats_control import TransferTerminalSignalWaiter
|
|
5
|
+
from beam_network_sdk.exceptions import (
|
|
6
|
+
BeamAPIError,
|
|
7
|
+
BeamAuthError,
|
|
8
|
+
BeamError,
|
|
9
|
+
BeamRouteRecoveryPendingError,
|
|
10
|
+
BeamTimeoutError,
|
|
11
|
+
)
|
|
12
|
+
from beam_network_sdk.models import (
|
|
13
|
+
AttachSignedUrlsResponse,
|
|
14
|
+
AzureProviderDestination,
|
|
15
|
+
AzureProviderSource,
|
|
16
|
+
CallbackConfig,
|
|
17
|
+
ChunkDestinationSigningTarget,
|
|
18
|
+
ChunkSigningPlanItem,
|
|
19
|
+
CompactTransferPlanDescriptor,
|
|
20
|
+
CompactTransferPlanDestination,
|
|
21
|
+
CompactTransferPlanFormulas,
|
|
22
|
+
CompactTransferPlanSource,
|
|
23
|
+
DestConfig,
|
|
24
|
+
DestinationStatusInfo,
|
|
25
|
+
DistributeResponse,
|
|
26
|
+
GCSProviderDestination,
|
|
27
|
+
GCSProviderSource,
|
|
28
|
+
HippiusProviderDestination,
|
|
29
|
+
HippiusProviderSource,
|
|
30
|
+
MultipartGroupManifest,
|
|
31
|
+
PreparedDestination,
|
|
32
|
+
PreparedHttpSource,
|
|
33
|
+
ProviderDestinationConfig,
|
|
34
|
+
ProviderSourceConfig,
|
|
35
|
+
R2ProviderDestination,
|
|
36
|
+
R2ProviderSource,
|
|
37
|
+
S3ProviderDestination,
|
|
38
|
+
S3ProviderSource,
|
|
39
|
+
SignedChunkRoute,
|
|
40
|
+
SourceConfig,
|
|
41
|
+
SourceStatusInfo,
|
|
42
|
+
TransferCancelResponse,
|
|
43
|
+
TransferCreateRequest,
|
|
44
|
+
TransferCreateResponse,
|
|
45
|
+
TransferPlanResponse,
|
|
46
|
+
TransferPrepareResponse,
|
|
47
|
+
TransferStatusInfo,
|
|
48
|
+
TransferTerminalEvent,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
__version__ = "0.7.3"
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"BeamSDK",
|
|
55
|
+
"BEAM_DEV_URL",
|
|
56
|
+
"BEAM_PROD_URL",
|
|
57
|
+
"SourceConfig",
|
|
58
|
+
"DestConfig",
|
|
59
|
+
"S3ProviderSource",
|
|
60
|
+
"S3ProviderDestination",
|
|
61
|
+
"R2ProviderSource",
|
|
62
|
+
"R2ProviderDestination",
|
|
63
|
+
"GCSProviderSource",
|
|
64
|
+
"GCSProviderDestination",
|
|
65
|
+
"AzureProviderSource",
|
|
66
|
+
"AzureProviderDestination",
|
|
67
|
+
"HippiusProviderSource",
|
|
68
|
+
"HippiusProviderDestination",
|
|
69
|
+
"MultipartGroupManifest",
|
|
70
|
+
"ProviderSourceConfig",
|
|
71
|
+
"ProviderDestinationConfig",
|
|
72
|
+
"PreparedHttpSource",
|
|
73
|
+
"PreparedDestination",
|
|
74
|
+
"ChunkDestinationSigningTarget",
|
|
75
|
+
"ChunkSigningPlanItem",
|
|
76
|
+
"CompactTransferPlanDescriptor",
|
|
77
|
+
"CompactTransferPlanDestination",
|
|
78
|
+
"CompactTransferPlanFormulas",
|
|
79
|
+
"CompactTransferPlanSource",
|
|
80
|
+
"SignedChunkRoute",
|
|
81
|
+
"CallbackConfig",
|
|
82
|
+
"TransferCreateRequest",
|
|
83
|
+
"TransferCreateResponse",
|
|
84
|
+
"TransferPlanResponse",
|
|
85
|
+
"TransferPrepareResponse",
|
|
86
|
+
"TransferCancelResponse",
|
|
87
|
+
"AttachSignedUrlsResponse",
|
|
88
|
+
"DestinationStatusInfo",
|
|
89
|
+
"SourceStatusInfo",
|
|
90
|
+
"TransferStatusInfo",
|
|
91
|
+
"TransferTerminalEvent",
|
|
92
|
+
"TransferTerminalSignalWaiter",
|
|
93
|
+
"DistributeResponse",
|
|
94
|
+
"BeamError",
|
|
95
|
+
"BeamAuthError",
|
|
96
|
+
"BeamAPIError",
|
|
97
|
+
"BeamRouteRecoveryPendingError",
|
|
98
|
+
"BeamTimeoutError",
|
|
99
|
+
]
|