loopcut 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopcut/__init__.py +5 -0
- loopcut/__main__.py +8 -0
- loopcut/addon/VENDORED.md +21 -0
- loopcut/addon/__init__.py +8 -0
- loopcut/addon/blobs.py +444 -0
- loopcut/addon/bundles.py +569 -0
- loopcut/addon/cloud_http.py +256 -0
- loopcut/addon/config.py +5 -0
- loopcut/addon/credentials.py +6 -0
- loopcut/agent_tools.json +37 -0
- loopcut/catalog.json +955 -0
- loopcut/catalog.py +90 -0
- loopcut/cli.py +265 -0
- loopcut/mcp_server.py +340 -0
- loopcut/pack.py +154 -0
- loopcut/service.py +839 -0
- loopcut/state.py +81 -0
- loopcut-0.1.0.dist-info/METADATA +198 -0
- loopcut-0.1.0.dist-info/RECORD +23 -0
- loopcut-0.1.0.dist-info/WHEEL +5 -0
- loopcut-0.1.0.dist-info/entry_points.txt +2 -0
- loopcut-0.1.0.dist-info/licenses/LICENSE +674 -0
- loopcut-0.1.0.dist-info/top_level.txt +1 -0
loopcut/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""Loopcut's cloud tools for agents and scripts: render a saved .blend on cloud GPUs, search free
|
|
2
|
+
asset libraries, make 3D models, remesh, sound effects and voice lines, paid from the user's
|
|
3
|
+
Loopcut credits, with the results downloaded as files."""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
loopcut/__main__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
if sys.version_info < (3, 10): # The vendored add-on modules use 3.10 syntax.
|
|
4
|
+
raise SystemExit(f"loopcut needs Python 3.10 or newer (this is {sys.version.split()[0]}). "
|
|
5
|
+
"Run it with uv instead: uvx loopcut")
|
|
6
|
+
from .cli import main
|
|
7
|
+
|
|
8
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Vendored add-on modules
|
|
2
|
+
|
|
3
|
+
`bundles.py`, `blobs.py` and `cloud_http.py` are byte-for-byte copies of the Loopcut add-on's
|
|
4
|
+
modules (`scripts/addons_core/loopcut/` in the Loopcut Blender repository, GPL-3.0-or-later), so the
|
|
5
|
+
connector packs and uploads scenes exactly as the add-on does: the same dependency
|
|
6
|
+
collection, canonical manifest numbers and multipart upload retries that were fixed live on
|
|
7
|
+
2026-10-17.
|
|
8
|
+
|
|
9
|
+
Source: add-on development commit `fc92f9a4b7c9`.
|
|
10
|
+
|
|
11
|
+
Do not edit them here. Change the add-on, then copy the files again:
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
python3 scripts/sync_addon.py /path/to/loopcut/blender # copies from that checkout's working tree
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
`tests/test_vendored.py` fails when the copies differ from an add-on checkout named by
|
|
18
|
+
`LOOPCUT_ADDON_DIR`, and is skipped without it.
|
|
19
|
+
|
|
20
|
+
`__init__.py`, `config.py` and `credentials.py` are local stand-ins for the add-on package,
|
|
21
|
+
not copies.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Verbatim copies of the Loopcut add-on's bundle, blob and HTTP modules; see VENDORED.md.
|
|
2
|
+
|
|
3
|
+
The copies import bl_info, config and credentials from their package, so this package
|
|
4
|
+
supplies small stand-ins. The loopcut package always passes cloud_http.Client an api_key and
|
|
5
|
+
client_headers, so cloud_http never reads Blender or the add-on's stored key.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
bl_info = {"name": "Loopcut connector", "version": (0, 1, 0)}
|
loopcut/addon/blobs.py
ADDED
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
"""Content-addressed file uploads: ask first, stream only absent bytes to presigned URLs.
|
|
2
|
+
|
|
3
|
+
The service key belongs only on service requests. Storage PUTs carry contract/version headers
|
|
4
|
+
and the file's MIME and length, never Authorization. No file is read into memory in full.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import contextlib
|
|
8
|
+
import hashlib
|
|
9
|
+
import mimetypes
|
|
10
|
+
import re
|
|
11
|
+
import stat
|
|
12
|
+
import tempfile
|
|
13
|
+
import time
|
|
14
|
+
import urllib.request
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from . import cloud_http
|
|
19
|
+
|
|
20
|
+
CHUNK_SIZE = 1024 * 1024
|
|
21
|
+
BATCH_SIZE = 100
|
|
22
|
+
MAX_BLOB_SIZE = 200 * 1024 * 1024 # Contract upload-urls item limit.
|
|
23
|
+
PUT_ATTEMPTS = 5
|
|
24
|
+
RETRY_DELAYS = (1, 2, 4, 8) # Seconds before tries 2-5: large PUTs drop or corrupt mid-body now and then.
|
|
25
|
+
RETRY_STATUSES = {408, 429, 500, 502, 503, 504}
|
|
26
|
+
URL_MARGIN = 60 # Seconds: a presigned URL this close to its expires_at is replaced before a try.
|
|
27
|
+
MULTIPART = "/v1/blobs/multipart/" # start, urls, complete, abort (contract 0.6, SPEC 3.3).
|
|
28
|
+
SERVICE_ATTEMPTS = 3 # Multipart service calls; parts themselves get PUT_ATTEMPTS each.
|
|
29
|
+
_HASH = re.compile(r"[0-9a-f]{64}\Z")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def sha256(path, *, max_bytes=None):
|
|
33
|
+
digest = hashlib.sha256()
|
|
34
|
+
size = 0
|
|
35
|
+
with Path(path).open("rb") as handle:
|
|
36
|
+
for chunk in iter(lambda: handle.read(CHUNK_SIZE), b""):
|
|
37
|
+
size += len(chunk)
|
|
38
|
+
if max_bytes is not None and size > max_bytes:
|
|
39
|
+
raise ValueError("Blob exceeds the upload size limit")
|
|
40
|
+
digest.update(chunk)
|
|
41
|
+
return digest.hexdigest()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _hash(value):
|
|
45
|
+
if not isinstance(value, str) or not _HASH.fullmatch(value):
|
|
46
|
+
raise ValueError("Expected a lowercase SHA-256 hash")
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _batches(items, size):
|
|
51
|
+
if not isinstance(size, int) or isinstance(size, bool) or size < 1:
|
|
52
|
+
raise ValueError("Batch size must be positive")
|
|
53
|
+
for start in range(0, len(items), size):
|
|
54
|
+
yield items[start:start + size]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _invalid():
|
|
58
|
+
return cloud_http.CloudError("Invalid blob response from Loopcut", "invalid_response")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def check(hashes, *, client=None, batch_size=BATCH_SIZE):
|
|
62
|
+
client = client or cloud_http.Client()
|
|
63
|
+
hashes = list(dict.fromkeys(_hash(value) for value in hashes))
|
|
64
|
+
missing = []
|
|
65
|
+
for batch in _batches(hashes, batch_size):
|
|
66
|
+
reply = client.request("POST", "/v1/blobs/check", {"sha256": batch})
|
|
67
|
+
absent = reply.get("missing") if isinstance(reply, dict) else None
|
|
68
|
+
if (not isinstance(absent, list) or any(not isinstance(h, str) or h not in batch for h in absent)
|
|
69
|
+
or len(set(absent)) != len(absent)):
|
|
70
|
+
raise _invalid()
|
|
71
|
+
missing.extend(absent)
|
|
72
|
+
return missing
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def upload_urls(items, *, client=None, batch_size=BATCH_SIZE):
|
|
76
|
+
client = client or cloud_http.Client()
|
|
77
|
+
validated = []
|
|
78
|
+
for item in items:
|
|
79
|
+
size, mime = item["size"], item["mime"]
|
|
80
|
+
if (not isinstance(size, int) or isinstance(size, bool) or not 0 <= size <= MAX_BLOB_SIZE
|
|
81
|
+
or not isinstance(mime, str) or not mime or any(ord(c) < 32 for c in mime)):
|
|
82
|
+
raise ValueError("Invalid blob size or MIME type")
|
|
83
|
+
validated.append({"sha256": _hash(item["sha256"]), "size": size, "mime": mime})
|
|
84
|
+
hashes = [item["sha256"] for item in validated]
|
|
85
|
+
if len(hashes) != len(set(hashes)):
|
|
86
|
+
raise ValueError("Duplicate blob hash")
|
|
87
|
+
urls = []
|
|
88
|
+
for batch in _batches(validated, batch_size):
|
|
89
|
+
reply = client.request("POST", "/v1/blobs/upload-urls", {"items": batch})
|
|
90
|
+
rows = reply.get("items") if isinstance(reply, dict) else None
|
|
91
|
+
wanted = {item["sha256"] for item in batch}
|
|
92
|
+
if not isinstance(rows, list):
|
|
93
|
+
raise _invalid()
|
|
94
|
+
found = set()
|
|
95
|
+
for row in rows:
|
|
96
|
+
if (not isinstance(row, dict) or not isinstance(row.get("sha256"), str)
|
|
97
|
+
or row["sha256"] not in wanted or row["sha256"] in found
|
|
98
|
+
or not isinstance(row.get("expires_at"), str)):
|
|
99
|
+
raise _invalid()
|
|
100
|
+
try:
|
|
101
|
+
cloud_http.validate_url(row.get("put_url"))
|
|
102
|
+
except ValueError:
|
|
103
|
+
raise _invalid() from None
|
|
104
|
+
found.add(row["sha256"])
|
|
105
|
+
# Ready hashes are omitted (SPEC 3.3): storage already has them, so they need no PUT.
|
|
106
|
+
urls.extend(rows)
|
|
107
|
+
return urls
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _upload_size(path):
|
|
111
|
+
info = path.stat()
|
|
112
|
+
if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_BLOB_SIZE:
|
|
113
|
+
raise ValueError("Blob must be a regular file within the upload size limit")
|
|
114
|
+
return info.st_size
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _deadline(expires_at):
|
|
118
|
+
try:
|
|
119
|
+
return datetime.fromisoformat(expires_at.replace("Z", "+00:00")).timestamp()
|
|
120
|
+
except (AttributeError, TypeError, ValueError):
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _check_cancel(cancel):
|
|
125
|
+
if cancel is not None and cancel.is_set():
|
|
126
|
+
raise cloud_http.CloudError("Cloud call canceled before submission", "canceled")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class _Body:
|
|
130
|
+
"""`length` bytes of the frozen copy from its current position, as a PUT body that stops
|
|
131
|
+
mid-stream once the call is canceled. A whole-file PUT sends all of it; a part, its window."""
|
|
132
|
+
|
|
133
|
+
def __init__(self, handle, cancel, length):
|
|
134
|
+
self.handle, self.cancel, self.left = handle, cancel, length
|
|
135
|
+
|
|
136
|
+
def read(self, size=-1):
|
|
137
|
+
_check_cancel(self.cancel)
|
|
138
|
+
if self.left <= 0:
|
|
139
|
+
return b""
|
|
140
|
+
data = self.handle.read(self.left if size is None or size < 0 else min(size, self.left))
|
|
141
|
+
self.left -= len(data)
|
|
142
|
+
return data
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _mime(mime):
|
|
146
|
+
if not isinstance(mime, str) or not mime or any(ord(c) < 32 for c in mime):
|
|
147
|
+
raise ValueError("Invalid MIME type")
|
|
148
|
+
return mime
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
@contextlib.contextmanager
|
|
152
|
+
def _frozen(path, expected_hash, expected_size):
|
|
153
|
+
"""A private copy of the file's bytes, checked against the hash and size the server was told.
|
|
154
|
+
|
|
155
|
+
Freeze bytes before opening HTTP: the source can be edited while a PUT is streaming.
|
|
156
|
+
"""
|
|
157
|
+
with tempfile.TemporaryFile() as handle:
|
|
158
|
+
digest, size = hashlib.sha256(), 0
|
|
159
|
+
with Path(path).open("rb") as source:
|
|
160
|
+
for chunk in iter(lambda: source.read(CHUNK_SIZE), b""):
|
|
161
|
+
size += len(chunk)
|
|
162
|
+
if size > MAX_BLOB_SIZE:
|
|
163
|
+
raise ValueError("Blob exceeds the upload size limit")
|
|
164
|
+
digest.update(chunk)
|
|
165
|
+
handle.write(chunk)
|
|
166
|
+
if (expected_hash is not None and digest.hexdigest() != expected_hash
|
|
167
|
+
or expected_size is not None and size != expected_size):
|
|
168
|
+
raise ValueError("File changed before upload")
|
|
169
|
+
yield handle, size
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _wait(cancel, seconds):
|
|
173
|
+
if cancel is not None:
|
|
174
|
+
cancel.wait(seconds)
|
|
175
|
+
else:
|
|
176
|
+
time.sleep(seconds)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _send(client, url, handle, offset, length, headers, *, amount, part="", noun="Upload", expires_at=None,
|
|
180
|
+
refresh=None, cancel=None, progress=None):
|
|
181
|
+
"""PUT `length` bytes of the frozen copy from `offset`: the one retry loop for whole files and parts.
|
|
182
|
+
|
|
183
|
+
Retries dropped connections, 408, 429 and 5xx up to PUT_ATTEMPTS; other refusals are final. A repeat
|
|
184
|
+
is safe: the URL names the content hash (and a part's number), so it can only store the same bytes.
|
|
185
|
+
Before a try on a URL near expires_at, refresh() returns a fresh (put_url, expires_at), or None when
|
|
186
|
+
storage already has the blob. Returns the response headers, or None after such a refresh.
|
|
187
|
+
`amount` ("82.0 MB") and `part` (" (part 4/16)") name the upload in messages; `noun` in progress.
|
|
188
|
+
"""
|
|
189
|
+
deadline = _deadline(expires_at)
|
|
190
|
+
for attempt in range(1, PUT_ATTEMPTS + 1):
|
|
191
|
+
_check_cancel(cancel)
|
|
192
|
+
if refresh is not None and deadline is not None and time.time() > deadline - URL_MARGIN:
|
|
193
|
+
fresh = refresh()
|
|
194
|
+
if fresh is None:
|
|
195
|
+
return None
|
|
196
|
+
url, deadline = fresh[0], _deadline(fresh[1])
|
|
197
|
+
cloud_http.validate_url(url)
|
|
198
|
+
handle.seek(offset)
|
|
199
|
+
request = urllib.request.Request(url, data=_Body(handle, cancel, length), headers=headers, method="PUT")
|
|
200
|
+
try:
|
|
201
|
+
_, status, response_headers = cloud_http.send(request, client.transport, client.timeout)
|
|
202
|
+
except cloud_http.CloudError as ex:
|
|
203
|
+
if ex.code != "connection_failed" and ex.status not in RETRY_STATUSES:
|
|
204
|
+
if ex.code == "http_error" and ex.status is not None:
|
|
205
|
+
raise cloud_http.CloudError(f"Cloud storage refused the {amount} upload{part} (HTTP {ex.status})",
|
|
206
|
+
"http_error", ex.status) from None
|
|
207
|
+
raise
|
|
208
|
+
error, cause = ex, ex.detail or f"HTTP {ex.status}"
|
|
209
|
+
else:
|
|
210
|
+
if not 200 <= status < 300:
|
|
211
|
+
raise cloud_http.CloudError("Storage did not accept the blob", "http_error", status)
|
|
212
|
+
return response_headers
|
|
213
|
+
if attempt < PUT_ATTEMPTS:
|
|
214
|
+
print(f"Loopcut: {amount} upload{part} interrupted ({cause}); retrying ({attempt + 1}/{PUT_ATTEMPTS})")
|
|
215
|
+
if progress is not None:
|
|
216
|
+
progress(f"{noun} interrupted, retrying ({attempt + 1}/{PUT_ATTEMPTS})…")
|
|
217
|
+
_wait(cancel, RETRY_DELAYS[min(attempt, len(RETRY_DELAYS)) - 1])
|
|
218
|
+
if error.code == "connection_failed":
|
|
219
|
+
message = (f"Upload to cloud storage was interrupted: {amount}{part}, {PUT_ATTEMPTS} tries, last {cause}. "
|
|
220
|
+
"Check the connection and try again.")
|
|
221
|
+
else:
|
|
222
|
+
message = (f"Cloud storage did not accept the {amount} upload{part}: {cause} after {PUT_ATTEMPTS} tries. "
|
|
223
|
+
"Try again later.")
|
|
224
|
+
print(f"Loopcut: {message}")
|
|
225
|
+
raise cloud_http.CloudError(message, error.code, error.status, detail=error.detail)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _amount(size):
|
|
229
|
+
return f"{size / 1_000_000:,.1f} MB"
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def put(url, path, mime, *, client=None, expected_hash=None, expected_size=None, expires_at=None,
|
|
233
|
+
refresh=None, cancel=None, progress=None):
|
|
234
|
+
"""PUT a frozen copy, retrying dropped connections, 408, 429 and 5xx; other refusals are final.
|
|
235
|
+
|
|
236
|
+
A repeat is safe: the URL names the content hash, so it can only store the same bytes (a 25 MB
|
|
237
|
+
scene bundle's PUT broke at 5-17 MB with TLS "bad record mac", live test 2026-10-17).
|
|
238
|
+
Before a try on a URL near expires_at, refresh() returns a fresh (put_url, expires_at), or
|
|
239
|
+
None when storage already has the blob. cancel.is_set() stops between tries and mid-body.
|
|
240
|
+
"""
|
|
241
|
+
client = client or cloud_http.Client()
|
|
242
|
+
cloud_http.validate_url(url)
|
|
243
|
+
_mime(mime)
|
|
244
|
+
path = Path(path)
|
|
245
|
+
_upload_size(path)
|
|
246
|
+
with _frozen(path, expected_hash, expected_size) as (handle, size):
|
|
247
|
+
headers = {**client.version_headers(), "Content-Type": mime, "Content-Length": str(size)}
|
|
248
|
+
_send(client, url, handle, 0, size, headers, amount=_amount(size), expires_at=expires_at,
|
|
249
|
+
refresh=refresh, cancel=cancel, progress=progress)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
# ------------------------------------------------------------------ multipart (contract 0.6, SPEC 3.3)
|
|
253
|
+
|
|
254
|
+
class _MultipartOff(Exception):
|
|
255
|
+
"""The server has multipart switched off (404 not_found): send this file with one PUT."""
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def multipart_capability(value):
|
|
259
|
+
"""The manifest's capabilities.blob_multipart if it is well formed, else None (single PUTs only)."""
|
|
260
|
+
size = value.get("min_size") if isinstance(value, dict) else None
|
|
261
|
+
return {"min_size": size} if isinstance(size, int) and not isinstance(size, bool) and size >= 1 else None
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _service(client, action, payload, cancel=None):
|
|
265
|
+
"""POST a multipart call. Each is safe to repeat (start replaces, complete is idempotent), so a
|
|
266
|
+
dropped connection, 429 (after its retry_after) or 502-504 is tried again, up to SERVICE_ATTEMPTS."""
|
|
267
|
+
for attempt in range(1, SERVICE_ATTEMPTS + 1):
|
|
268
|
+
_check_cancel(cancel)
|
|
269
|
+
try:
|
|
270
|
+
return client.request("POST", MULTIPART + action, payload)
|
|
271
|
+
except cloud_http.CloudError as ex:
|
|
272
|
+
if ex.code != "connection_failed" and ex.status not in {429, 502, 503, 504} or attempt == SERVICE_ATTEMPTS:
|
|
273
|
+
raise
|
|
274
|
+
after = ex.extra.get("retry_after")
|
|
275
|
+
delay = after if isinstance(after, int) and not isinstance(after, bool) and 0 <= after <= 60 else \
|
|
276
|
+
RETRY_DELAYS[min(attempt, len(RETRY_DELAYS)) - 1]
|
|
277
|
+
print(f"Loopcut: multipart {action} failed ({ex.detail or ex.code}); "
|
|
278
|
+
f"retrying ({attempt + 1}/{SERVICE_ATTEMPTS})")
|
|
279
|
+
_wait(cancel, delay)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _parts(reply):
|
|
283
|
+
"""Validate a start or urls reply's parts: {part_number: put_url}."""
|
|
284
|
+
rows = reply.get("parts")
|
|
285
|
+
if not isinstance(rows, list) or not rows or not isinstance(reply.get("expires_at"), str):
|
|
286
|
+
raise _invalid()
|
|
287
|
+
urls = {}
|
|
288
|
+
for row in rows:
|
|
289
|
+
number = row.get("part_number") if isinstance(row, dict) else None
|
|
290
|
+
if not isinstance(number, int) or isinstance(number, bool) or number < 1 or number in urls:
|
|
291
|
+
raise _invalid()
|
|
292
|
+
try:
|
|
293
|
+
cloud_http.validate_url(row.get("put_url"))
|
|
294
|
+
except ValueError:
|
|
295
|
+
raise _invalid() from None
|
|
296
|
+
urls[number] = row["put_url"]
|
|
297
|
+
return urls
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
class _Upload:
|
|
301
|
+
"""One file's multipart upload: its id, part layout and current part URLs."""
|
|
302
|
+
|
|
303
|
+
def __init__(self, client, reply, digest, size, cancel):
|
|
304
|
+
self.client, self.digest, self.size, self.cancel = client, digest, size, cancel
|
|
305
|
+
upload_id, part_size = reply.get("upload_id"), reply.get("part_size")
|
|
306
|
+
if (not isinstance(upload_id, str) or not 0 < len(upload_id) <= 1024
|
|
307
|
+
or any(not 33 <= ord(c) <= 126 for c in upload_id)
|
|
308
|
+
or not isinstance(part_size, int) or isinstance(part_size, bool) or part_size < 1):
|
|
309
|
+
raise _invalid()
|
|
310
|
+
self.upload_id, self.part_size = upload_id, part_size
|
|
311
|
+
self.count = max(1, -(-size // part_size))
|
|
312
|
+
self.urls, self.expires_at = _parts(reply), reply["expires_at"]
|
|
313
|
+
if sorted(self.urls) != list(range(1, self.count + 1)):
|
|
314
|
+
raise _invalid()
|
|
315
|
+
|
|
316
|
+
def refresh(self, number):
|
|
317
|
+
"""Fresh URLs for this part and every later one, in one call; returns (put_url, expires_at)."""
|
|
318
|
+
reply = _service(self.client, "urls", {"sha256": self.digest, "upload_id": self.upload_id,
|
|
319
|
+
"part_numbers": list(range(number, self.count + 1))}, self.cancel)
|
|
320
|
+
if (not isinstance(reply, dict) or reply.get("sha256") != self.digest
|
|
321
|
+
or reply.get("upload_id") != self.upload_id):
|
|
322
|
+
raise _invalid()
|
|
323
|
+
urls = _parts(reply)
|
|
324
|
+
if number not in urls or any(n < number or n > self.count for n in urls):
|
|
325
|
+
raise _invalid()
|
|
326
|
+
self.urls.update(urls)
|
|
327
|
+
self.expires_at = reply["expires_at"]
|
|
328
|
+
return self.urls[number], self.expires_at
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _abort(client, digest, upload_id):
|
|
332
|
+
"""One try, best effort (often after a cancel): storage also discards unfinished uploads after a day."""
|
|
333
|
+
try:
|
|
334
|
+
client.request("POST", MULTIPART + "abort", {"sha256": digest, "upload_id": upload_id})
|
|
335
|
+
except (cloud_http.CloudError, ValueError, OSError):
|
|
336
|
+
pass
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def put_multipart(path, digest, size, mime, *, client=None, cancel=None, progress=None):
|
|
340
|
+
"""Upload one file in the server's parts, each PUT through _send's retries; True if bytes went up.
|
|
341
|
+
|
|
342
|
+
False when storage already has the hash. Raises _MultipartOff when the server answers the start
|
|
343
|
+
with 404 (multipart switched off), so the caller sends one PUT instead. Any failure or cancel
|
|
344
|
+
after the start aborts the upload.
|
|
345
|
+
"""
|
|
346
|
+
client = client or cloud_http.Client()
|
|
347
|
+
_mime(mime)
|
|
348
|
+
path = Path(path)
|
|
349
|
+
with _frozen(path, digest, size) as (handle, size):
|
|
350
|
+
try:
|
|
351
|
+
reply = _service(client, "start", {"sha256": digest, "size": size, "mime": mime}, cancel)
|
|
352
|
+
except cloud_http.CloudError as ex:
|
|
353
|
+
if ex.status == 404 and ex.code == "not_found":
|
|
354
|
+
raise _MultipartOff() from None
|
|
355
|
+
raise
|
|
356
|
+
if not isinstance(reply, dict) or reply.get("sha256") != digest or type(reply.get("stored")) is not bool:
|
|
357
|
+
raise _invalid()
|
|
358
|
+
if reply["stored"]:
|
|
359
|
+
return False
|
|
360
|
+
try:
|
|
361
|
+
upload = _Upload(client, reply, digest, size, cancel)
|
|
362
|
+
except cloud_http.CloudError:
|
|
363
|
+
# A reply we can't use may still have opened an upload: end it.
|
|
364
|
+
if isinstance(reply.get("upload_id"), str) and 0 < len(reply["upload_id"]) <= 1024:
|
|
365
|
+
_abort(client, digest, reply["upload_id"])
|
|
366
|
+
raise
|
|
367
|
+
amount, done = _amount(size), False
|
|
368
|
+
try:
|
|
369
|
+
etags = []
|
|
370
|
+
for number in range(1, upload.count + 1):
|
|
371
|
+
offset = (number - 1) * upload.part_size
|
|
372
|
+
length = min(upload.part_size, size - offset)
|
|
373
|
+
label = f"part {number}/{upload.count}"
|
|
374
|
+
if progress is not None:
|
|
375
|
+
progress(f"Uploading {path.name}: {label}…")
|
|
376
|
+
headers = {**client.version_headers(), "Content-Type": mime, "Content-Length": str(length)}
|
|
377
|
+
answer = _send(client, upload.urls[number], handle, offset, length, headers, amount=amount,
|
|
378
|
+
part=f" ({label})", noun=label.capitalize(), expires_at=upload.expires_at,
|
|
379
|
+
refresh=lambda number=number: upload.refresh(number), cancel=cancel, progress=progress)
|
|
380
|
+
etag = next((v for k, v in (answer or {}).items() if k.lower() == "etag"), None)
|
|
381
|
+
if not isinstance(etag, str) or not 0 < len(etag) <= 256 or any(not 33 <= ord(c) <= 126 for c in etag):
|
|
382
|
+
raise cloud_http.CloudError(f"Cloud storage did not confirm {label} of the {amount} upload",
|
|
383
|
+
"invalid_response")
|
|
384
|
+
etags.append({"part_number": number, "etag": etag})
|
|
385
|
+
reply = _service(client, "complete", {"sha256": digest, "upload_id": upload.upload_id, "parts": etags},
|
|
386
|
+
cancel)
|
|
387
|
+
if not isinstance(reply, dict) or reply.get("sha256") != digest:
|
|
388
|
+
raise _invalid()
|
|
389
|
+
done = True
|
|
390
|
+
finally:
|
|
391
|
+
if not done:
|
|
392
|
+
_abort(client, digest, upload.upload_id)
|
|
393
|
+
return True
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _fresh_url(item, client):
|
|
397
|
+
"""A new (put_url, expires_at) for one blob, or None once storage has it."""
|
|
398
|
+
rows = upload_urls([item], client=client)
|
|
399
|
+
return (rows[0]["put_url"], rows[0]["expires_at"]) if rows else None
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def upload_missing(files, *, client=None, batch_size=BATCH_SIZE, cancel=None, progress=None, multipart=None):
|
|
403
|
+
"""Files are descriptors {path, sha256?, size?, mime?}; return the hashes PUT this time.
|
|
404
|
+
|
|
405
|
+
Supplied digests and lengths are verified locally before HTTP so stale manifests cannot
|
|
406
|
+
publish different bytes under an old hash. Duplicate content is uploaded once.
|
|
407
|
+
cancel and progress reach put(): a canceled call stops, a retry says so.
|
|
408
|
+
multipart is the manifest's capabilities.blob_multipart: files of its min_size and up go in
|
|
409
|
+
parts (put_multipart), falling back to one PUT if the server has switched it off.
|
|
410
|
+
"""
|
|
411
|
+
client = client or cloud_http.Client()
|
|
412
|
+
unique = {}
|
|
413
|
+
for item in files:
|
|
414
|
+
path = Path(item["path"])
|
|
415
|
+
size = _upload_size(path)
|
|
416
|
+
digest = sha256(path, max_bytes=MAX_BLOB_SIZE)
|
|
417
|
+
if "sha256" in item and _hash(item["sha256"]) != digest:
|
|
418
|
+
raise ValueError(f"File changed since hashing: {path.name}")
|
|
419
|
+
if "size" in item and item["size"] != size:
|
|
420
|
+
raise ValueError(f"File changed since sizing: {path.name}")
|
|
421
|
+
mime = item.get("mime") or mimetypes.guess_type(path.name)[0] or "application/octet-stream"
|
|
422
|
+
unique.setdefault(digest, {"sha256": digest, "size": size, "mime": mime, "path": path})
|
|
423
|
+
missing = check(unique, client=client, batch_size=batch_size)
|
|
424
|
+
capability = multipart_capability(multipart)
|
|
425
|
+
parts = {digest for digest in missing if capability and unique[digest]["size"] >= capability["min_size"]}
|
|
426
|
+
large = [unique[digest] for digest in missing if digest in parts]
|
|
427
|
+
items = [unique[digest] for digest in missing if digest not in parts]
|
|
428
|
+
uploaded = []
|
|
429
|
+
for item in large:
|
|
430
|
+
try:
|
|
431
|
+
if put_multipart(item["path"], item["sha256"], item["size"], item["mime"], client=client,
|
|
432
|
+
cancel=cancel, progress=progress):
|
|
433
|
+
uploaded.append(item["sha256"])
|
|
434
|
+
except _MultipartOff:
|
|
435
|
+
items.append(item) # Switched off since the manifest was read: one PUT, as before.
|
|
436
|
+
for batch in _batches(items, batch_size):
|
|
437
|
+
for row in upload_urls(batch, client=client, batch_size=batch_size):
|
|
438
|
+
item = unique[row["sha256"]]
|
|
439
|
+
# A batch's URLs share one expiry; later files in a long batch may need fresh ones.
|
|
440
|
+
put(row["put_url"], item["path"], item["mime"], client=client, expected_hash=item["sha256"],
|
|
441
|
+
expected_size=item["size"], expires_at=row["expires_at"], cancel=cancel, progress=progress,
|
|
442
|
+
refresh=lambda item=item: _fresh_url(item, client))
|
|
443
|
+
uploaded.append(row["sha256"])
|
|
444
|
+
return uploaded
|