nerdstack-ark 1.0.3__tar.gz → 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/PKG-INFO +20 -7
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/README.md +19 -6
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/pyproject.toml +1 -1
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/_shared.py +44 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/async_client.py +139 -1
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/models.py +9 -1
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/sync.py +148 -1
- nerdstack_ark-1.0.4/tests/test_stream_upload.py +168 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/.gitignore +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/LICENSE +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/examples/django/apps.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/examples/django/views.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/examples/fastapi/app.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/examples/flask/app.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/__init__.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/errors.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/py.typed +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/src/ark_py/s3.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/tests/conftest.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/tests/test_async.py +0 -0
- {nerdstack_ark-1.0.3 → nerdstack_ark-1.0.4}/tests/test_sync.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: nerdstack-ark
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.4
|
|
4
4
|
Summary: Official Python SDK for Ark storage, with sync, async, and S3-compatible access.
|
|
5
5
|
Project-URL: Homepage, https://ark.nerdstackgrp.com
|
|
6
6
|
Project-URL: Documentation, https://github.com/joshhumphrey02/ark-sdk/tree/master/packages/ark-py#readme
|
|
@@ -132,9 +132,21 @@ session = ark.create_client_session(ttl_seconds=900)
|
|
|
132
132
|
|
|
133
133
|
## Ark Streams
|
|
134
134
|
|
|
135
|
-
Both synchronous and asynchronous clients expose
|
|
136
|
-
|
|
137
|
-
|
|
135
|
+
Both synchronous and asynchronous clients expose Ark Streams. `upload` is the
|
|
136
|
+
one call most applications need: it creates the video and sends the bytes,
|
|
137
|
+
resuming from the server's acknowledged offset if the connection drops.
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
stream = ark.streams.upload(
|
|
141
|
+
"./launch.mp4",
|
|
142
|
+
title="Product launch",
|
|
143
|
+
app_id=app_id,
|
|
144
|
+
on_progress=lambda sent, total: print(f"{sent}/{total}"),
|
|
145
|
+
)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Use `create` directly when the bytes belong to someone else -- a browser, or a
|
|
149
|
+
worker -- and you only need the upload ticket:
|
|
138
150
|
|
|
139
151
|
```python
|
|
140
152
|
creation = ark.streams.create("Product launch", size_bytes, app_id=app_id)
|
|
@@ -147,9 +159,10 @@ ark.streams.refresh_upload_url(stream.id, app_id=app_id)
|
|
|
147
159
|
ark.streams.delete(stream.id, app_id=app_id)
|
|
148
160
|
```
|
|
149
161
|
|
|
150
|
-
The same methods on `AsyncArk.streams` are awaitable.
|
|
151
|
-
|
|
152
|
-
|
|
162
|
+
The same methods on `AsyncArk.streams` are awaitable. Encoding continues after
|
|
163
|
+
an upload returns, so poll `get` until `status` is `ready`; `hls_url`,
|
|
164
|
+
`thumbnail_url` and `embed_url` are `None` until then. `embed_url` is an
|
|
165
|
+
Ark-hosted player page that can go straight into an iframe.
|
|
153
166
|
|
|
154
167
|
## S3-compatible access
|
|
155
168
|
|
|
@@ -89,9 +89,21 @@ session = ark.create_client_session(ttl_seconds=900)
|
|
|
89
89
|
|
|
90
90
|
## Ark Streams
|
|
91
91
|
|
|
92
|
-
Both synchronous and asynchronous clients expose
|
|
93
|
-
|
|
94
|
-
|
|
92
|
+
Both synchronous and asynchronous clients expose Ark Streams. `upload` is the
|
|
93
|
+
one call most applications need: it creates the video and sends the bytes,
|
|
94
|
+
resuming from the server's acknowledged offset if the connection drops.
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
stream = ark.streams.upload(
|
|
98
|
+
"./launch.mp4",
|
|
99
|
+
title="Product launch",
|
|
100
|
+
app_id=app_id,
|
|
101
|
+
on_progress=lambda sent, total: print(f"{sent}/{total}"),
|
|
102
|
+
)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Use `create` directly when the bytes belong to someone else -- a browser, or a
|
|
106
|
+
worker -- and you only need the upload ticket:
|
|
95
107
|
|
|
96
108
|
```python
|
|
97
109
|
creation = ark.streams.create("Product launch", size_bytes, app_id=app_id)
|
|
@@ -104,9 +116,10 @@ ark.streams.refresh_upload_url(stream.id, app_id=app_id)
|
|
|
104
116
|
ark.streams.delete(stream.id, app_id=app_id)
|
|
105
117
|
```
|
|
106
118
|
|
|
107
|
-
The same methods on `AsyncArk.streams` are awaitable.
|
|
108
|
-
|
|
109
|
-
|
|
119
|
+
The same methods on `AsyncArk.streams` are awaitable. Encoding continues after
|
|
120
|
+
an upload returns, so poll `get` until `status` is `ready`; `hls_url`,
|
|
121
|
+
`thumbnail_url` and `embed_url` are `None` until then. `embed_url` is an
|
|
122
|
+
Ark-hosted player page that can go straight into an iframe.
|
|
110
123
|
|
|
111
124
|
## S3-compatible access
|
|
112
125
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import base64
|
|
3
4
|
import mimetypes
|
|
4
5
|
import os
|
|
5
6
|
from collections.abc import Iterable, Iterator, Mapping
|
|
@@ -180,3 +181,46 @@ def sorted_parts(parts: Iterable[dict[str, object]]) -> list[dict[str, object]]:
|
|
|
180
181
|
return value
|
|
181
182
|
|
|
182
183
|
return sorted(parts, key=part_number)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# --- Ark Streams resumable upload (TUS) -------------------------------------
|
|
187
|
+
#
|
|
188
|
+
# Video goes to the encoding network over TUS rather than through the presigned
|
|
189
|
+
# path files use: an encode is long enough that a dropped connection is normal
|
|
190
|
+
# rather than exceptional, so the protocol has to be able to say "you already
|
|
191
|
+
# have the first N bytes, continue from there".
|
|
192
|
+
|
|
193
|
+
#: TUS sends metadata as base64, so a filename with non-ASCII characters
|
|
194
|
+
#: survives the header intact.
|
|
195
|
+
def tus_metadata(value: str) -> str:
|
|
196
|
+
return base64.b64encode(value.encode("utf-8")).decode("ascii")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
#: Default slice sent per PATCH. Large enough that a long video is not thousands
|
|
200
|
+
#: of round trips, small enough that a failure re-sends little.
|
|
201
|
+
DEFAULT_VIDEO_CHUNK_SIZE = 64 * 1024 * 1024
|
|
202
|
+
|
|
203
|
+
#: Consecutive failures tolerated per chunk before giving up. Each retry first
|
|
204
|
+
#: asks the server what it actually holds, so a retry never duplicates bytes.
|
|
205
|
+
MAX_VIDEO_CHUNK_RETRIES = 2
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def validate_chunk_size(chunk_size: int) -> None:
|
|
209
|
+
if isinstance(chunk_size, bool) or not isinstance(chunk_size, int) or chunk_size <= 0:
|
|
210
|
+
raise invalid_argument("chunk_size must be a positive integer")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def resumed_offset(header: str | None, total: int, fallback: Exception) -> int:
|
|
214
|
+
"""Read an Upload-Offset the server reported, or re-raise.
|
|
215
|
+
|
|
216
|
+
A server that answers with an offset past the end of the file, or with
|
|
217
|
+
something unparseable, cannot be resumed from safely -- continuing would
|
|
218
|
+
either skip bytes or corrupt the video.
|
|
219
|
+
"""
|
|
220
|
+
try:
|
|
221
|
+
offset = int(header or "")
|
|
222
|
+
except ValueError:
|
|
223
|
+
raise fallback from None
|
|
224
|
+
if offset < 0 or offset > total:
|
|
225
|
+
raise fallback
|
|
226
|
+
return offset
|
|
@@ -4,8 +4,9 @@ import asyncio
|
|
|
4
4
|
import builtins
|
|
5
5
|
import mimetypes
|
|
6
6
|
import os
|
|
7
|
-
from collections.abc import AsyncIterable, AsyncIterator, Mapping
|
|
7
|
+
from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping
|
|
8
8
|
from contextlib import suppress
|
|
9
|
+
from pathlib import Path
|
|
9
10
|
from types import TracebackType
|
|
10
11
|
from typing import Any, BinaryIO, TypeVar, cast
|
|
11
12
|
|
|
@@ -14,6 +15,8 @@ import httpx
|
|
|
14
15
|
from ._shared import (
|
|
15
16
|
DEFAULT_BASE_URL,
|
|
16
17
|
DEFAULT_CONTENT_TYPE,
|
|
18
|
+
DEFAULT_VIDEO_CHUNK_SIZE,
|
|
19
|
+
MAX_VIDEO_CHUNK_RETRIES,
|
|
17
20
|
UploadSource,
|
|
18
21
|
api_url,
|
|
19
22
|
image_url,
|
|
@@ -21,9 +24,12 @@ from ._shared import (
|
|
|
21
24
|
query_string,
|
|
22
25
|
read_exact,
|
|
23
26
|
resolve_upload_source,
|
|
27
|
+
resumed_offset,
|
|
24
28
|
segment,
|
|
25
29
|
sorted_parts,
|
|
30
|
+
tus_metadata,
|
|
26
31
|
upload_payload,
|
|
32
|
+
validate_chunk_size,
|
|
27
33
|
validate_size,
|
|
28
34
|
)
|
|
29
35
|
from .errors import ArkError, error_from_response, invalid_argument, network_error, upload_error
|
|
@@ -525,6 +531,138 @@ class AsyncStreams:
|
|
|
525
531
|
suffix = query_string({"appId": app_id})
|
|
526
532
|
await self._ark._request("DELETE", f"/streams/{segment(stream_id)}{suffix}")
|
|
527
533
|
|
|
534
|
+
async def upload(
|
|
535
|
+
self,
|
|
536
|
+
source: str | Path | BinaryIO,
|
|
537
|
+
*,
|
|
538
|
+
title: str | None = None,
|
|
539
|
+
size: int | None = None,
|
|
540
|
+
filename: str | None = None,
|
|
541
|
+
content_type: str | None = None,
|
|
542
|
+
app_id: str | None = None,
|
|
543
|
+
collection_id: str | None = None,
|
|
544
|
+
chunk_size: int = DEFAULT_VIDEO_CHUNK_SIZE,
|
|
545
|
+
on_progress: Callable[[int, int], None] | None = None,
|
|
546
|
+
) -> ArkStream:
|
|
547
|
+
"""Create a stream and upload the video, resuming across failures.
|
|
548
|
+
|
|
549
|
+
The async twin of `Streams.upload`. See that method for why this exists
|
|
550
|
+
rather than leaving TUS to the caller.
|
|
551
|
+
"""
|
|
552
|
+
validate_chunk_size(chunk_size)
|
|
553
|
+
resolved = resolve_upload_source(
|
|
554
|
+
source,
|
|
555
|
+
size=size,
|
|
556
|
+
filename=filename,
|
|
557
|
+
content_type=content_type or "video/mp4",
|
|
558
|
+
)
|
|
559
|
+
|
|
560
|
+
created = await self.create(
|
|
561
|
+
title or Path(resolved.filename).stem,
|
|
562
|
+
resolved.size,
|
|
563
|
+
app_id=app_id,
|
|
564
|
+
collection_id=collection_id,
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
upload_url = await self._create_tus_upload(created.upload.endpoint, resolved)
|
|
568
|
+
handle = resolved.path.open("rb") if resolved.path is not None else resolved.stream
|
|
569
|
+
try:
|
|
570
|
+
await self._send_chunks(
|
|
571
|
+
upload_url, handle, resolved.size, chunk_size, on_progress
|
|
572
|
+
)
|
|
573
|
+
finally:
|
|
574
|
+
if resolved.path is not None:
|
|
575
|
+
handle.close()
|
|
576
|
+
return created.stream
|
|
577
|
+
|
|
578
|
+
async def _create_tus_upload(self, endpoint: str, resolved: UploadSource) -> str:
|
|
579
|
+
url = self._ark._url(endpoint) if endpoint.startswith("/") else endpoint
|
|
580
|
+
try:
|
|
581
|
+
response = await self._ark._client.request(
|
|
582
|
+
"POST",
|
|
583
|
+
url,
|
|
584
|
+
headers={
|
|
585
|
+
"authorization": f"Bearer {self._ark._token}",
|
|
586
|
+
"Tus-Resumable": "1.0.0",
|
|
587
|
+
"Upload-Length": str(resolved.size),
|
|
588
|
+
"Upload-Metadata": (
|
|
589
|
+
f"filename {tus_metadata(resolved.filename)},"
|
|
590
|
+
f"filetype {tus_metadata(resolved.content_type)}"
|
|
591
|
+
),
|
|
592
|
+
},
|
|
593
|
+
)
|
|
594
|
+
except httpx.HTTPError as error:
|
|
595
|
+
raise network_error(error) from error
|
|
596
|
+
if response.is_error:
|
|
597
|
+
raise error_from_response(response)
|
|
598
|
+
|
|
599
|
+
location = response.headers.get("location")
|
|
600
|
+
if not location:
|
|
601
|
+
raise ArkError("INTERNAL_ERROR", "Ark did not return an upload location")
|
|
602
|
+
return str(httpx.URL(url).join(location))
|
|
603
|
+
|
|
604
|
+
async def _send_chunks(
|
|
605
|
+
self,
|
|
606
|
+
upload_url: str,
|
|
607
|
+
handle: BinaryIO,
|
|
608
|
+
total: int,
|
|
609
|
+
chunk_size: int,
|
|
610
|
+
on_progress: Callable[[int, int], None] | None,
|
|
611
|
+
) -> None:
|
|
612
|
+
offset = 0
|
|
613
|
+
retries = 0
|
|
614
|
+
while offset < total:
|
|
615
|
+
handle.seek(offset)
|
|
616
|
+
chunk = handle.read(min(chunk_size, total - offset))
|
|
617
|
+
if not chunk:
|
|
618
|
+
raise ArkError(
|
|
619
|
+
"INVALID_ARGUMENT",
|
|
620
|
+
"The video source ended before the declared size was reached",
|
|
621
|
+
)
|
|
622
|
+
try:
|
|
623
|
+
response = await self._ark._client.request(
|
|
624
|
+
"PATCH",
|
|
625
|
+
upload_url,
|
|
626
|
+
headers={
|
|
627
|
+
"authorization": f"Bearer {self._ark._token}",
|
|
628
|
+
"Tus-Resumable": "1.0.0",
|
|
629
|
+
"Upload-Offset": str(offset),
|
|
630
|
+
"Content-Type": "application/offset+octet-stream",
|
|
631
|
+
},
|
|
632
|
+
content=chunk,
|
|
633
|
+
)
|
|
634
|
+
if response.is_error:
|
|
635
|
+
raise error_from_response(response)
|
|
636
|
+
acknowledged = response.headers.get("upload-offset")
|
|
637
|
+
try:
|
|
638
|
+
offset = max(offset + len(chunk), int(acknowledged or ""))
|
|
639
|
+
except ValueError:
|
|
640
|
+
offset += len(chunk)
|
|
641
|
+
retries = 0
|
|
642
|
+
if on_progress is not None:
|
|
643
|
+
on_progress(offset, total)
|
|
644
|
+
except ArkError as error:
|
|
645
|
+
if retries >= MAX_VIDEO_CHUNK_RETRIES:
|
|
646
|
+
raise
|
|
647
|
+
retries += 1
|
|
648
|
+
offset = await self._head_offset(upload_url, total, error)
|
|
649
|
+
|
|
650
|
+
async def _head_offset(self, upload_url: str, total: int, fallback: ArkError) -> int:
|
|
651
|
+
try:
|
|
652
|
+
head = await self._ark._client.request(
|
|
653
|
+
"HEAD",
|
|
654
|
+
upload_url,
|
|
655
|
+
headers={
|
|
656
|
+
"authorization": f"Bearer {self._ark._token}",
|
|
657
|
+
"Tus-Resumable": "1.0.0",
|
|
658
|
+
},
|
|
659
|
+
)
|
|
660
|
+
except httpx.HTTPError:
|
|
661
|
+
raise fallback from None
|
|
662
|
+
if head.is_error:
|
|
663
|
+
raise fallback
|
|
664
|
+
return resumed_offset(head.headers.get("upload-offset"), total, fallback)
|
|
665
|
+
|
|
528
666
|
|
|
529
667
|
def _as_async_iterable(source: AsyncSource) -> AsyncIterator[bytes] | None:
|
|
530
668
|
method = getattr(source, "__aiter__", None)
|
|
@@ -101,7 +101,15 @@ StreamStatus = Literal["created", "uploading", "processing", "ready", "failed"]
|
|
|
101
101
|
|
|
102
102
|
@dataclass(frozen=True, slots=True)
|
|
103
103
|
class ArkStream:
|
|
104
|
-
"""A video managed by Ark Streams.
|
|
104
|
+
"""A video managed by Ark Streams.
|
|
105
|
+
|
|
106
|
+
``hls_url``, ``thumbnail_url`` and ``embed_url`` are None until ``status``
|
|
107
|
+
is ``"ready"``: an encode takes time, and returning a URL that 404s would be
|
|
108
|
+
worse than reporting that it is unfinished. Poll ``streams.get`` until then.
|
|
109
|
+
|
|
110
|
+
``embed_url`` is an Ark-hosted player page -- put it straight in an iframe.
|
|
111
|
+
Playback is signed server-side per viewer, so it carries no credential.
|
|
112
|
+
"""
|
|
105
113
|
|
|
106
114
|
id: str
|
|
107
115
|
title: str
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import builtins
|
|
4
|
-
from collections.abc import Mapping
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
5
|
from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
|
|
6
6
|
from contextlib import suppress
|
|
7
7
|
from pathlib import Path
|
|
@@ -12,6 +12,8 @@ import httpx
|
|
|
12
12
|
|
|
13
13
|
from ._shared import (
|
|
14
14
|
DEFAULT_BASE_URL,
|
|
15
|
+
DEFAULT_VIDEO_CHUNK_SIZE,
|
|
16
|
+
MAX_VIDEO_CHUNK_RETRIES,
|
|
15
17
|
UploadSource,
|
|
16
18
|
api_url,
|
|
17
19
|
ensure_stream_complete,
|
|
@@ -22,9 +24,12 @@ from ._shared import (
|
|
|
22
24
|
query_string,
|
|
23
25
|
read_exact,
|
|
24
26
|
resolve_upload_source,
|
|
27
|
+
resumed_offset,
|
|
25
28
|
segment,
|
|
26
29
|
sorted_parts,
|
|
30
|
+
tus_metadata,
|
|
27
31
|
upload_payload,
|
|
32
|
+
validate_chunk_size,
|
|
28
33
|
)
|
|
29
34
|
from .errors import ArkError, error_from_response, network_error, upload_error
|
|
30
35
|
from .models import (
|
|
@@ -479,3 +484,145 @@ class Streams:
|
|
|
479
484
|
def delete(self, stream_id: str, *, app_id: str | None = None) -> None:
|
|
480
485
|
suffix = query_string({"appId": app_id})
|
|
481
486
|
self._ark._request("DELETE", f"/streams/{segment(stream_id)}{suffix}")
|
|
487
|
+
|
|
488
|
+
def upload(
|
|
489
|
+
self,
|
|
490
|
+
source: str | Path | BinaryIO,
|
|
491
|
+
*,
|
|
492
|
+
title: str | None = None,
|
|
493
|
+
size: int | None = None,
|
|
494
|
+
filename: str | None = None,
|
|
495
|
+
content_type: str | None = None,
|
|
496
|
+
app_id: str | None = None,
|
|
497
|
+
collection_id: str | None = None,
|
|
498
|
+
chunk_size: int = DEFAULT_VIDEO_CHUNK_SIZE,
|
|
499
|
+
on_progress: Callable[[int, int], None] | None = None,
|
|
500
|
+
) -> ArkStream:
|
|
501
|
+
"""Create a stream and upload the video, resuming across failures.
|
|
502
|
+
|
|
503
|
+
`create()` only returns a ticket; the bytes still have to be sent, and
|
|
504
|
+
doing that correctly means speaking TUS. Without this a Python caller
|
|
505
|
+
had to implement the protocol themselves, which is the one part of
|
|
506
|
+
Streams they should not have to think about.
|
|
507
|
+
|
|
508
|
+
Returns the stream as it was at creation. Encoding continues afterwards,
|
|
509
|
+
so poll `get()` for `status == "ready"` before using the playback URLs.
|
|
510
|
+
"""
|
|
511
|
+
validate_chunk_size(chunk_size)
|
|
512
|
+
resolved = resolve_upload_source(
|
|
513
|
+
source,
|
|
514
|
+
size=size,
|
|
515
|
+
filename=filename,
|
|
516
|
+
content_type=content_type or "video/mp4",
|
|
517
|
+
)
|
|
518
|
+
|
|
519
|
+
created = self.create(
|
|
520
|
+
title or Path(resolved.filename).stem,
|
|
521
|
+
resolved.size,
|
|
522
|
+
app_id=app_id,
|
|
523
|
+
collection_id=collection_id,
|
|
524
|
+
)
|
|
525
|
+
|
|
526
|
+
upload_url = self._create_tus_upload(created.upload.endpoint, resolved)
|
|
527
|
+
handle = resolved.path.open("rb") if resolved.path is not None else resolved.stream
|
|
528
|
+
try:
|
|
529
|
+
self._send_chunks(upload_url, handle, resolved.size, chunk_size, on_progress)
|
|
530
|
+
finally:
|
|
531
|
+
if resolved.path is not None:
|
|
532
|
+
handle.close()
|
|
533
|
+
return created.stream
|
|
534
|
+
|
|
535
|
+
def _create_tus_upload(self, endpoint: str, resolved: UploadSource) -> str:
|
|
536
|
+
"""Open the TUS upload and return the URL to send chunks to."""
|
|
537
|
+
url = self._ark._url(endpoint) if endpoint.startswith("/") else endpoint
|
|
538
|
+
try:
|
|
539
|
+
response = self._ark._client.request(
|
|
540
|
+
"POST",
|
|
541
|
+
url,
|
|
542
|
+
headers={
|
|
543
|
+
"authorization": f"Bearer {self._ark._token}",
|
|
544
|
+
"Tus-Resumable": "1.0.0",
|
|
545
|
+
"Upload-Length": str(resolved.size),
|
|
546
|
+
"Upload-Metadata": (
|
|
547
|
+
f"filename {tus_metadata(resolved.filename)},"
|
|
548
|
+
f"filetype {tus_metadata(resolved.content_type)}"
|
|
549
|
+
),
|
|
550
|
+
},
|
|
551
|
+
)
|
|
552
|
+
except httpx.HTTPError as error:
|
|
553
|
+
raise network_error(error) from error
|
|
554
|
+
if response.is_error:
|
|
555
|
+
raise error_from_response(response)
|
|
556
|
+
|
|
557
|
+
location = response.headers.get("location")
|
|
558
|
+
if not location:
|
|
559
|
+
raise ArkError("INTERNAL_ERROR", "Ark did not return an upload location")
|
|
560
|
+
# The server may answer with a relative location.
|
|
561
|
+
return str(httpx.URL(url).join(location))
|
|
562
|
+
|
|
563
|
+
def _send_chunks(
|
|
564
|
+
self,
|
|
565
|
+
upload_url: str,
|
|
566
|
+
handle: BinaryIO,
|
|
567
|
+
total: int,
|
|
568
|
+
chunk_size: int,
|
|
569
|
+
on_progress: Callable[[int, int], None] | None,
|
|
570
|
+
) -> None:
|
|
571
|
+
offset = 0
|
|
572
|
+
retries = 0
|
|
573
|
+
while offset < total:
|
|
574
|
+
handle.seek(offset)
|
|
575
|
+
chunk = handle.read(min(chunk_size, total - offset))
|
|
576
|
+
if not chunk:
|
|
577
|
+
raise ArkError(
|
|
578
|
+
"INVALID_ARGUMENT",
|
|
579
|
+
"The video source ended before the declared size was reached",
|
|
580
|
+
)
|
|
581
|
+
try:
|
|
582
|
+
response = self._ark._client.request(
|
|
583
|
+
"PATCH",
|
|
584
|
+
upload_url,
|
|
585
|
+
headers={
|
|
586
|
+
"authorization": f"Bearer {self._ark._token}",
|
|
587
|
+
"Tus-Resumable": "1.0.0",
|
|
588
|
+
"Upload-Offset": str(offset),
|
|
589
|
+
"Content-Type": "application/offset+octet-stream",
|
|
590
|
+
},
|
|
591
|
+
content=chunk,
|
|
592
|
+
)
|
|
593
|
+
if response.is_error:
|
|
594
|
+
raise error_from_response(response)
|
|
595
|
+
# Trust the server's acknowledged offset over our own arithmetic:
|
|
596
|
+
# it is the only party that knows what actually landed.
|
|
597
|
+
acknowledged = response.headers.get("upload-offset")
|
|
598
|
+
try:
|
|
599
|
+
offset = max(offset + len(chunk), int(acknowledged or ""))
|
|
600
|
+
except ValueError:
|
|
601
|
+
offset += len(chunk)
|
|
602
|
+
retries = 0
|
|
603
|
+
if on_progress is not None:
|
|
604
|
+
on_progress(offset, total)
|
|
605
|
+
except ArkError as error:
|
|
606
|
+
if retries >= MAX_VIDEO_CHUNK_RETRIES:
|
|
607
|
+
raise
|
|
608
|
+
retries += 1
|
|
609
|
+
# Ask what the server holds rather than assuming the chunk was
|
|
610
|
+
# lost -- a failure after the bytes landed would otherwise send
|
|
611
|
+
# them twice.
|
|
612
|
+
offset = self._head_offset(upload_url, total, error)
|
|
613
|
+
|
|
614
|
+
def _head_offset(self, upload_url: str, total: int, fallback: ArkError) -> int:
|
|
615
|
+
try:
|
|
616
|
+
head = self._ark._client.request(
|
|
617
|
+
"HEAD",
|
|
618
|
+
upload_url,
|
|
619
|
+
headers={
|
|
620
|
+
"authorization": f"Bearer {self._ark._token}",
|
|
621
|
+
"Tus-Resumable": "1.0.0",
|
|
622
|
+
},
|
|
623
|
+
)
|
|
624
|
+
except httpx.HTTPError:
|
|
625
|
+
raise fallback from None
|
|
626
|
+
if head.is_error:
|
|
627
|
+
raise fallback
|
|
628
|
+
return resumed_offset(head.headers.get("upload-offset"), total, fallback)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
import pytest
|
|
7
|
+
from conftest import json_response
|
|
8
|
+
|
|
9
|
+
from ark_py import Ark, ArkError
|
|
10
|
+
|
|
11
|
+
STREAM = {
|
|
12
|
+
"id": "stream-1",
|
|
13
|
+
"title": "clip",
|
|
14
|
+
"status": "created",
|
|
15
|
+
"encodeProgress": 0,
|
|
16
|
+
"durationSeconds": 0,
|
|
17
|
+
"width": 0,
|
|
18
|
+
"height": 0,
|
|
19
|
+
"size": 9,
|
|
20
|
+
"thumbnailUrl": None,
|
|
21
|
+
"hlsUrl": None,
|
|
22
|
+
"embedUrl": None,
|
|
23
|
+
"createdAt": "2026-09-03T00:00:00.000Z",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class TusServer:
|
|
28
|
+
"""A TUS endpoint that records what it was sent.
|
|
29
|
+
|
|
30
|
+
`fail_at_offset` makes exactly one PATCH fail *after* accepting the bytes,
|
|
31
|
+
which is the case worth testing: a client that assumes a failed request
|
|
32
|
+
transferred nothing would resend those bytes and corrupt the video.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(self, total: int, *, fail_at_offset: int | None = None) -> None:
|
|
36
|
+
self.total = total
|
|
37
|
+
self.fail_at_offset = fail_at_offset
|
|
38
|
+
self.received = bytearray()
|
|
39
|
+
self.patches: list[int] = []
|
|
40
|
+
self.head_calls = 0
|
|
41
|
+
|
|
42
|
+
def handler(self, request: httpx.Request) -> httpx.Response:
|
|
43
|
+
if request.url.path.endswith("/streams") and request.method == "POST":
|
|
44
|
+
return json_response({"stream": STREAM, "upload": {"endpoint": "/streams/stream-1/upload"}})
|
|
45
|
+
|
|
46
|
+
if request.url.path == "/api/v2/streams/stream-1/upload" and request.method == "POST":
|
|
47
|
+
assert request.headers["upload-length"] == str(self.total)
|
|
48
|
+
assert "filename" in request.headers["upload-metadata"]
|
|
49
|
+
return httpx.Response(201, headers={"location": "/api/v2/streams/stream-1/upload/abc"})
|
|
50
|
+
|
|
51
|
+
if request.url.path.endswith("/upload/abc") and request.method == "HEAD":
|
|
52
|
+
self.head_calls += 1
|
|
53
|
+
return httpx.Response(200, headers={"upload-offset": str(len(self.received))})
|
|
54
|
+
|
|
55
|
+
if request.url.path.endswith("/upload/abc") and request.method == "PATCH":
|
|
56
|
+
offset = int(request.headers["upload-offset"])
|
|
57
|
+
self.patches.append(offset)
|
|
58
|
+
body = request.read()
|
|
59
|
+
# The server only accepts a write that starts where it left off.
|
|
60
|
+
assert offset == len(self.received), "client resent or skipped bytes"
|
|
61
|
+
self.received.extend(body)
|
|
62
|
+
if self.fail_at_offset is not None and offset == self.fail_at_offset:
|
|
63
|
+
self.fail_at_offset = None
|
|
64
|
+
return httpx.Response(500, json={"error": "flaky"})
|
|
65
|
+
return httpx.Response(204, headers={"upload-offset": str(len(self.received))})
|
|
66
|
+
|
|
67
|
+
raise AssertionError(f"unexpected {request.method} {request.url}")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def client_for(server: TusServer) -> Ark:
|
|
71
|
+
return Ark(
|
|
72
|
+
"token",
|
|
73
|
+
base_url="https://ark.test",
|
|
74
|
+
client=httpx.Client(transport=httpx.MockTransport(server.handler)),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_uploads_a_file_in_chunks(tmp_path: Path) -> None:
|
|
79
|
+
path = tmp_path / "clip.mp4"
|
|
80
|
+
path.write_bytes(b"abcdefghi")
|
|
81
|
+
server = TusServer(9)
|
|
82
|
+
|
|
83
|
+
stream = client_for(server).streams.upload(path, chunk_size=4)
|
|
84
|
+
|
|
85
|
+
assert stream.id == "stream-1"
|
|
86
|
+
assert bytes(server.received) == b"abcdefghi"
|
|
87
|
+
assert server.patches == [0, 4, 8]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_resumes_from_the_server_offset_after_a_failure(tmp_path: Path) -> None:
|
|
91
|
+
path = tmp_path / "clip.mp4"
|
|
92
|
+
path.write_bytes(b"abcdefghi")
|
|
93
|
+
# The failure lands after the server stored bytes 4-7.
|
|
94
|
+
server = TusServer(9, fail_at_offset=4)
|
|
95
|
+
|
|
96
|
+
client_for(server).streams.upload(path, chunk_size=4)
|
|
97
|
+
|
|
98
|
+
# The whole file arrives exactly once: the retry asked where to continue
|
|
99
|
+
# rather than resending the chunk that had in fact landed.
|
|
100
|
+
assert bytes(server.received) == b"abcdefghi"
|
|
101
|
+
assert server.head_calls == 1
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_reports_progress(tmp_path: Path) -> None:
|
|
105
|
+
path = tmp_path / "clip.mp4"
|
|
106
|
+
path.write_bytes(b"abcdefghi")
|
|
107
|
+
seen: list[tuple[int, int]] = []
|
|
108
|
+
|
|
109
|
+
client_for(TusServer(9)).streams.upload(
|
|
110
|
+
path, chunk_size=4, on_progress=lambda done, total: seen.append((done, total))
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
assert seen == [(4, 9), (8, 9), (9, 9)]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_derives_the_title_from_the_filename(tmp_path: Path) -> None:
|
|
117
|
+
path = tmp_path / "my-launch-video.mp4"
|
|
118
|
+
path.write_bytes(b"abc")
|
|
119
|
+
titles: list[str] = []
|
|
120
|
+
|
|
121
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
122
|
+
if request.url.path.endswith("/streams") and request.method == "POST":
|
|
123
|
+
import json as _json
|
|
124
|
+
|
|
125
|
+
titles.append(_json.loads(request.read())["title"])
|
|
126
|
+
return json_response({"stream": STREAM, "upload": {"endpoint": "/streams/stream-1/upload"}})
|
|
127
|
+
if request.method == "POST":
|
|
128
|
+
return httpx.Response(201, headers={"location": "/api/v2/streams/stream-1/upload/abc"})
|
|
129
|
+
return httpx.Response(204, headers={"upload-offset": "3"})
|
|
130
|
+
|
|
131
|
+
Ark(
|
|
132
|
+
"token",
|
|
133
|
+
base_url="https://ark.test",
|
|
134
|
+
client=httpx.Client(transport=httpx.MockTransport(handler)),
|
|
135
|
+
).streams.upload(path)
|
|
136
|
+
|
|
137
|
+
assert titles == ["my-launch-video"]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_rejects_an_invalid_chunk_size(tmp_path: Path) -> None:
|
|
141
|
+
path = tmp_path / "clip.mp4"
|
|
142
|
+
path.write_bytes(b"abc")
|
|
143
|
+
|
|
144
|
+
with pytest.raises(ArkError) as excinfo:
|
|
145
|
+
client_for(TusServer(3)).streams.upload(path, chunk_size=0)
|
|
146
|
+
|
|
147
|
+
assert excinfo.value.code == "INVALID_ARGUMENT"
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@pytest.mark.asyncio
|
|
151
|
+
async def test_async_upload_matches_the_sync_client(tmp_path: Path) -> None:
|
|
152
|
+
"""The async client must transfer identically, not merely succeed."""
|
|
153
|
+
from ark_py import AsyncArk
|
|
154
|
+
|
|
155
|
+
path = tmp_path / "clip.mp4"
|
|
156
|
+
path.write_bytes(b"abcdefghi")
|
|
157
|
+
server = TusServer(9, fail_at_offset=4)
|
|
158
|
+
|
|
159
|
+
ark = AsyncArk(
|
|
160
|
+
"token",
|
|
161
|
+
base_url="https://ark.test",
|
|
162
|
+
client=httpx.AsyncClient(transport=httpx.MockTransport(server.handler)),
|
|
163
|
+
)
|
|
164
|
+
stream = await ark.streams.upload(path, chunk_size=4)
|
|
165
|
+
|
|
166
|
+
assert stream.id == "stream-1"
|
|
167
|
+
assert bytes(server.received) == b"abcdefghi"
|
|
168
|
+
assert server.head_calls == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|