abysshub 0.1.1.dev4__tar.gz → 0.1.1.dev5__tar.gz

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: abysshub
-Version: 0.1.1.dev4
+Version: 0.1.1.dev5
 Summary: The official Python library for the Abyss API.
 Project-URL: Homepage, https://abysshub.com
 Project-URL: Source, https://github.com/Abysshub/abysshub-sdk
@@ -119,7 +119,7 @@ paths = run.save("output")  # [Path("output/charts/a.png"), Path("output/output.
 - **A string passes through** as an `https` URL or an upload id. A multi-file Field takes a list, which may mix all three kinds.
 - **Before the press,** each file is uploaded with `POST /v1/widgets/{widget}/uploads`, then sent to storage, in parallel, and its id goes in the input. An upload refusal (`held_input_*`) raises before any press. A press that drops before any headers is sent again with the same ids; it never uploads again.
 - **`run.save(dir)`** writes every output file under `dir` at its `path`, making folders, and returns the paths it wrote.
-- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`.
+- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`. A file of 64 MiB or more downloads as 16 parallel byte ranges of its one URL.
 
 ## Errors
 

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/README.md

@@ -102,7 +102,7 @@ paths = run.save("output")  # [Path("output/charts/a.png"), Path("output/output.
 - **A string passes through** as an `https` URL or an upload id. A multi-file Field takes a list, which may mix all three kinds.
 - **Before the press,** each file is uploaded with `POST /v1/widgets/{widget}/uploads`, then sent to storage, in parallel, and its id goes in the input. An upload refusal (`held_input_*`) raises before any press. A press that drops before any headers is sent again with the same ids; it never uploads again.
 - **`run.save(dir)`** writes every output file under `dir` at its `path`, making folders, and returns the paths it wrote.
-- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`.
+- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`. A file of 64 MiB or more downloads as 16 parallel byte ranges of its one URL.
 
 ## Errors
 

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "abysshub"
-version = "0.1.1.dev4"
+version = "0.1.1.dev5"
 description = "The official Python library for the Abyss API."
 readme = "README.md"
 license = "MIT"

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/src/abysshub/_async_client.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import math
-from collections.abc import Callable, Mapping
+from collections.abc import Awaitable, Callable, Mapping
 from types import TracebackType
 from typing import Any
 
@@ -12,6 +12,7 @@ from ._client import (
     REATTACH_PAUSE,
     _Base,
     _Call,
+    _dropped_download,
     _expect,
     _has_ended,
     _heard,
@@ -213,13 +214,24 @@ class AsyncAbyss(_Base[AsyncRun]):
         self._stored(call, field, filename, stored)
         return grant.id
 
-    async def _download(self, url: str) -> tuple[int, bytes]:
-        """Downloads an output file from storage, without the API key."""
+    async def _download(self, url: str, range_header: str | None) -> tuple[int, Callable[[], Awaitable[bytes]]]:
+        """Opens an output file in storage, without the API key, with a ``Range`` header or
+        none: returns its status, and a call that reads its content and closes it."""
+        headers = {"Range": range_header} if range_header else None
         try:
-            response = await self._http.get(url)
+            response = await self._http.send(self._http.build_request("GET", url, headers=headers), stream=True)
         except httpx.TransportError as error:
-            raise AbyssError("connection_error", "The connection to storage dropped during a download.") from error
-        return response.status_code, response.content
+            raise _dropped_download() from error
+
+        async def read() -> bytes:
+            try:
+                return await response.aread()
+            except httpx.TransportError as error:
+                raise _dropped_download() from error
+            finally:
+                await response.aclose()
+
+        return response.status_code, read
 
     async def _get_run(self, call: _Call, id: str, wait: bool) -> dict[str, Any]:
         """Reads a run with ``GET /v1/runs/{id}``, holding with ``?wait=true``. A cut hold is read again."""

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/src/abysshub/_client.py

@@ -445,13 +445,24 @@ class Abyss(_Base[Run]):
         self._stored(call, field, filename, stored)
         return grant.id
 
-    def _download(self, url: str) -> tuple[int, bytes]:
-        """Downloads an output file from storage, without the API key."""
+    def _download(self, url: str, range_header: str | None) -> tuple[int, Callable[[], bytes]]:
+        """Opens an output file in storage, without the API key, with a ``Range`` header or
+        none: returns its status, and a call that reads its content and closes it."""
+        headers = {"Range": range_header} if range_header else None
         try:
-            response = self._http.get(url)
+            response = self._http.send(self._http.build_request("GET", url, headers=headers), stream=True)
         except httpx.TransportError as error:
-            raise AbyssError("connection_error", "The connection to storage dropped during a download.") from error
-        return response.status_code, response.content
+            raise _dropped_download() from error
+
+        def read() -> bytes:
+            try:
+                return response.read()
+            except httpx.TransportError as error:
+                raise _dropped_download() from error
+            finally:
+                response.close()
+
+        return response.status_code, read
 
     def _get_run(self, call: _Call, id: str, wait: bool) -> dict[str, Any]:
         """Reads a run with ``GET /v1/runs/{id}``, holding with ``?wait=true``. A cut hold is read again."""
@@ -544,6 +555,10 @@ def _backoff(attempt: int) -> float:
     return min(RETRY_PAUSE * 2 ** min(attempt, 16), RETRY_PAUSE_CEILING)
 
 
+def _dropped_download() -> AbyssError:
+    return AbyssError("connection_error", "The connection to storage dropped during a download.")
+
+
 def _expect(reply: _Reply, is_: Callable[[Any], bool], message: str) -> Any:
     """The reply's body when it is what the route answers, else ``unexpected_response``."""
     if is_(reply.body):

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/src/abysshub/_run.py

@@ -1,22 +1,32 @@
 from __future__ import annotations
 
+import asyncio
+import math
 import os
 from collections.abc import Awaitable, Callable
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generic, Literal, TypeVar
 
 from ._error import AbyssError
 
-Download = Callable[[str], tuple[int, bytes]]
-"""Downloads a URL from storage: answers its status and content."""
+Download = Callable[[str, str | None], tuple[int, Callable[[], bytes]]]
+"""Opens a URL in storage, with a ``Range`` header or none: answers its status, and a call
+that reads its content."""
 
 Reread = Callable[[str], dict[str, Any]]
 """Reads a run again with ``GET /v1/runs/{id}``."""
 
-AsyncDownload = Callable[[str], Awaitable[tuple[int, bytes]]]
+AsyncDownload = Callable[[str, str | None], Awaitable[tuple[int, Callable[[], Awaitable[bytes]]]]]
 AsyncReread = Callable[[str], Awaitable[dict[str, Any]]]
 
+RANGED_FROM = 64 * 1024 * 1024
+"""A file this many bytes or larger downloads as parallel byte ranges: 64 MiB."""
+
+RANGES = 16
+"""How many parallel byte ranges a large file downloads as."""
+
 RunStatus = Literal["queued", "running", "succeeded", "failed"]
 """A run's place: ``succeeded`` and ``failed`` are final."""
 
@@ -45,14 +55,32 @@ class RunFile(_RunFileFields):
     _refresh: Callable[[], None] | None = field(default=None, init=False, repr=False, compare=False)
 
     def read(self) -> bytes:
-        """Downloads the file. An expired ``url`` is refreshed once, by reading the run again."""
+        """Downloads the file. An expired ``url`` is refreshed once, by reading the run again.
+        A file of ``RANGED_FROM`` bytes or more downloads as ``RANGES`` parallel byte ranges
+        of its one ``url``: the first goes alone, and once it answers 206 the others go
+        together. A storage that ignores ``Range`` answers the whole file to the first."""
         if self._download is None:
             raise _no_client(self.path)
-        status, content = self._download(self.url)
+        download = self._download
+        parts = _ranges(self.size)
+        first = _range(parts[0]) if parts else None
+        status, body = download(self.url, first)
         if status == 403 and self._refresh is not None:
+            body()
             self._refresh()
-            status, content = self._download(self.url)
-        return _downloaded(self.path, status, content)
+            status, body = download(self.url, first)
+        if not parts or status != 206:
+            return _downloaded(self.path, status, body())
+
+        def part(index: int) -> bytes:
+            if index == 0:
+                return _part(self.path, parts[0], status, body())
+            part_status, part_body = download(self.url, _range(parts[index]))
+            return _part(self.path, parts[index], part_status, part_body())
+
+        with ThreadPoolExecutor(len(parts)) as pool:
+            reads = [pool.submit(part, index) for index in range(len(parts))]
+            return b"".join(read.result() for read in reads)
 
     def save(self, path: str | os.PathLike[str]) -> Path:
         """Downloads the file to ``path``, making its folders, and returns ``path``."""
@@ -67,14 +95,35 @@ class AsyncRunFile(_RunFileFields):
     _refresh: Callable[[], Awaitable[None]] | None = field(default=None, init=False, repr=False, compare=False)
 
     async def read(self) -> bytes:
-        """Downloads the file. An expired ``url`` is refreshed once, by reading the run again."""
+        """Downloads the file. An expired ``url`` is refreshed once, by reading the run again.
+        A large file downloads as parallel byte ranges, as ``RunFile.read()`` does."""
         if self._download is None:
             raise _no_client(self.path)
-        status, content = await self._download(self.url)
+        download = self._download
+        parts = _ranges(self.size)
+        first = _range(parts[0]) if parts else None
+        status, body = await download(self.url, first)
         if status == 403 and self._refresh is not None:
+            await body()
             await self._refresh()
-            status, content = await self._download(self.url)
-        return _downloaded(self.path, status, content)
+            status, body = await download(self.url, first)
+        if not parts or status != 206:
+            return _downloaded(self.path, status, await body())
+
+        async def part(index: int) -> bytes:
+            if index == 0:
+                return _part(self.path, parts[0], status, await body())
+            part_status, part_body = await download(self.url, _range(parts[index]))
+            return _part(self.path, parts[index], part_status, await part_body())
+
+        reads = [asyncio.ensure_future(part(index)) for index in range(len(parts))]
+        try:
+            return b"".join(await asyncio.gather(*reads))
+        except BaseException:
+            for read in reads:
+                read.cancel()
+            await asyncio.gather(*reads, return_exceptions=True)
+            raise
 
     async def save(self, path: str | os.PathLike[str]) -> Path:
         """Downloads the file to ``path``, making its folders, and returns ``path``."""
@@ -204,10 +253,42 @@ def _no_client(path: str) -> AbyssError:
 
 def _downloaded(path: str, status: int, content: bytes) -> bytes:
     if not 200 <= status < 300:
-        raise AbyssError("unexpected_response", f"The download of {path} answered {status}.", status=status)
+        raise _answered(path, status)
     return content
 
 
+def _ranges(size: int) -> list[tuple[int, int]] | None:
+    """The byte ranges, first and last byte included, a file of ``size`` bytes downloads
+    as, or None when it downloads whole."""
+    if size < RANGED_FROM:
+        return None
+    step = math.ceil(size / RANGES)
+    return [(first, min(first + step, size) - 1) for first in range(0, size, step)]
+
+
+def _range(part: tuple[int, int]) -> str:
+    """The ``Range`` header naming ``part``."""
+    return f"bytes={part[0]}-{part[1]}"
+
+
+def _part(path: str, part: tuple[int, int], status: int, content: bytes) -> bytes:
+    """The bytes of one range, which must answer 206 with exactly the range's length."""
+    if status != 206:
+        raise _answered(path, status)
+    first, last = part
+    if len(content) != last - first + 1:
+        raise AbyssError(
+            "unexpected_response",
+            f"The download of {path} answered {len(content)} bytes for the range {first}-{last}.",
+            status=status,
+        )
+    return content
+
+
+def _answered(path: str, status: int) -> AbyssError:
+    return AbyssError("unexpected_response", f"The download of {path} answered {status}.", status=status)
+
+
 def _write(target: Path, content: bytes) -> Path:
     target.parent.mkdir(parents=True, exist_ok=True)
     target.write_bytes(content)

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/tests/fake_server.py

@@ -104,7 +104,9 @@ class Replay:
             found.append(f"expected {request['method']} {request['path']}, got {req.command} {req.path}")
         for name, value in (request.get("headers") or {}).items():
             got = req.headers.get(name)
-            if value == "{uuid}":
+            if value is None:
+                ok = got is None
+            elif value == "{uuid}":
                 ok = bool(UUID.match(got or "")) and (self._uuid is None or got == self._uuid)
             else:
                 ok = got == self._fill(value)
@@ -177,12 +179,23 @@ class Replay:
             return
         if "body" in response:
             _write_chunk(req, json.dumps(fill_every(response["body"], self._fill)).encode())
+        elif "pattern" in response:
+            _write_chunk(req, pattern(*response["pattern"]))
         elif response.get("text"):
             _write_chunk(req, response["text"].encode())
         req.wfile.write(b"0\r\n\r\n")
         req.wfile.flush()
 
 
+def pattern(first: int, last: int) -> bytes:
+    """The generated bytes ``first`` to ``last``, byte n being n % 251: a large file a
+    recording names in a few characters."""
+    start = first % 251
+    cycle = bytes(range(start, 251)) + bytes(range(start))
+    size = last - first + 1
+    return (cycle * (size // 251 + 1))[:size]
+
+
 def _write_chunk(req: BaseHTTPRequestHandler, data: bytes) -> None:
     req.wfile.write(b"%x\r\n%s\r\n" % (len(data), data))
     req.wfile.flush()

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/tests/test_files.py

@@ -11,7 +11,7 @@ from typing import Any
 
 import pytest
 from caller import with_files
-from fake_server import Replay, load_exchanges
+from fake_server import Replay, load_exchanges, pattern
 
 from abysshub import Abyss, AbyssError, AsyncAbyss, file
 
@@ -19,6 +19,7 @@ KEY = "abyss_sk_fake_for_tests"
 RECORDINGS = {recording["name"]: recording for recording in load_exchanges("files-*.json")}
 UPLOAD = RECORDINGS["files-upload-press.json"]
 EXPIRED = RECORDINGS["files-output-expired.json"]
+RANGED = RECORDINGS["files-output-ranged.json"]
 CONTENT = UPLOAD["call"]["files"]["report.pdf"]
 
 
@@ -147,3 +148,52 @@ def test_async_each_output_file_has_save_and_read_refreshing_an_expired_url_once
 
     with serving(EXPIRED) as (base, dir):
         asyncio.run(work(base, dir))
+
+
+def test_a_file_of_64_mib_or_more_reads_as_its_16_ranges_byte_for_byte() -> None:
+    with against(RANGED) as (abyss, _):
+        run = abyss.run(RANGED["call"]["widget"], RANGED["call"]["input"])
+        assert run.output_files[0].read() == pattern(0, RANGED["outcome"]["saved"]["frames.bin"]["pattern"] - 1)
+
+
+FAILURES: dict[str, tuple[dict[str, Any], str, int | None]] = {
+    "an error status": (
+        {"status": 503, "headers": {"Content-Type": "application/xml"}, "text": "<Error><Code>SlowDown</Code></Error>"},
+        "unexpected_response",
+        503,
+    ),
+    "a drop": ({"drop": True}, "connection_error", None),
+}
+
+
+@pytest.mark.parametrize("client", ["Abyss", "AsyncAbyss"])
+@pytest.mark.parametrize("failure", FAILURES)
+def test_a_range_that_fails_ends_the_download_with_the_error_a_whole_download_raises(
+    failure: str, client: str
+) -> None:
+    response, code, status = FAILURES[failure]
+    recording = copy.deepcopy(RANGED)
+    recording["exchanges"][9]["response"] = response
+
+    async def read_async(base: str) -> bytes:
+        async with AsyncAbyss(api_key=KEY, base_url=base) as abyss:
+            run = await abyss.run(RANGED["call"]["widget"], RANGED["call"]["input"])
+            return await run.output_files[0].read()
+
+    # The other ranges may be stopped before they reach the server, so not every
+    # recorded request need be sent.
+    server = Replay(recording, KEY)
+    try:
+        with pytest.raises(AbyssError) as raised:
+            if client == "Abyss":
+                with Abyss(api_key=KEY, base_url=server.base) as abyss:
+                    abyss.run(RANGED["call"]["widget"], RANGED["call"]["input"]).output_files[0].read()
+            else:
+                asyncio.run(read_async(server.base))
+        assert raised.value.code == code
+        assert raised.value.status == status
+        if status is not None:
+            assert raised.value.message == f"The download of frames.bin answered {status}."
+        assert server.problems == []
+    finally:
+        server.close()

{abysshub-0.1.1.dev4 → abysshub-0.1.1.dev5}/tests/test_replay.py

@@ -8,7 +8,7 @@ from typing import Any
 
 import pytest
 from caller import caller_input, perform, perform_async, with_files
-from fake_server import Replay, load_exchanges
+from fake_server import Replay, load_exchanges, pattern
 
 from abysshub import Abyss, AbyssError, AsyncAbyss, AsyncRun, Run
 
@@ -91,8 +91,12 @@ def test_replay(recording: dict[str, Any], client: str) -> None:
                     assert_fields(error.run, run, "error.run")
             if "saved" in outcome:
                 assert saved == [dir / save / path for path in outcome["saved"]], "run.save() returns the paths it wrote"
+                # A saved ``pattern`` is a whole file of that many generated bytes.
                 for path, content in outcome["saved"].items():
-                    assert (dir / save / path).read_text("utf-8") == content, f"saved {path}"
+                    if isinstance(content, dict):
+                        assert (dir / save / path).read_bytes() == pattern(0, content["pattern"] - 1), f"saved {path}"
+                    else:
+                        assert (dir / save / path).read_text("utf-8") == content, f"saved {path}"
     finally:
         server.close()