abysshub 0.0.1.dev8__tar.gz → 0.0.1.dev9__tar.gz

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: abysshub
-Version: 0.0.1.dev8
+Version: 0.0.1.dev9
 Summary: The official Python library for the Abyss API.
 Project-URL: Homepage, https://abysshub.com
 Project-URL: Source, https://github.com/Abysshub/abysshub-sdk
@@ -63,6 +63,24 @@ The press holds until the run ends. When the hold ends early (a deploy cuts it,
 
 A **run** is a dataclass with `/v1`'s fields: `id`, `widget`, `status` (`queued`, `running`, `succeeded` or `failed`), `price`, `result` (the Widget's JSON, a `dict` when it wrote an object), `output_files`, `error`, `created_at`, `started_at` and `ended_at`.
 
+## Files
+
+```python
+from abysshub import Abyss, file
+
+run = abyss.run("widget_1660", {
+    "report": file("report.pdf"),
+    "data": [file("jan.csv"), file(csv_bytes, filename="feb.csv"), "https://example.com/sales/mar.csv"],
+})
+paths = run.save("output")  # [Path("output/charts/a.png"), Path("output/output.json")]
+```
+
+- **`file(path)`** is a file on disk, read when it is uploaded. A file Field also takes a `Path`, an open file (`open("report.pdf", "rb")`) or `bytes`. `file(data, filename=...)` names data. A file without a name uploads under its Field's name.
+- **A string passes through** as an `https` URL or an upload id. A multi-file Field takes a list, which may mix all three kinds.
+- **Before the press,** each file is uploaded with `POST /v1/widgets/{widget}/uploads`, then sent to storage, in parallel, and its id goes in the input. An upload refusal (`held_input_*`) raises before any press. A press that drops before any headers is sent again with the same ids; it never uploads again.
+- **`run.save(dir)`** writes every output file under `dir` at its `path`, making folders, and returns the paths it wrote.
+- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`.
+
 ## Errors
 
 A refusal or a failed run raises one `AbyssError`. Branch on `code`:

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/README.md

@@ -46,6 +46,24 @@ The press holds until the run ends. When the hold ends early (a deploy cuts it,
 
 A **run** is a dataclass with `/v1`'s fields: `id`, `widget`, `status` (`queued`, `running`, `succeeded` or `failed`), `price`, `result` (the Widget's JSON, a `dict` when it wrote an object), `output_files`, `error`, `created_at`, `started_at` and `ended_at`.
 
+## Files
+
+```python
+from abysshub import Abyss, file
+
+run = abyss.run("widget_1660", {
+    "report": file("report.pdf"),
+    "data": [file("jan.csv"), file(csv_bytes, filename="feb.csv"), "https://example.com/sales/mar.csv"],
+})
+paths = run.save("output")  # [Path("output/charts/a.png"), Path("output/output.json")]
+```
+
+- **`file(path)`** is a file on disk, read when it is uploaded. A file Field also takes a `Path`, an open file (`open("report.pdf", "rb")`) or `bytes`. `file(data, filename=...)` names data. A file without a name uploads under its Field's name.
+- **A string passes through** as an `https` URL or an upload id. A multi-file Field takes a list, which may mix all three kinds.
+- **Before the press,** each file is uploaded with `POST /v1/widgets/{widget}/uploads`, then sent to storage, in parallel, and its id goes in the input. An upload refusal (`held_input_*`) raises before any press. A press that drops before any headers is sent again with the same ids; it never uploads again.
+- **`run.save(dir)`** writes every output file under `dir` at its `path`, making folders, and returns the paths it wrote.
+- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`.
+
 ## Errors
 
 A refusal or a failed run raises one `AbyssError`. Branch on `code`:

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "abysshub"
-version = "0.0.1.dev8"
+version = "0.0.1.dev9"
 description = "The official Python library for the Abyss API."
 readme = "README.md"
 license = "MIT"

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/src/abysshub/__init__.py

@@ -1,6 +1,7 @@
 """The official Python library for the Abyss API."""
 from ._client import Abyss
 from ._error import AbyssError
+from ._files import FileData, InputFile, file
 from ._run import Run, RunError, RunFile, RunStatus
 
-__all__ = ["Abyss", "AbyssError", "Run", "RunError", "RunFile", "RunStatus"]
+__all__ = ["Abyss", "AbyssError", "FileData", "InputFile", "Run", "RunError", "RunFile", "RunStatus", "file"]

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/src/abysshub/_client.py

@@ -13,6 +13,7 @@ from urllib.parse import quote
 import httpx
 
 from ._error import AbyssError
+from ._files import upload_files
 from ._http import request, timeouts
 from ._run import Run, RunError, parse_run
 
@@ -23,6 +24,9 @@ DEV_KEY_PREFIX = "abyss_sk_dev_"
 REATTACH_PAUSE = 1.0
 """Seconds between two re-attaches that both ended without the run's ending."""
 
+RETRY_PAUSE = 0.5
+"""Seconds before the first retry after a drop; the pause doubles with each retry."""
+
 
 @dataclass
 class _Reply:
@@ -80,15 +84,17 @@ class Abyss:
         max_price: float | None = None,
         idempotency_key: str | None = None,
     ) -> Run:
-        """Presses a Widget and waits for the run's ending.
+        """Uploads the input's files, presses a Widget and waits for the run's ending.
 
+        A file Field takes ``file(path)``, a ``Path``, an open file or ``bytes``, or a
+        list of them; a string passes through as an ``https`` URL or an upload id.
         Re-attaches through ``Location`` whenever the hold ends early. Returns the
         succeeded run; raises ``AbyssError`` on a refusal or a failed run.
         """
         press = self._press(widget, input, max_price, idempotency_key)
         reply = press if _has_ended(press) else self._hold(self._wait_url(press), _has_ended)
         assert reply.run is not None
-        run = parse_run(reply.run)
+        run = self._run(reply.run)
         if run.status == "failed":
             error = run.error or RunError("platform_fault", "The run failed.")
             raise AbyssError(error.code, error.message, request_id=reply.request_id, run=run)
@@ -101,14 +107,72 @@ class Abyss:
         max_price: float | None,
         idempotency_key: str | None,
     ) -> _Reply:
-        body: dict[str, Any] = {"input": dict(input)}
+        """Uploads the input's files and sends the press. Its retries send the same body
+        and ``Idempotency-Key``, so the uploads are never redone."""
+        widget_url = self._widget_url(widget)
+        uploaded = upload_files(input, lambda field, filename, data: self._upload(widget_url, field, filename, data))
+        body: dict[str, Any] = {"input": uploaded}
         if max_price is not None:
             body["max_price"] = max_price
         headers = {
             "Content-Type": "application/json",
             "Idempotency-Key": idempotency_key if idempotency_key is not None else str(uuid.uuid4()),
         }
-        return self._send("POST", f"{self._widget_url(widget)}/runs", headers, json.dumps(body).encode())
+        return self._send("POST", f"{widget_url}/runs", headers, json.dumps(body).encode())
+
+    def _upload(self, widget_url: str, field: str, filename: str, data: bytes) -> str:
+        """Asks for an upload, sends the file to storage and answers the upload's id."""
+        grant = self._send(
+            "POST",
+            f"{widget_url}/uploads",
+            {"Content-Type": "application/json"},
+            json.dumps({"field": field, "filename": filename, "size": len(data)}).encode(),
+        )
+        if not _is_upload(grant.body):
+            raise AbyssError(
+                "unexpected_response",
+                "The upload answered without an id and a form.",
+                request_id=grant.request_id,
+            )
+        upload = grant.body["upload"]
+        try:
+            stored = self._http.post(
+                upload["url"],
+                data={name: str(value) for name, value in upload["fields"].items()},
+                files={"file": (filename, data)},
+            )
+        except httpx.TransportError as error:
+            raise AbyssError(
+                "connection_error",
+                f"The connection to storage dropped during the upload of {filename}.",
+                param=field,
+            ) from error
+        if not stored.is_success:
+            raise AbyssError(
+                "unexpected_response",
+                f"Storage answered {stored.status_code} to the upload of {filename}.",
+                status=stored.status_code,
+                param=field,
+            )
+        return str(grant.body["id"])
+
+    def _download(self, url: str) -> tuple[int, bytes]:
+        """Downloads an output file from storage, without the API key."""
+        try:
+            response = self._http.get(url)
+        except httpx.TransportError as error:
+            raise AbyssError("connection_error", "The connection to storage dropped during a download.") from error
+        return response.status_code, response.content
+
+    def _reread(self, id: str) -> dict[str, Any]:
+        """Reads a run with ``GET /v1/runs/{id}``. A cut read is read again."""
+        reply = self._hold(f"{self.base_url}/v1/runs/{quote(id, safe='')}", lambda reply: not reply.cut)
+        if reply.run is None:
+            raise AbyssError("unexpected_response", "The read answered without a run.", request_id=reply.request_id)
+        return reply.run
+
+    def _run(self, data: dict[str, Any]) -> Run:
+        return parse_run(data, self._reread, self._download)
 
     def _hold(self, url: str, done: Callable[[_Reply], bool]) -> _Reply:
         """Reads ``url`` until ``done``. Re-attaching has no limit and is not a retry."""
@@ -132,20 +196,22 @@ class Abyss:
         return str(httpx.URL(f"{self.base_url}/").join(location).copy_set_param("wait", "true"))
 
     def _send(self, method: str, url: str, headers: dict[str, str] | None = None, content: bytes | None = None) -> _Reply:
-        """Sends one request to `/v1`. Every refusal raises at once."""
-        try:
-            answer = request(
-                self._http,
-                method,
-                url,
-                {"Authorization": f"Bearer {self._api_key}", "Accept": "application/json", **(headers or {})},
-                content,
-            )
-        except httpx.TransportError as error:
-            raise AbyssError(
-                "connection_error",
-                "The connection dropped, or timed out, before the API answered.",
-            ) from error
+        """Sends one request to `/v1`. A drop before any headers is sent again, at most
+        ``max_retries`` times; every refusal raises at once."""
+        sent_headers = {"Authorization": f"Bearer {self._api_key}", "Accept": "application/json", **(headers or {})}
+        attempt = 0
+        while True:
+            try:
+                answer = request(self._http, method, url, sent_headers, content)
+                break
+            except httpx.TransportError as error:
+                if attempt >= self.max_retries:
+                    raise AbyssError(
+                        "connection_error",
+                        "The connection dropped, or timed out, before the API answered.",
+                    ) from error
+                time.sleep(RETRY_PAUSE * 2**attempt)
+                attempt += 1
         request_id = answer.headers.get("request-id")
         parsed = _parse(answer.text)
         if answer.status in (200, 202):
@@ -191,6 +257,13 @@ def _is_run(value: Any) -> bool:
     return isinstance(value, dict) and isinstance(value.get("id"), str) and isinstance(value.get("status"), str)
 
 
+def _is_upload(value: Any) -> bool:
+    if not isinstance(value, dict) or not isinstance(value.get("id"), str):
+        return False
+    upload = value.get("upload")
+    return isinstance(upload, dict) and isinstance(upload.get("url"), str) and isinstance(upload.get("fields"), dict)
+
+
 def _has_ended(reply: _Reply) -> bool:
     """Whether the reply carries a run that has succeeded or failed."""
     return reply.run is not None and reply.run["status"] in ("succeeded", "failed")

abysshub-0.0.1.dev9/src/abysshub/_files.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Callable, Mapping
+from concurrent.futures import Future, ThreadPoolExecutor
+from pathlib import Path
+from typing import IO, Any, Union
+
+FileData = Union[bytes, bytearray, memoryview, os.PathLike[str], IO[bytes], IO[str]]
+"""File data: bytes in memory, a ``Path``, or an open file."""
+
+MAX_PARALLEL_UPLOADS = 8
+
+
+class InputFile:
+    """A file for a file Field: a path, read when it is uploaded, or data. Made by ``file()``."""
+
+    source: str | FileData
+    filename: str | None
+    """The name to upload under. Defaults to the path's or the open file's name, else the Field's name."""
+
+    def __init__(self, source: str | FileData, filename: str | None = None) -> None:
+        self.source = source
+        self.filename = filename
+
+    def __repr__(self) -> str:
+        return f"file({self.source!r})"
+
+
+def file(source: str | FileData, *, filename: str | None = None) -> InputFile:
+    """A file for a file Field. A string is a path, read when the file is uploaded."""
+    return InputFile(source, filename)
+
+
+Uploader = Callable[[str, str, bytes], str]
+"""Uploads one file for a Field, by its name and content, and answers the upload's id."""
+
+
+def upload_files(input: Mapping[str, Any], upload: Uploader) -> dict[str, Any]:
+    """The input with every file value uploaded, in parallel, and replaced by its upload's id.
+
+    Strings pass through as an ``https`` URL or an upload id, and a list may mix all
+    kinds. The caller's input is never changed.
+    """
+    uploaded = dict(input)
+    jobs: list[tuple[Callable[[str], None], str, InputFile | FileData]] = []
+    for field, value in input.items():
+        if is_file(value):
+            jobs.append((_setter(uploaded, field), field, value))
+        elif isinstance(value, list) and any(is_file(item) for item in value):
+            items = list(value)
+            uploaded[field] = items
+            jobs.extend((_setter(items, index), field, item) for index, item in enumerate(value) if is_file(item))
+    if not jobs:
+        return uploaded
+
+    def upload_one(field: str, value: InputFile | FileData) -> str:
+        filename, data = _load(value, field)
+        return upload(field, filename, data)
+
+    with ThreadPoolExecutor(max_workers=min(len(jobs), MAX_PARALLEL_UPLOADS)) as pool:
+        futures: list[Future[str]] = [pool.submit(upload_one, field, value) for _, field, value in jobs]
+    for (put, _, _), future in zip(jobs, futures):
+        put(future.result())
+    return uploaded
+
+
+def is_file(value: Any) -> bool:
+    """Whether the value is a file to upload, rather than a Field's plain value."""
+    return isinstance(value, (InputFile, bytes, bytearray, memoryview, os.PathLike)) or callable(
+        getattr(value, "read", None)
+    )
+
+
+def _setter(target: Any, at: Any) -> Callable[[str], None]:
+    def put(upload_id: str) -> None:
+        target[at] = upload_id
+
+    return put
+
+
+def _load(value: InputFile | FileData, field: str) -> tuple[str, bytes]:
+    """The file's name and content."""
+    given = value if isinstance(value, InputFile) else InputFile(value)
+    source = given.source
+    if isinstance(source, (str, os.PathLike)):
+        path = Path(source)
+        return given.filename or path.name, path.read_bytes()
+    if isinstance(source, (bytes, bytearray, memoryview)):
+        return given.filename or field, bytes(source)
+    content = source.read()
+    data = content.encode("utf-8") if isinstance(content, str) else bytes(content)
+    name = getattr(source, "name", None)
+    return given.filename or (Path(name).name if isinstance(name, str) and name else field), data

abysshub-0.0.1.dev9/src/abysshub/_run.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+from ._error import AbyssError
+
+Download = Callable[[str], tuple[int, bytes]]
+"""Downloads a URL from storage: answers its status and content."""
+
+Reread = Callable[[str], dict[str, Any]]
+"""Reads a run again with ``GET /v1/runs/{id}``."""
+
+RunStatus = Literal["queued", "running", "succeeded", "failed"]
+"""A run's place: ``succeeded`` and ``failed`` are final."""
+
+
+@dataclass
+class RunError:
+    """Why a run failed: ``widget_fault``, ``platform_fault`` or ``budget_exceeded``."""
+
+    code: str
+    message: str
+
+
+@dataclass
+class RunFile:
+    """One file a run wrote. Its ``url`` is signed fresh on every read of the run, and expires."""
+
+    path: str
+    size: int
+    content_type: str
+    url: str
+    _download: Download | None = field(default=None, init=False, repr=False, compare=False)
+    _refresh: Callable[[], None] | None = field(default=None, init=False, repr=False, compare=False)
+
+    def read(self) -> bytes:
+        """Downloads the file. An expired ``url`` is refreshed once, by reading the run again."""
+        if self._download is None:
+            raise AbyssError("unexpected_response", f"The output file {self.path} has no client to download it.")
+        status, content = self._download(self.url)
+        if status == 403 and self._refresh is not None:
+            self._refresh()
+            status, content = self._download(self.url)
+        if not 200 <= status < 300:
+            raise AbyssError(
+                "unexpected_response",
+                f"The download of {self.path} answered {status}.",
+                status=status,
+            )
+        return content
+
+    def save(self, path: str | os.PathLike[str]) -> Path:
+        """Downloads the file to ``path``, making its folders, and returns ``path``."""
+        target = Path(path)
+        content = self.read()
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(content)
+        return target
+
+
+@dataclass
+class Run:
+    """A run, with `/v1`'s names."""
+
+    id: str
+    widget: str
+    status: RunStatus
+    price: float
+    result: Any
+    output_files: list[RunFile]
+    error: RunError | None
+    created_at: str
+    started_at: str | None
+    ended_at: str | None
+    _reread: Reread | None = field(default=None, init=False, repr=False, compare=False)
+
+    def save(self, dir: str | os.PathLike[str]) -> list[Path]:
+        """Saves every output file under ``dir``, at its ``path``, and returns the paths it wrote."""
+        folder = Path(dir)
+        root = folder.resolve()
+        saved = []
+        for output in self.output_files:
+            target = folder / output.path
+            resolved = target.resolve()
+            if resolved == root or not resolved.is_relative_to(root):
+                raise AbyssError(
+                    "unexpected_response",
+                    f"The output file {output.path} would be saved outside {folder}.",
+                )
+            saved.append(output.save(target))
+        return saved
+
+    def _refresh_urls(self) -> None:
+        """Reads the run again and gives every output file its freshly signed ``url``."""
+        if self._reread is None:
+            return
+        fresh = {f.get("path"): f.get("url") for f in self._reread(self.id).get("output_files") or []}
+        for output in self.output_files:
+            url = fresh.get(output.path)
+            if isinstance(url, str):
+                output.url = url
+
+
+def parse_run(data: dict[str, Any], reread: Reread | None = None, download: Download | None = None) -> Run:
+    """The run `/v1` sent as ``data``. With ``reread`` and ``download``, its output files can be read."""
+    error = data.get("error")
+    run = Run(
+        id=data["id"],
+        widget=data.get("widget", ""),
+        status=data["status"],
+        price=data.get("price", 0),
+        result=data.get("result"),
+        output_files=[
+            RunFile(path=f["path"], size=f["size"], content_type=f["content_type"], url=f["url"])
+            for f in data.get("output_files") or []
+        ],
+        error=RunError(code=error["code"], message=error["message"]) if isinstance(error, dict) else None,
+        created_at=data.get("created_at", ""),
+        started_at=data.get("started_at"),
+        ended_at=data.get("ended_at"),
+    )
+    run._reread = reread
+    for output in run.output_files:
+        output._download = download
+        output._refresh = run._refresh_urls
+    return run

abysshub-0.0.1.dev9/tests/caller.py

@@ -0,0 +1,43 @@
+"""What a recorded call's caller holds: its files on disk, and its input as a caller writes it."""
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+from abysshub import Abyss, file
+
+
+def caller_input(input: dict[str, Any], dir: Path, files: dict[str, str]) -> dict[str, Any]:
+    """The caller's input: ``{"$file": name}`` is ``file(path)``, and ``{"$bytes": name}``
+    its content in memory, given with its name."""
+
+    def value(item: Any) -> Any:
+        if isinstance(item, dict) and "$file" in item:
+            return file(str(dir / item["$file"]))
+        if isinstance(item, dict) and "$bytes" in item:
+            return file(files[item["$bytes"]].encode(), filename=item["$bytes"])
+        return item
+
+    return {field: [value(i) for i in item] if isinstance(item, list) else value(item) for field, item in input.items()}
+
+
+@contextmanager
+def with_files(files: dict[str, str]) -> Iterator[Path]:
+    """A folder holding the caller's files, removed afterwards."""
+    with TemporaryDirectory(prefix="abysshub-") as name:
+        dir = Path(name)
+        for path, content in files.items():
+            (dir / path).write_bytes(content.encode())
+        yield dir
+
+
+def perform(abyss: Abyss, call: dict[str, Any], input: dict[str, Any]) -> Any:
+    """The recorded call, made through the method it names."""
+    method = call.get("method", "run")
+    options = call.get("options") or {}
+    if method == "run":
+        return abyss.run(call["widget"], input, **options)
+    raise AssertionError(f"no such method: {method}")

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/tests/fake_server.py

@@ -8,6 +8,8 @@ import socket
 import threading
 import time
 from collections.abc import Callable
+from email.message import Message
+from email.parser import BytesParser
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Any
@@ -39,7 +41,7 @@ class Replay:
         self.recording = recording
         self.key = key
         self.problems: list[str] = []
-        self.served = 0
+        self.served: set[int] = set()
         self._uuid: str | None = None
         self._lock = threading.Lock()
         self._closing = threading.Event()
@@ -63,7 +65,7 @@ class Replay:
         self._thread.start()
 
     def remaining(self) -> int:
-        return len(self.recording["exchanges"]) - self.served
+        return len(self.recording["exchanges"]) - len(self.served)
 
     def close(self) -> None:
         self._closing.set()
@@ -73,6 +75,23 @@ class Replay:
     def _fill(self, value: str) -> str:
         return value.replace("{base}", self.base).replace("{key}", self.key)
 
+    def _candidates(self) -> list[int]:
+        """The exchanges the next request may answer: the first one not yet served, or
+        every unserved one in its block when it is marked ``parallel``."""
+        exchanges = self.recording["exchanges"]
+        first = next((index for index in range(len(exchanges)) if index not in self.served), None)
+        if first is None:
+            return []
+        if not exchanges[first].get("parallel"):
+            return [first]
+        block = []
+        index = first
+        while index < len(exchanges) and exchanges[index].get("parallel"):
+            if index not in self.served:
+                block.append(index)
+            index += 1
+        return block
+
     def _differences(self, index: int, req: BaseHTTPRequestHandler, sent: bytes) -> list[str]:
         exchange = self.recording["exchanges"][index]
         request = exchange["request"]
@@ -91,16 +110,23 @@ class Replay:
             found.append("storage was sent the API key")
         if "body" in request and _parse_or_text(sent.decode("utf-8")) != request["body"]:
             found.append(f"body is {sent.decode('utf-8')}")
+        if "form" in request:
+            form = _read_form(req.headers.get("Content-Type"), sent)
+            if form != request["form"]:
+                found.append(f"form is {json.dumps(form)}")
         return found
 
     def _choose(self, req: BaseHTTPRequestHandler, sent: bytes) -> int | None:
+        """The candidate the request matches, else the first one, noting why."""
         with self._lock:
-            index = self.served
-            if index >= len(self.recording["exchanges"]):
+            candidates = self._candidates()
+            if not candidates:
                 self.problems.append(f"unexpected {req.command} {req.path}")
                 return None
-            self.served += 1
-            self.problems.extend(f"#{index + 1}: {problem}" for problem in self._differences(index, req, sent))
+            index = next((other for other in candidates if not self._differences(other, req, sent)), candidates[0])
+            found = self._differences(index, req, sent)
+            self.served.add(index)
+            self.problems.extend(f"#{index + 1}: {problem}" for problem in found)
             for name, value in (self.recording["exchanges"][index]["request"].get("headers") or {}).items():
                 if value == "{uuid}" and self._uuid is None:
                     self._uuid = req.headers.get(name)
@@ -125,6 +151,10 @@ class Replay:
             req.send_header("Content-Type", "application/json")
         for name, value in (response.get("headers") or {}).items():
             req.send_header(name, self._fill(value))
+        if response["status"] == 204:
+            req.end_headers()
+            req.wfile.flush()
+            return
         # Chunked, as Node sends it, so a cut is told apart from the body's end.
         req.send_header("Transfer-Encoding", "chunked")
         req.end_headers()
@@ -154,6 +184,24 @@ def _write_chunk(req: BaseHTTPRequestHandler, data: bytes) -> None:
     req.wfile.flush()
 
 
+def _read_form(content_type: str | None, sent: bytes) -> dict[str, Any]:
+    """A storage upload form, as the recordings write it: each field's text, with the
+    file (which must come last) as its content."""
+    message = BytesParser().parsebytes(b"Content-Type: %s\r\n\r\n%s" % ((content_type or "").encode(), sent))
+    if not message.is_multipart():
+        return {"(not a form)": content_type}
+    form: dict[str, Any] = {}
+    name = None
+    for part in message.get_payload():
+        assert isinstance(part, Message)
+        name = part.get_param("name", header="content-disposition")
+        content = part.get_payload(decode=True)
+        form[str(name)] = content.decode("utf-8") if isinstance(content, bytes) else content
+    if name != "file":
+        form["(the file is not last)"] = True
+    return form
+
+
 def _parse_or_text(text: str) -> Any:
     try:
         return json.loads(text)

abysshub-0.0.1.dev9/tests/test_files.py

@@ -0,0 +1,117 @@
+"""Files in and out, beyond the recorded calls: every kind of file a caller may pass,
+and each output file's save() and read()."""
+from __future__ import annotations
+
+import copy
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+
+import pytest
+from caller import with_files
+from fake_server import Replay, load_exchanges
+
+from abysshub import Abyss, AbyssError, file
+
+KEY = "abyss_sk_fake_for_tests"
+RECORDINGS = {recording["name"]: recording for recording in load_exchanges("files-*.json")}
+UPLOAD = RECORDINGS["files-upload-press.json"]
+EXPIRED = RECORDINGS["files-output-expired.json"]
+CONTENT = UPLOAD["call"]["files"]["report.pdf"]
+
+
+@contextmanager
+def against(recording: dict[str, Any]) -> Iterator[tuple[Abyss, Path]]:
+    server = Replay(recording, KEY)
+    try:
+        with with_files(recording["call"].get("files") or {}) as dir, Abyss(api_key=KEY, base_url=server.base) as abyss:
+            yield abyss, dir
+        assert server.problems == []
+        assert server.remaining() == 0, "every recorded request was sent"
+    finally:
+        server.close()
+
+
+SOURCES: dict[str, Callable[[Path], Any]] = {
+    "file(path)": lambda dir: file(str(dir / "report.pdf")),
+    "file(Path)": lambda dir: file(dir / "report.pdf"),
+    "a Path": lambda dir: dir / "report.pdf",
+    "file(bytes)": lambda dir: file(CONTENT.encode(), filename="report.pdf"),
+}
+
+
+@pytest.mark.parametrize("kind", SOURCES)
+def test_every_kind_of_file_uploads_like_file_path(kind: str) -> None:
+    with against(UPLOAD) as (abyss, dir):
+        run = abyss.run(UPLOAD["call"]["widget"], {"report": SOURCES[kind](dir)})
+        assert run.status == "succeeded"
+
+
+def test_an_open_file_uploads_under_its_name() -> None:
+    with against(UPLOAD) as (abyss, dir), open(dir / "report.pdf", "rb") as report:
+        run = abyss.run("widget_1660", {"report": report})
+        assert run.status == "succeeded"
+
+
+def test_bytes_upload_under_the_fields_name() -> None:
+    recording = copy.deepcopy(UPLOAD)
+    recording["exchanges"][0]["request"]["body"]["filename"] = "report"
+    with against(recording) as (abyss, _):
+        assert abyss.run("widget_1660", {"report": CONTENT.encode()}).status == "succeeded"
+
+
+def test_the_callers_input_is_never_changed() -> None:
+    with against(UPLOAD) as (abyss, dir):
+        report = file(str(dir / "report.pdf"))
+        input = {"report": report}
+        abyss.run("widget_1660", input)
+        assert input == {"report": report}
+
+
+def test_a_wrong_upload_form_is_noted() -> None:
+    recording = copy.deepcopy(UPLOAD)
+    recording["exchanges"][1]["request"]["form"]["file"] = "another report"
+    server = Replay(recording, KEY)
+    try:
+        with with_files(recording["call"]["files"]) as dir, Abyss(api_key=KEY, base_url=server.base) as abyss:
+            abyss.run("widget_1660", {"report": file(dir / "report.pdf")})
+        assert len(server.problems) == 1
+        assert server.problems[0].startswith("#2: form is ")
+    finally:
+        server.close()
+
+
+def test_each_output_file_has_save_and_read_refreshing_an_expired_url_once() -> None:
+    with against(EXPIRED) as (abyss, dir):
+        run = abyss.run(EXPIRED["call"]["widget"], EXPIRED["call"]["input"])
+        chart, output = run.output_files
+        path = dir / "deep" / "chart.png"
+        assert chart.save(path) == path
+        assert path.read_text("utf-8") == EXPIRED["outcome"]["saved"]["charts/a.png"]
+        content = output.read()
+        assert isinstance(content, bytes)
+        assert content.decode("utf-8") == EXPIRED["outcome"]["saved"]["output.json"]
+
+
+def test_a_url_still_refused_after_the_refresh_raises() -> None:
+    recording = copy.deepcopy(EXPIRED)
+    recording["exchanges"] = recording["exchanges"][:3]
+    refused = copy.deepcopy(recording["exchanges"][1])
+    refused["request"]["path"] = refused["request"]["path"].replace("expired", "fresh")
+    recording["exchanges"].append(refused)
+    with against(recording) as (abyss, _):
+        run = abyss.run(EXPIRED["call"]["widget"], EXPIRED["call"]["input"])
+        with pytest.raises(AbyssError) as raised:
+            run.output_files[0].read()
+        assert raised.value.status == 403
+
+
+def test_run_save_refuses_an_output_path_that_leaves_its_folder() -> None:
+    recording = copy.deepcopy(EXPIRED)
+    recording["exchanges"] = recording["exchanges"][:1]
+    recording["exchanges"][0]["response"]["body"]["output_files"][0]["path"] = "../escape.png"
+    with against(recording) as (abyss, dir):
+        run = abyss.run(EXPIRED["call"]["widget"], EXPIRED["call"]["input"])
+        with pytest.raises(AbyssError):
+            run.save(dir / "output")

{abysshub-0.0.1.dev8 → abysshub-0.0.1.dev9}/tests/test_replay.py

@@ -5,12 +5,13 @@ from __future__ import annotations
 from typing import Any
 
 import pytest
+from caller import caller_input, perform, with_files
 from fake_server import Replay, load_exchanges
 
 from abysshub import Abyss, AbyssError, Run
 
 KEY = "abyss_sk_fake_for_tests"
-RECORDINGS = load_exchanges("run-*.json")
+RECORDINGS = load_exchanges("run-*.json") + load_exchanges("files-*.json")
 
 
 def assert_fields(actual: Any, expected: Any, where: str) -> None:
@@ -30,41 +31,39 @@ def assert_fields(actual: Any, expected: Any, where: str) -> None:
         assert actual == expected, where
 
 
-def perform(abyss: Abyss, call: dict[str, Any]) -> Any:
-    """The recorded call, made through the method it names."""
-    method = call.get("method", "run")
-    options = call.get("options") or {}
-    if method == "run":
-        return abyss.run(call["widget"], call.get("input") or {}, **options)
-    raise AssertionError(f"no such method: {method}")
-
-
 @pytest.mark.parametrize("recording", RECORDINGS, ids=[r["name"] for r in RECORDINGS])
 def test_replay(recording: dict[str, Any]) -> None:
+    call = recording["call"]
+    files = call.get("files") or {}
+    save = call.get("save")
     server = Replay(recording, KEY)
     try:
-        client = recording["call"].get("client") or {}
-        with Abyss(api_key=KEY, base_url=server.base, **client) as abyss:
+        with with_files(files) as dir, Abyss(api_key=KEY, base_url=server.base, **(call.get("client") or {})) as abyss:
             try:
-                value, error = perform(abyss, recording["call"]), None
+                value, error = perform(abyss, call, caller_input(call.get("input") or {}, dir, files)), None
             except AbyssError as raised:
                 value, error = None, raised
+            saved = value.save(dir / save) if save and isinstance(value, Run) else None
 
-        assert server.problems == []
-        assert server.remaining() == 0, "every recorded request was sent"
-        outcome = recording["outcome"]
-        if "run" in outcome:
-            assert isinstance(value, Run), f"expected a run, got {error!r}"
-            assert_fields(value, outcome["run"], "run")
-        else:
-            assert error is not None, f"expected an AbyssError, got {value!r}"
-            fields = dict(outcome["error"])
-            run = fields.pop("run")
-            assert_fields(error, fields, "error")
-            if run is None:
-                assert error.run is None
+            assert server.problems == []
+            assert server.remaining() == 0, "every recorded request was sent"
+            outcome = recording["outcome"]
+            if "run" in outcome:
+                assert isinstance(value, Run), f"expected a run, got {error!r}"
+                assert_fields(value, outcome["run"], "run")
             else:
-                assert_fields(error.run, run, "error.run")
+                assert error is not None, f"expected an AbyssError, got {value!r}"
+                fields = dict(outcome["error"])
+                run = fields.pop("run")
+                assert_fields(error, fields, "error")
+                if run is None:
+                    assert error.run is None
+                else:
+                    assert_fields(error.run, run, "error.run")
+            if "saved" in outcome:
+                assert saved == [dir / save / path for path in outcome["saved"]], "run.save() returns the paths it wrote"
+                for path, content in outcome["saved"].items():
+                    assert (dir / save / path).read_text("utf-8") == content, f"saved {path}"
     finally:
         server.close()
 

abysshub-0.0.1.dev8/src/abysshub/_run.py

@@ -1,61 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Any, Literal
-
-RunStatus = Literal["queued", "running", "succeeded", "failed"]
-"""A run's place: ``succeeded`` and ``failed`` are final."""
-
-
-@dataclass
-class RunError:
-    """Why a run failed: ``widget_fault``, ``platform_fault`` or ``budget_exceeded``."""
-
-    code: str
-    message: str
-
-
-@dataclass
-class RunFile:
-    """One file a run wrote. Its ``url`` is signed fresh on every read of the run, and expires."""
-
-    path: str
-    size: int
-    content_type: str
-    url: str
-
-
-@dataclass
-class Run:
-    """A run, with `/v1`'s names."""
-
-    id: str
-    widget: str
-    status: RunStatus
-    price: float
-    result: Any
-    output_files: list[RunFile]
-    error: RunError | None
-    created_at: str
-    started_at: str | None
-    ended_at: str | None
-
-
-def parse_run(data: dict[str, Any]) -> Run:
-    """The run `/v1` sent as ``data``."""
-    error = data.get("error")
-    return Run(
-        id=data["id"],
-        widget=data.get("widget", ""),
-        status=data["status"],
-        price=data.get("price", 0),
-        result=data.get("result"),
-        output_files=[
-            RunFile(path=f["path"], size=f["size"], content_type=f["content_type"], url=f["url"])
-            for f in data.get("output_files") or []
-        ],
-        error=RunError(code=error["code"], message=error["message"]) if isinstance(error, dict) else None,
-        created_at=data.get("created_at", ""),
-        started_at=data.get("started_at"),
-        ended_at=data.get("ended_at"),
-    )