abysshub 0.0.1.dev7__tar.gz → 0.0.1.dev9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- abysshub-0.0.1.dev9/PKG-INFO +92 -0
- abysshub-0.0.1.dev9/README.md +75 -0
- {abysshub-0.0.1.dev7 → abysshub-0.0.1.dev9}/pyproject.toml +2 -2
- abysshub-0.0.1.dev9/src/abysshub/__init__.py +7 -0
- abysshub-0.0.1.dev9/src/abysshub/_client.py +269 -0
- abysshub-0.0.1.dev9/src/abysshub/_error.py +45 -0
- abysshub-0.0.1.dev9/src/abysshub/_files.py +94 -0
- abysshub-0.0.1.dev9/src/abysshub/_http.py +44 -0
- abysshub-0.0.1.dev9/src/abysshub/_run.py +130 -0
- abysshub-0.0.1.dev9/tests/caller.py +43 -0
- abysshub-0.0.1.dev9/tests/fake_server.py +209 -0
- abysshub-0.0.1.dev9/tests/test_client.py +41 -0
- abysshub-0.0.1.dev9/tests/test_files.py +117 -0
- abysshub-0.0.1.dev9/tests/test_http.py +59 -0
- abysshub-0.0.1.dev9/tests/test_replay.py +80 -0
- abysshub-0.0.1.dev7/PKG-INFO +0 -21
- abysshub-0.0.1.dev7/README.md +0 -5
- abysshub-0.0.1.dev7/src/abysshub/__init__.py +0 -1
- {abysshub-0.0.1.dev7 → abysshub-0.0.1.dev9}/.gitignore +0 -0
- {abysshub-0.0.1.dev7 → abysshub-0.0.1.dev9}/LICENSE +0 -0
- {abysshub-0.0.1.dev7 → abysshub-0.0.1.dev9}/src/abysshub/py.typed +0 -0
- {abysshub-0.0.1.dev7 → abysshub-0.0.1.dev9}/tests/test_package.py +0 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: abysshub
|
|
3
|
+
Version: 0.0.1.dev9
|
|
4
|
+
Summary: The official Python library for the Abyss API.
|
|
5
|
+
Project-URL: Homepage, https://abysshub.com
|
|
6
|
+
Project-URL: Source, https://github.com/Abysshub/abysshub-sdk
|
|
7
|
+
Project-URL: Issues, https://github.com/Abysshub/abysshub-sdk/issues
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Typing :: Typed
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Requires-Dist: httpx<1,>=0.27
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# abysshub
|
|
19
|
+
|
|
20
|
+
The official Python library for the [Abyss](https://abysshub.com) API.
|
|
21
|
+
|
|
22
|
+
Pre-release: the Abyss API opens soon.
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from abysshub import Abyss, AbyssError
|
|
26
|
+
|
|
27
|
+
abyss = Abyss() # reads ABYSS_API_KEY
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
run = abyss.run("widget_131", {"amount": 250000})
|
|
31
|
+
print(run.result)
|
|
32
|
+
except AbyssError as error:
|
|
33
|
+
print(error.code, error.param, error.request_id, error.run)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`run()` presses the Widget and waits for the run to end, however long it takes, then returns the succeeded run. A refusal or a failed run raises `AbyssError`.
|
|
37
|
+
|
|
38
|
+
It runs on Python 3.10+, with one dependency, `httpx`.
|
|
39
|
+
|
|
40
|
+
## The client
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
abyss = Abyss(api_key=None, base_url=None, max_retries=2)
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
- **`api_key`:** defaults to `ABYSS_API_KEY`. Without a key, `Abyss()` raises `AbyssError` with `code == "invalid_api_key"`.
|
|
47
|
+
- **`base_url`:** defaults to `ABYSS_BASE_URL`, else `https://api.abysshub.com`. A key starting `abyss_sk_dev_` goes to `https://api.dev.abysshub.com`.
|
|
48
|
+
|
|
49
|
+
`abyss.close()` closes its connections; `with Abyss() as abyss:` does it for you.
|
|
50
|
+
|
|
51
|
+
## Running a Widget
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
run = abyss.run(widget, input, max_price=None, idempotency_key=None)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
- **`widget`** is `widget_<id>`, or an unlisted Widget's private token.
|
|
58
|
+
- **`input`** is the Widget's Fields by name, as a `dict`.
|
|
59
|
+
- **`max_price`:** the most the run may cost, in Byssium. A higher price is refused with `price_above_max`, and nothing is charged.
|
|
60
|
+
- **`idempotency_key`:** each call presses with a fresh UUID `Idempotency-Key` unless you give one.
|
|
61
|
+
|
|
62
|
+
The press holds until the run ends. When the hold ends early (a deploy cuts it, the API's holds are all taken, or an hour passes), `run()` re-attaches with `GET <Location>?wait=true` until the run ends. Re-attaching has no limit and is not a retry. Connecting has 10 s, and 30 s of silence counts as a cut.
|
|
63
|
+
|
|
64
|
+
A **run** is a dataclass with `/v1`'s fields: `id`, `widget`, `status` (`queued`, `running`, `succeeded` or `failed`), `price`, `result` (the Widget's JSON, a `dict` when it wrote an object), `output_files`, `error`, `created_at`, `started_at` and `ended_at`.
|
|
65
|
+
|
|
66
|
+
## Files
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from abysshub import Abyss, file
|
|
70
|
+
|
|
71
|
+
run = abyss.run("widget_1660", {
|
|
72
|
+
"report": file("report.pdf"),
|
|
73
|
+
"data": [file("jan.csv"), file(csv_bytes, filename="feb.csv"), "https://example.com/sales/mar.csv"],
|
|
74
|
+
})
|
|
75
|
+
paths = run.save("output") # [Path("output/charts/a.png"), Path("output/output.json")]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
- **`file(path)`** is a file on disk, read when it is uploaded. A file Field also takes a `Path`, an open file (`open("report.pdf", "rb")`) or `bytes`. `file(data, filename=...)` names data. A file without a name uploads under its Field's name.
|
|
79
|
+
- **A string passes through** as an `https` URL or an upload id. A multi-file Field takes a list, which may mix all three kinds.
|
|
80
|
+
- **Before the press,** each file is uploaded with `POST /v1/widgets/{widget}/uploads`, then sent to storage, in parallel, and its id goes in the input. An upload refusal (`held_input_*`) raises before any press. A press that drops before any headers is sent again with the same ids; it never uploads again.
|
|
81
|
+
- **`run.save(dir)`** writes every output file under `dir` at its `path`, making folders, and returns the paths it wrote.
|
|
82
|
+
- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`.
|
|
83
|
+
|
|
84
|
+
## Errors
|
|
85
|
+
|
|
86
|
+
A refusal or a failed run raises one `AbyssError`. Branch on `code`:
|
|
87
|
+
|
|
88
|
+
- **`code`, `message`, `param`, `doc_url`, `request_id`:** from `/v1`'s error body. A failed run's `code` is `widget_fault`, `platform_fault` or `budget_exceeded`.
|
|
89
|
+
- **`status`:** the HTTP status of a refusal; `None` for a failed run.
|
|
90
|
+
- **`run`:** the failed run; `None` for a refusal.
|
|
91
|
+
|
|
92
|
+
The library's own codes: `invalid_api_key` (no key), `connection_error` (the connection dropped before the API answered) and `unexpected_response` (an answer that is not `/v1`'s).
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# abysshub
|
|
2
|
+
|
|
3
|
+
The official Python library for the [Abyss](https://abysshub.com) API.
|
|
4
|
+
|
|
5
|
+
Pre-release: the Abyss API opens soon.
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from abysshub import Abyss, AbyssError
|
|
9
|
+
|
|
10
|
+
abyss = Abyss() # reads ABYSS_API_KEY
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
run = abyss.run("widget_131", {"amount": 250000})
|
|
14
|
+
print(run.result)
|
|
15
|
+
except AbyssError as error:
|
|
16
|
+
print(error.code, error.param, error.request_id, error.run)
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
`run()` presses the Widget and waits for the run to end, however long it takes, then returns the succeeded run. A refusal or a failed run raises `AbyssError`.
|
|
20
|
+
|
|
21
|
+
It runs on Python 3.10+, with one dependency, `httpx`.
|
|
22
|
+
|
|
23
|
+
## The client
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
abyss = Abyss(api_key=None, base_url=None, max_retries=2)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- **`api_key`:** defaults to `ABYSS_API_KEY`. Without a key, `Abyss()` raises `AbyssError` with `code == "invalid_api_key"`.
|
|
30
|
+
- **`base_url`:** defaults to `ABYSS_BASE_URL`, else `https://api.abysshub.com`. A key starting `abyss_sk_dev_` goes to `https://api.dev.abysshub.com`.
|
|
31
|
+
|
|
32
|
+
`abyss.close()` closes its connections; `with Abyss() as abyss:` does it for you.
|
|
33
|
+
|
|
34
|
+
## Running a Widget
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
run = abyss.run(widget, input, max_price=None, idempotency_key=None)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
- **`widget`** is `widget_<id>`, or an unlisted Widget's private token.
|
|
41
|
+
- **`input`** is the Widget's Fields by name, as a `dict`.
|
|
42
|
+
- **`max_price`:** the most the run may cost, in Byssium. A higher price is refused with `price_above_max`, and nothing is charged.
|
|
43
|
+
- **`idempotency_key`:** each call presses with a fresh UUID `Idempotency-Key` unless you give one.
|
|
44
|
+
|
|
45
|
+
The press holds until the run ends. When the hold ends early (a deploy cuts it, the API's holds are all taken, or an hour passes), `run()` re-attaches with `GET <Location>?wait=true` until the run ends. Re-attaching has no limit and is not a retry. Connecting has 10 s, and 30 s of silence counts as a cut.
|
|
46
|
+
|
|
47
|
+
A **run** is a dataclass with `/v1`'s fields: `id`, `widget`, `status` (`queued`, `running`, `succeeded` or `failed`), `price`, `result` (the Widget's JSON, a `dict` when it wrote an object), `output_files`, `error`, `created_at`, `started_at` and `ended_at`.
|
|
48
|
+
|
|
49
|
+
## Files
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from abysshub import Abyss, file
|
|
53
|
+
|
|
54
|
+
run = abyss.run("widget_1660", {
|
|
55
|
+
"report": file("report.pdf"),
|
|
56
|
+
"data": [file("jan.csv"), file(csv_bytes, filename="feb.csv"), "https://example.com/sales/mar.csv"],
|
|
57
|
+
})
|
|
58
|
+
paths = run.save("output") # [Path("output/charts/a.png"), Path("output/output.json")]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
- **`file(path)`** is a file on disk, read when it is uploaded. A file Field also takes a `Path`, an open file (`open("report.pdf", "rb")`) or `bytes`. `file(data, filename=...)` names data. A file without a name uploads under its Field's name.
|
|
62
|
+
- **A string passes through** as an `https` URL or an upload id. A multi-file Field takes a list, which may mix all three kinds.
|
|
63
|
+
- **Before the press,** each file is uploaded with `POST /v1/widgets/{widget}/uploads`, then sent to storage, in parallel, and its id goes in the input. An upload refusal (`held_input_*`) raises before any press. A press that drops before any headers is sent again with the same ids; it never uploads again.
|
|
64
|
+
- **`run.save(dir)`** writes every output file under `dir` at its `path`, making folders, and returns the paths it wrote.
|
|
65
|
+
- **Each of `run.output_files`** has `f.save(path)` and `f.read()`, which returns `bytes`. An expired URL is refreshed once, by reading the run again with `GET /v1/runs/{id}`.
|
|
66
|
+
|
|
67
|
+
## Errors
|
|
68
|
+
|
|
69
|
+
A refusal or a failed run raises one `AbyssError`. Branch on `code`:
|
|
70
|
+
|
|
71
|
+
- **`code`, `message`, `param`, `doc_url`, `request_id`:** from `/v1`'s error body. A failed run's `code` is `widget_fault`, `platform_fault` or `budget_exceeded`.
|
|
72
|
+
- **`status`:** the HTTP status of a refusal; `None` for a failed run.
|
|
73
|
+
- **`run`:** the failed run; `None` for a refusal.
|
|
74
|
+
|
|
75
|
+
The library's own codes: `invalid_api_key` (no key), `connection_error` (the connection dropped before the API answered) and `unexpected_response` (an answer that is not `/v1`'s).
|
|
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "abysshub"
|
|
7
|
-
version = "0.0.1.
|
|
7
|
+
version = "0.0.1.dev9"
|
|
8
8
|
description = "The official Python library for the Abyss API."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
11
11
|
license-files = ["LICENSE"]
|
|
12
12
|
requires-python = ">=3.10"
|
|
13
|
-
dependencies = []
|
|
13
|
+
dependencies = ["httpx>=0.27,<1"]
|
|
14
14
|
classifiers = [
|
|
15
15
|
"Programming Language :: Python :: 3",
|
|
16
16
|
"Typing :: Typed",
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""The official Python library for the Abyss API."""
|
|
2
|
+
from ._client import Abyss
|
|
3
|
+
from ._error import AbyssError
|
|
4
|
+
from ._files import FileData, InputFile, file
|
|
5
|
+
from ._run import Run, RunError, RunFile, RunStatus
|
|
6
|
+
|
|
7
|
+
__all__ = ["Abyss", "AbyssError", "FileData", "InputFile", "Run", "RunError", "RunFile", "RunStatus", "file"]
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import time
|
|
6
|
+
import uuid
|
|
7
|
+
from collections.abc import Callable, Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from types import TracebackType
|
|
10
|
+
from typing import Any
|
|
11
|
+
from urllib.parse import quote
|
|
12
|
+
|
|
13
|
+
import httpx
|
|
14
|
+
|
|
15
|
+
from ._error import AbyssError
|
|
16
|
+
from ._files import upload_files
|
|
17
|
+
from ._http import request, timeouts
|
|
18
|
+
from ._run import Run, RunError, parse_run
|
|
19
|
+
|
|
20
|
+
API = "https://api.abysshub.com"
|
|
21
|
+
DEV_API = "https://api.dev.abysshub.com"
|
|
22
|
+
DEV_KEY_PREFIX = "abyss_sk_dev_"
|
|
23
|
+
|
|
24
|
+
REATTACH_PAUSE = 1.0
|
|
25
|
+
"""Seconds between two re-attaches that both ended without the run's ending."""
|
|
26
|
+
|
|
27
|
+
RETRY_PAUSE = 0.5
|
|
28
|
+
"""Seconds before the first retry after a drop; the pause doubles with each retry."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class _Reply:
|
|
33
|
+
body: Any
|
|
34
|
+
run: dict[str, Any] | None
|
|
35
|
+
cut: bool
|
|
36
|
+
"""Whether the connection was cut, or went silent, after the headers."""
|
|
37
|
+
location: str | None
|
|
38
|
+
request_id: str | None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Abyss:
|
|
42
|
+
"""A client for the Abyss API.
|
|
43
|
+
|
|
44
|
+
The key defaults to ``ABYSS_API_KEY``, and the address to ``ABYSS_BASE_URL``,
|
|
45
|
+
else the address the key belongs to.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
base_url: str
|
|
49
|
+
max_retries: int
|
|
50
|
+
"""How many times a request is sent again after a drop before any headers, a 429, a 409 or a 5xx."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, api_key: str | None = None, base_url: str | None = None, max_retries: int = 2) -> None:
|
|
53
|
+
if api_key is None:
|
|
54
|
+
api_key = os.environ.get("ABYSS_API_KEY")
|
|
55
|
+
if not api_key:
|
|
56
|
+
raise AbyssError("invalid_api_key", "No Abyss API key: pass api_key, or set ABYSS_API_KEY.")
|
|
57
|
+
self._api_key = api_key
|
|
58
|
+
if base_url is None:
|
|
59
|
+
base_url = os.environ.get("ABYSS_BASE_URL") or (DEV_API if api_key.startswith(DEV_KEY_PREFIX) else API)
|
|
60
|
+
self.base_url = base_url.rstrip("/")
|
|
61
|
+
self.max_retries = max_retries
|
|
62
|
+
self._http = httpx.Client(timeout=timeouts())
|
|
63
|
+
|
|
64
|
+
def close(self) -> None:
|
|
65
|
+
"""Closes the client's connections."""
|
|
66
|
+
self._http.close()
|
|
67
|
+
|
|
68
|
+
def __enter__(self) -> Abyss:
|
|
69
|
+
return self
|
|
70
|
+
|
|
71
|
+
def __exit__(
|
|
72
|
+
self,
|
|
73
|
+
exc_type: type[BaseException] | None,
|
|
74
|
+
exc: BaseException | None,
|
|
75
|
+
tb: TracebackType | None,
|
|
76
|
+
) -> None:
|
|
77
|
+
self.close()
|
|
78
|
+
|
|
79
|
+
def run(
|
|
80
|
+
self,
|
|
81
|
+
widget: str,
|
|
82
|
+
input: Mapping[str, Any],
|
|
83
|
+
*,
|
|
84
|
+
max_price: float | None = None,
|
|
85
|
+
idempotency_key: str | None = None,
|
|
86
|
+
) -> Run:
|
|
87
|
+
"""Uploads the input's files, presses a Widget and waits for the run's ending.
|
|
88
|
+
|
|
89
|
+
A file Field takes ``file(path)``, a ``Path``, an open file or ``bytes``, or a
|
|
90
|
+
list of them; a string passes through as an ``https`` URL or an upload id.
|
|
91
|
+
Re-attaches through ``Location`` whenever the hold ends early. Returns the
|
|
92
|
+
succeeded run; raises ``AbyssError`` on a refusal or a failed run.
|
|
93
|
+
"""
|
|
94
|
+
press = self._press(widget, input, max_price, idempotency_key)
|
|
95
|
+
reply = press if _has_ended(press) else self._hold(self._wait_url(press), _has_ended)
|
|
96
|
+
assert reply.run is not None
|
|
97
|
+
run = self._run(reply.run)
|
|
98
|
+
if run.status == "failed":
|
|
99
|
+
error = run.error or RunError("platform_fault", "The run failed.")
|
|
100
|
+
raise AbyssError(error.code, error.message, request_id=reply.request_id, run=run)
|
|
101
|
+
return run
|
|
102
|
+
|
|
103
|
+
def _press(
|
|
104
|
+
self,
|
|
105
|
+
widget: str,
|
|
106
|
+
input: Mapping[str, Any],
|
|
107
|
+
max_price: float | None,
|
|
108
|
+
idempotency_key: str | None,
|
|
109
|
+
) -> _Reply:
|
|
110
|
+
"""Uploads the input's files and sends the press. Its retries send the same body
|
|
111
|
+
and ``Idempotency-Key``, so the uploads are never redone."""
|
|
112
|
+
widget_url = self._widget_url(widget)
|
|
113
|
+
uploaded = upload_files(input, lambda field, filename, data: self._upload(widget_url, field, filename, data))
|
|
114
|
+
body: dict[str, Any] = {"input": uploaded}
|
|
115
|
+
if max_price is not None:
|
|
116
|
+
body["max_price"] = max_price
|
|
117
|
+
headers = {
|
|
118
|
+
"Content-Type": "application/json",
|
|
119
|
+
"Idempotency-Key": idempotency_key if idempotency_key is not None else str(uuid.uuid4()),
|
|
120
|
+
}
|
|
121
|
+
return self._send("POST", f"{widget_url}/runs", headers, json.dumps(body).encode())
|
|
122
|
+
|
|
123
|
+
def _upload(self, widget_url: str, field: str, filename: str, data: bytes) -> str:
|
|
124
|
+
"""Asks for an upload, sends the file to storage and answers the upload's id."""
|
|
125
|
+
grant = self._send(
|
|
126
|
+
"POST",
|
|
127
|
+
f"{widget_url}/uploads",
|
|
128
|
+
{"Content-Type": "application/json"},
|
|
129
|
+
json.dumps({"field": field, "filename": filename, "size": len(data)}).encode(),
|
|
130
|
+
)
|
|
131
|
+
if not _is_upload(grant.body):
|
|
132
|
+
raise AbyssError(
|
|
133
|
+
"unexpected_response",
|
|
134
|
+
"The upload answered without an id and a form.",
|
|
135
|
+
request_id=grant.request_id,
|
|
136
|
+
)
|
|
137
|
+
upload = grant.body["upload"]
|
|
138
|
+
try:
|
|
139
|
+
stored = self._http.post(
|
|
140
|
+
upload["url"],
|
|
141
|
+
data={name: str(value) for name, value in upload["fields"].items()},
|
|
142
|
+
files={"file": (filename, data)},
|
|
143
|
+
)
|
|
144
|
+
except httpx.TransportError as error:
|
|
145
|
+
raise AbyssError(
|
|
146
|
+
"connection_error",
|
|
147
|
+
f"The connection to storage dropped during the upload of {filename}.",
|
|
148
|
+
param=field,
|
|
149
|
+
) from error
|
|
150
|
+
if not stored.is_success:
|
|
151
|
+
raise AbyssError(
|
|
152
|
+
"unexpected_response",
|
|
153
|
+
f"Storage answered {stored.status_code} to the upload of {filename}.",
|
|
154
|
+
status=stored.status_code,
|
|
155
|
+
param=field,
|
|
156
|
+
)
|
|
157
|
+
return str(grant.body["id"])
|
|
158
|
+
|
|
159
|
+
def _download(self, url: str) -> tuple[int, bytes]:
|
|
160
|
+
"""Downloads an output file from storage, without the API key."""
|
|
161
|
+
try:
|
|
162
|
+
response = self._http.get(url)
|
|
163
|
+
except httpx.TransportError as error:
|
|
164
|
+
raise AbyssError("connection_error", "The connection to storage dropped during a download.") from error
|
|
165
|
+
return response.status_code, response.content
|
|
166
|
+
|
|
167
|
+
def _reread(self, id: str) -> dict[str, Any]:
|
|
168
|
+
"""Reads a run with ``GET /v1/runs/{id}``. A cut read is read again."""
|
|
169
|
+
reply = self._hold(f"{self.base_url}/v1/runs/{quote(id, safe='')}", lambda reply: not reply.cut)
|
|
170
|
+
if reply.run is None:
|
|
171
|
+
raise AbyssError("unexpected_response", "The read answered without a run.", request_id=reply.request_id)
|
|
172
|
+
return reply.run
|
|
173
|
+
|
|
174
|
+
def _run(self, data: dict[str, Any]) -> Run:
|
|
175
|
+
return parse_run(data, self._reread, self._download)
|
|
176
|
+
|
|
177
|
+
def _hold(self, url: str, done: Callable[[_Reply], bool]) -> _Reply:
|
|
178
|
+
"""Reads ``url`` until ``done``. Re-attaching has no limit and is not a retry."""
|
|
179
|
+
reply = self._send("GET", url)
|
|
180
|
+
while not done(reply):
|
|
181
|
+
time.sleep(REATTACH_PAUSE)
|
|
182
|
+
reply = self._send("GET", url)
|
|
183
|
+
return reply
|
|
184
|
+
|
|
185
|
+
def _widget_url(self, widget: str) -> str:
|
|
186
|
+
return f"{self.base_url}/v1/widgets/{quote(widget, safe='')}"
|
|
187
|
+
|
|
188
|
+
def _wait_url(self, reply: _Reply) -> str:
|
|
189
|
+
location = reply.location or (f"/v1/runs/{quote(reply.run['id'], safe='')}" if reply.run else None)
|
|
190
|
+
if not location:
|
|
191
|
+
raise AbyssError(
|
|
192
|
+
"unexpected_response",
|
|
193
|
+
"The press answered without a Location to read the run at.",
|
|
194
|
+
request_id=reply.request_id,
|
|
195
|
+
)
|
|
196
|
+
return str(httpx.URL(f"{self.base_url}/").join(location).copy_set_param("wait", "true"))
|
|
197
|
+
|
|
198
|
+
def _send(self, method: str, url: str, headers: dict[str, str] | None = None, content: bytes | None = None) -> _Reply:
|
|
199
|
+
"""Sends one request to `/v1`. A drop before any headers is sent again, at most
|
|
200
|
+
``max_retries`` times; every refusal raises at once."""
|
|
201
|
+
sent_headers = {"Authorization": f"Bearer {self._api_key}", "Accept": "application/json", **(headers or {})}
|
|
202
|
+
attempt = 0
|
|
203
|
+
while True:
|
|
204
|
+
try:
|
|
205
|
+
answer = request(self._http, method, url, sent_headers, content)
|
|
206
|
+
break
|
|
207
|
+
except httpx.TransportError as error:
|
|
208
|
+
if attempt >= self.max_retries:
|
|
209
|
+
raise AbyssError(
|
|
210
|
+
"connection_error",
|
|
211
|
+
"The connection dropped, or timed out, before the API answered.",
|
|
212
|
+
) from error
|
|
213
|
+
time.sleep(RETRY_PAUSE * 2**attempt)
|
|
214
|
+
attempt += 1
|
|
215
|
+
request_id = answer.headers.get("request-id")
|
|
216
|
+
parsed = _parse(answer.text)
|
|
217
|
+
if answer.status in (200, 202):
|
|
218
|
+
run = parsed if _is_run(parsed) else None
|
|
219
|
+
return _Reply(parsed, run, answer.text is None, answer.headers.get("location"), request_id)
|
|
220
|
+
raise _refusal(answer.status, parsed, request_id)
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _refusal(status: int, body: Any, request_id: str | None) -> AbyssError:
|
|
224
|
+
error = body.get("error") if isinstance(body, dict) else None
|
|
225
|
+
if not isinstance(error, dict) or not isinstance(error.get("code"), str):
|
|
226
|
+
return AbyssError(
|
|
227
|
+
"unexpected_response",
|
|
228
|
+
f"The API answered {status} without an error body.",
|
|
229
|
+
status=status,
|
|
230
|
+
request_id=request_id,
|
|
231
|
+
)
|
|
232
|
+
body_request_id = body.get("request_id")
|
|
233
|
+
return AbyssError(
|
|
234
|
+
error["code"],
|
|
235
|
+
_text(error.get("message")) or error["code"],
|
|
236
|
+
status=status,
|
|
237
|
+
param=_text(error.get("param")),
|
|
238
|
+
doc_url=_text(error.get("doc_url")),
|
|
239
|
+
request_id=body_request_id if isinstance(body_request_id, str) else request_id,
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _text(value: Any) -> str | None:
|
|
244
|
+
return value if isinstance(value, str) else None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _parse(text: str | None) -> Any:
|
|
248
|
+
if text is None:
|
|
249
|
+
return None
|
|
250
|
+
try:
|
|
251
|
+
return json.loads(text)
|
|
252
|
+
except ValueError:
|
|
253
|
+
return None
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _is_run(value: Any) -> bool:
|
|
257
|
+
return isinstance(value, dict) and isinstance(value.get("id"), str) and isinstance(value.get("status"), str)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _is_upload(value: Any) -> bool:
|
|
261
|
+
if not isinstance(value, dict) or not isinstance(value.get("id"), str):
|
|
262
|
+
return False
|
|
263
|
+
upload = value.get("upload")
|
|
264
|
+
return isinstance(upload, dict) and isinstance(upload.get("url"), str) and isinstance(upload.get("fields"), dict)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _has_ended(reply: _Reply) -> bool:
|
|
268
|
+
"""Whether the reply carries a run that has succeeded or failed."""
|
|
269
|
+
return reply.run is not None and reply.run["status"] in ("succeeded", "failed")
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
if TYPE_CHECKING:
|
|
6
|
+
from ._run import Run
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AbyssError(Exception):
|
|
10
|
+
"""The one error the library raises: a refusal from `/v1`, or a run that failed.
|
|
11
|
+
|
|
12
|
+
Branch on ``code``. A failed run carries the run as ``run``.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
code: str
|
|
16
|
+
message: str
|
|
17
|
+
status: int | None
|
|
18
|
+
"""The HTTP status of a refusal; None for a failed run."""
|
|
19
|
+
param: str | None
|
|
20
|
+
doc_url: str | None
|
|
21
|
+
request_id: str | None
|
|
22
|
+
run: Run | None
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
code: str,
|
|
27
|
+
message: str,
|
|
28
|
+
*,
|
|
29
|
+
status: int | None = None,
|
|
30
|
+
param: str | None = None,
|
|
31
|
+
doc_url: str | None = None,
|
|
32
|
+
request_id: str | None = None,
|
|
33
|
+
run: Run | None = None,
|
|
34
|
+
) -> None:
|
|
35
|
+
super().__init__(message)
|
|
36
|
+
self.code = code
|
|
37
|
+
self.message = message
|
|
38
|
+
self.status = status
|
|
39
|
+
self.param = param
|
|
40
|
+
self.doc_url = doc_url
|
|
41
|
+
self.request_id = request_id
|
|
42
|
+
self.run = run
|
|
43
|
+
|
|
44
|
+
def __repr__(self) -> str:
|
|
45
|
+
return f"AbyssError(code={self.code!r}, message={self.message!r})"
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from collections.abc import Callable, Mapping
|
|
5
|
+
from concurrent.futures import Future, ThreadPoolExecutor
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import IO, Any, Union
|
|
8
|
+
|
|
9
|
+
FileData = Union[bytes, bytearray, memoryview, os.PathLike[str], IO[bytes], IO[str]]
|
|
10
|
+
"""File data: bytes in memory, a ``Path``, or an open file."""
|
|
11
|
+
|
|
12
|
+
MAX_PARALLEL_UPLOADS = 8
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class InputFile:
|
|
16
|
+
"""A file for a file Field: a path, read when it is uploaded, or data. Made by ``file()``."""
|
|
17
|
+
|
|
18
|
+
source: str | FileData
|
|
19
|
+
filename: str | None
|
|
20
|
+
"""The name to upload under. Defaults to the path's or the open file's name, else the Field's name."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, source: str | FileData, filename: str | None = None) -> None:
|
|
23
|
+
self.source = source
|
|
24
|
+
self.filename = filename
|
|
25
|
+
|
|
26
|
+
def __repr__(self) -> str:
|
|
27
|
+
return f"file({self.source!r})"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def file(source: str | FileData, *, filename: str | None = None) -> InputFile:
|
|
31
|
+
"""A file for a file Field. A string is a path, read when the file is uploaded."""
|
|
32
|
+
return InputFile(source, filename)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
Uploader = Callable[[str, str, bytes], str]
|
|
36
|
+
"""Uploads one file for a Field, by its name and content, and answers the upload's id."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def upload_files(input: Mapping[str, Any], upload: Uploader) -> dict[str, Any]:
|
|
40
|
+
"""The input with every file value uploaded, in parallel, and replaced by its upload's id.
|
|
41
|
+
|
|
42
|
+
Strings pass through as an ``https`` URL or an upload id, and a list may mix all
|
|
43
|
+
kinds. The caller's input is never changed.
|
|
44
|
+
"""
|
|
45
|
+
uploaded = dict(input)
|
|
46
|
+
jobs: list[tuple[Callable[[str], None], str, InputFile | FileData]] = []
|
|
47
|
+
for field, value in input.items():
|
|
48
|
+
if is_file(value):
|
|
49
|
+
jobs.append((_setter(uploaded, field), field, value))
|
|
50
|
+
elif isinstance(value, list) and any(is_file(item) for item in value):
|
|
51
|
+
items = list(value)
|
|
52
|
+
uploaded[field] = items
|
|
53
|
+
jobs.extend((_setter(items, index), field, item) for index, item in enumerate(value) if is_file(item))
|
|
54
|
+
if not jobs:
|
|
55
|
+
return uploaded
|
|
56
|
+
|
|
57
|
+
def upload_one(field: str, value: InputFile | FileData) -> str:
|
|
58
|
+
filename, data = _load(value, field)
|
|
59
|
+
return upload(field, filename, data)
|
|
60
|
+
|
|
61
|
+
with ThreadPoolExecutor(max_workers=min(len(jobs), MAX_PARALLEL_UPLOADS)) as pool:
|
|
62
|
+
futures: list[Future[str]] = [pool.submit(upload_one, field, value) for _, field, value in jobs]
|
|
63
|
+
for (put, _, _), future in zip(jobs, futures):
|
|
64
|
+
put(future.result())
|
|
65
|
+
return uploaded
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def is_file(value: Any) -> bool:
|
|
69
|
+
"""Whether the value is a file to upload, rather than a Field's plain value."""
|
|
70
|
+
return isinstance(value, (InputFile, bytes, bytearray, memoryview, os.PathLike)) or callable(
|
|
71
|
+
getattr(value, "read", None)
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _setter(target: Any, at: Any) -> Callable[[str], None]:
|
|
76
|
+
def put(upload_id: str) -> None:
|
|
77
|
+
target[at] = upload_id
|
|
78
|
+
|
|
79
|
+
return put
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _load(value: InputFile | FileData, field: str) -> tuple[str, bytes]:
|
|
83
|
+
"""The file's name and content."""
|
|
84
|
+
given = value if isinstance(value, InputFile) else InputFile(value)
|
|
85
|
+
source = given.source
|
|
86
|
+
if isinstance(source, (str, os.PathLike)):
|
|
87
|
+
path = Path(source)
|
|
88
|
+
return given.filename or path.name, path.read_bytes()
|
|
89
|
+
if isinstance(source, (bytes, bytearray, memoryview)):
|
|
90
|
+
return given.filename or field, bytes(source)
|
|
91
|
+
content = source.read()
|
|
92
|
+
data = content.encode("utf-8") if isinstance(content, str) else bytes(content)
|
|
93
|
+
name = getattr(source, "name", None)
|
|
94
|
+
return given.filename or (Path(name).name if isinstance(name, str) and name else field), data
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
CONNECT_TIMEOUT = 10.0
|
|
8
|
+
"""Seconds to connect."""
|
|
9
|
+
|
|
10
|
+
SILENCE_TIMEOUT = 30.0
|
|
11
|
+
"""Seconds a response may stay silent before it counts as a cut."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def timeouts(connect: float = CONNECT_TIMEOUT, silence: float = SILENCE_TIMEOUT) -> httpx.Timeout:
|
|
15
|
+
"""httpx's timeouts: ``connect`` to connect, and ``silence`` between any two reads."""
|
|
16
|
+
return httpx.Timeout(silence, connect=connect)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Answer:
|
|
21
|
+
status: int
|
|
22
|
+
headers: httpx.Headers
|
|
23
|
+
text: str | None
|
|
24
|
+
"""The whole body, or None when the connection was cut or went silent after the headers."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def request(
|
|
28
|
+
client: httpx.Client,
|
|
29
|
+
method: str,
|
|
30
|
+
url: str,
|
|
31
|
+
headers: dict[str, str],
|
|
32
|
+
content: bytes | None = None,
|
|
33
|
+
) -> Answer:
|
|
34
|
+
"""Sends one request and reads its whole body.
|
|
35
|
+
|
|
36
|
+
A failure before the headers raises ``httpx.TransportError``; a cut after them
|
|
37
|
+
answers ``text=None``, so the caller can re-attach.
|
|
38
|
+
"""
|
|
39
|
+
with client.stream(method, url, headers=headers, content=content) as response:
|
|
40
|
+
try:
|
|
41
|
+
body = b"".join(response.iter_bytes())
|
|
42
|
+
except httpx.TransportError:
|
|
43
|
+
return Answer(response.status_code, response.headers, None)
|
|
44
|
+
return Answer(response.status_code, response.headers, body.decode("utf-8", errors="replace"))
|