wheelmachine 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wheelmachine might be problematic. Click here for more details.
- wheelmachine/__init__.py +230 -0
- wheelmachine/_client.py +550 -0
- wheelmachine/_errors.py +215 -0
- wheelmachine/_http.py +418 -0
- wheelmachine/_login.py +126 -0
- wheelmachine/_models.py +1987 -0
- wheelmachine/_pagination.py +36 -0
- wheelmachine/_resources.py +1323 -0
- wheelmachine/_version.py +3 -0
- wheelmachine/py.typed +0 -0
- wheelmachine-0.1.0.dist-info/METADATA +253 -0
- wheelmachine-0.1.0.dist-info/RECORD +15 -0
- wheelmachine-0.1.0.dist-info/WHEEL +4 -0
- wheelmachine-0.1.0.dist-info/licenses/LICENSE +202 -0
- wheelmachine-0.1.0.dist-info/licenses/NOTICE +4 -0
wheelmachine/_client.py
ADDED
|
@@ -0,0 +1,550 @@
|
|
|
1
|
+
"""The client: the key, the address, and how a call is made, retried and answered."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import random
|
|
8
|
+
import re
|
|
9
|
+
import time
|
|
10
|
+
from datetime import date, datetime, timezone
|
|
11
|
+
from email.utils import parsedate_to_datetime
|
|
12
|
+
from typing import Any, BinaryIO, Callable, Dict, List, Optional, Sequence, Tuple, Union
|
|
13
|
+
from urllib.parse import urlencode, quote
|
|
14
|
+
|
|
15
|
+
from ._errors import APIConnectionError, APITimeoutError, WheelmachineError, error_from_answer
|
|
16
|
+
from ._http import Answer, Body, BytesBody, Failure, OpenBody, Proxy, Target, Transport, is_timeout, proxy_from_environment
|
|
17
|
+
from ._models import (Code, CodeConnection, CodeGrant, FolderEntry, FolderGrant, FolderList, FolderListing, FolderTrash, GpuList, Me, RateLimit,
|
|
18
|
+
Region, ResponseInfo, Usage, UsageLimits, _answered)
|
|
19
|
+
from ._pagination import PageIterator
|
|
20
|
+
from ._resources import (Agents, Apps, Audit, ConfirmCodes, Credit, FileData, Keys, Machines, Models, Orgs, Ports, Projects, Requests, Secrets,
|
|
21
|
+
_file_body)
|
|
22
|
+
from ._version import __version__
|
|
23
|
+
|
|
24
|
+
__all__ = ["Wheelmachine", "DEFAULT_BASE_URL"]
|
|
25
|
+
|
|
26
|
+
DEFAULT_BASE_URL = "https://api.wheelmachine.ai"
|
|
27
|
+
USER_AGENT = "wheelmachine-python/" + __version__
|
|
28
|
+
|
|
29
|
+
#: Methods that do the same however many times they are sent, so a call that lost its connection is sent again.
|
|
30
|
+
#: A POST is sent again only when it carries an Idempotency-Key (the calls that create something), or when sending it
|
|
31
|
+
#: twice does what sending it once does (opening a port: `repeatable=True`).
|
|
32
|
+
_REPEATABLE = frozenset({"GET", "HEAD", "PUT", "PATCH", "DELETE"})
|
|
33
|
+
#: A Retry-After longer than this is not waited out: the error is raised at once, with `retry_after` on it.
|
|
34
|
+
LONGEST_RETRY_AFTER_S = 60.0
|
|
35
|
+
_BACKOFF_FIRST_S = 0.5
|
|
36
|
+
_BACKOFF_MOST_S = 8.0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _Secret:
|
|
40
|
+
"""The API key: kept out of every repr, so it cannot reach a log through this object."""
|
|
41
|
+
|
|
42
|
+
__slots__ = ("_value",)
|
|
43
|
+
|
|
44
|
+
def __init__(self, value: str) -> None:
|
|
45
|
+
self._value = value
|
|
46
|
+
|
|
47
|
+
def value(self) -> str:
|
|
48
|
+
return self._value
|
|
49
|
+
|
|
50
|
+
def __repr__(self) -> str:
|
|
51
|
+
return "<hidden>"
|
|
52
|
+
|
|
53
|
+
__str__ = __repr__
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _retry_after(value: Optional[str]) -> Optional[float]:
|
|
57
|
+
"""Retry-After in seconds: a number of seconds, or an HTTP date."""
|
|
58
|
+
if not value:
|
|
59
|
+
return None
|
|
60
|
+
value = value.strip()
|
|
61
|
+
try:
|
|
62
|
+
seconds = float(value)
|
|
63
|
+
except ValueError:
|
|
64
|
+
try:
|
|
65
|
+
when = parsedate_to_datetime(value)
|
|
66
|
+
except (TypeError, ValueError, IndexError):
|
|
67
|
+
return None
|
|
68
|
+
if when is None:
|
|
69
|
+
return None
|
|
70
|
+
if when.tzinfo is None:
|
|
71
|
+
when = when.replace(tzinfo=timezone.utc)
|
|
72
|
+
seconds = (when - datetime.now(timezone.utc)).total_seconds()
|
|
73
|
+
return max(0.0, seconds)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
_FIELD = re.compile(r"(?:^|;)\s*([a-z]+)=(\d+)")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _rate_limit(state: Optional[str], policy: Optional[str]) -> Optional[RateLimit]:
|
|
80
|
+
"""The RateLimit and RateLimit-Policy headers: `"key";r=599;t=1` and `"key";q=600;w=60`."""
|
|
81
|
+
if not state and not policy:
|
|
82
|
+
return None
|
|
83
|
+
s = dict(_FIELD.findall(state or ""))
|
|
84
|
+
p = dict(_FIELD.findall(policy or ""))
|
|
85
|
+
|
|
86
|
+
def num(d: Dict[str, str], k: str) -> Optional[int]:
|
|
87
|
+
return int(d[k]) if k in d else None
|
|
88
|
+
|
|
89
|
+
return RateLimit(limit=num(p, "q"), window_seconds=num(p, "w"), remaining=num(s, "r"), reset_seconds=num(s, "t"))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class Wheelmachine:
|
|
93
|
+
"""A client of the Wheelmachine API.
|
|
94
|
+
|
|
95
|
+
``api_key`` is read from ``WHEELMACHINE_API_KEY`` and ``base_url`` from ``WHEELMACHINE_BASE_URL`` when they are
|
|
96
|
+
not given. ``timeout`` (seconds) is how long a call may go without a byte coming; calls the API holds longer by
|
|
97
|
+
design (making a machine ready, waiting for a state, running a command) are given the time they need on top.
|
|
98
|
+
Calls refused for a moment (429, 503), and calls that lost their connection, are tried again up to
|
|
99
|
+
``max_retries`` times, with a growing wait or the one the API asked for. ``proxy`` (``http://host:port``) is
|
|
100
|
+
reached with CONNECT; without it, HTTPS_PROXY and NO_PROXY are followed.
|
|
101
|
+
|
|
102
|
+
Safe to share between threads. ``close()``, or a ``with`` block, closes its kept connections.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
machines: Machines
|
|
106
|
+
ports: Ports
|
|
107
|
+
apps: Apps
|
|
108
|
+
projects: Projects
|
|
109
|
+
keys: Keys
|
|
110
|
+
secrets: Secrets
|
|
111
|
+
orgs: Orgs
|
|
112
|
+
requests: Requests
|
|
113
|
+
audit: Audit
|
|
114
|
+
|
|
115
|
+
def __init__(
|
|
116
|
+
self,
|
|
117
|
+
api_key: Optional[str] = None,
|
|
118
|
+
base_url: Optional[str] = None,
|
|
119
|
+
timeout: float = 60.0,
|
|
120
|
+
max_retries: int = 3,
|
|
121
|
+
*,
|
|
122
|
+
proxy: Optional[str] = None,
|
|
123
|
+
) -> None:
|
|
124
|
+
key = (api_key if api_key is not None else os.environ.get("WHEELMACHINE_API_KEY", "")).strip()
|
|
125
|
+
if not key:
|
|
126
|
+
raise WheelmachineError("no API key: pass api_key=... or set WHEELMACHINE_API_KEY")
|
|
127
|
+
if any(c < "!" or c > "~" for c in key):
|
|
128
|
+
# never echoed: it would put the key in a log
|
|
129
|
+
raise WheelmachineError("the API key holds characters an API key cannot hold")
|
|
130
|
+
self._open(key, base_url, timeout, max_retries, proxy)
|
|
131
|
+
|
|
132
|
+
def _open(self, key: str, base_url: Optional[str], timeout: float, max_retries: int, proxy: Optional[str]) -> None:
|
|
133
|
+
"""Everything but the key's own checks: a client signing a program in through the browser has none yet."""
|
|
134
|
+
if not timeout or timeout <= 0:
|
|
135
|
+
raise ValueError("timeout is a number of seconds above 0")
|
|
136
|
+
if max_retries < 0:
|
|
137
|
+
raise ValueError("max_retries cannot be negative")
|
|
138
|
+
self._key = _Secret(key)
|
|
139
|
+
self._target = Target(base_url or os.environ.get("WHEELMACHINE_BASE_URL") or DEFAULT_BASE_URL)
|
|
140
|
+
self.timeout = float(timeout)
|
|
141
|
+
self.max_retries = int(max_retries)
|
|
142
|
+
self._transport = Transport(self._target, Proxy.parse(proxy) if proxy else proxy_from_environment(self._target))
|
|
143
|
+
# seams for tests: how a pause is taken, and the jitter of a backoff
|
|
144
|
+
self._sleep: Callable[[float], None] = time.sleep
|
|
145
|
+
self._jitter: Callable[[], float] = random.random
|
|
146
|
+
self.machines = Machines(self)
|
|
147
|
+
self.ports = Ports(self)
|
|
148
|
+
self.apps = Apps(self)
|
|
149
|
+
self.projects = Projects(self)
|
|
150
|
+
self.keys = Keys(self)
|
|
151
|
+
self.confirm_codes = ConfirmCodes(self)
|
|
152
|
+
self.secrets = Secrets(self)
|
|
153
|
+
self.models = Models(self)
|
|
154
|
+
self.agents = Agents(self)
|
|
155
|
+
self.orgs = Orgs(self)
|
|
156
|
+
self.requests = Requests(self)
|
|
157
|
+
self.audit = Audit(self)
|
|
158
|
+
self.credit = Credit(self)
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def base_url(self) -> str:
|
|
162
|
+
return self._target.url
|
|
163
|
+
|
|
164
|
+
def __repr__(self) -> str:
|
|
165
|
+
return "Wheelmachine(base_url={!r})".format(self.base_url)
|
|
166
|
+
|
|
167
|
+
def close(self) -> None:
|
|
168
|
+
"""Closes the kept connections. The client can still be used: it opens new ones."""
|
|
169
|
+
self._transport.close()
|
|
170
|
+
|
|
171
|
+
def __enter__(self) -> "Wheelmachine":
|
|
172
|
+
return self
|
|
173
|
+
|
|
174
|
+
def __exit__(self, *exc: Any) -> None:
|
|
175
|
+
self.close()
|
|
176
|
+
|
|
177
|
+
# --- the account and the catalogue ----------------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
def me(self) -> Me:
|
|
180
|
+
"""The account and key this client calls with, and whether the account can spend now. In an organisation, also
|
|
181
|
+
the person who made the key, their role and teams there, and their own limits there."""
|
|
182
|
+
data, info = self._json("GET", "/v1/me")
|
|
183
|
+
return _answered(Me.from_dict(data or {}), info)
|
|
184
|
+
|
|
185
|
+
def regions(self) -> List[Region]:
|
|
186
|
+
"""Places a machine can be asked to run near (``create(region=...)``)."""
|
|
187
|
+
data, _ = self._json("GET", "/v1/regions")
|
|
188
|
+
return [Region.from_dict(r) for r in _list_of(data)]
|
|
189
|
+
|
|
190
|
+
def gpus(self) -> GpuList:
|
|
191
|
+
"""GPUs on offer now, where they are and what an hour costs."""
|
|
192
|
+
data, info = self._json("GET", "/v1/gpus")
|
|
193
|
+
return _answered(GpuList.from_dict(data or {}), info)
|
|
194
|
+
|
|
195
|
+
def usage(self, *, month: Union[str, date, None] = None, project: Optional[str] = None) -> Usage:
|
|
196
|
+
"""What each machine ran for and cost in a month (``"2026-10"``; this month when left out)."""
|
|
197
|
+
query: List[Tuple[str, str]] = []
|
|
198
|
+
if month is not None:
|
|
199
|
+
query.append(("month", month.strftime("%Y-%m") if isinstance(month, date) else month))
|
|
200
|
+
if project is not None:
|
|
201
|
+
query.append(("project", project))
|
|
202
|
+
data, info = self._json("GET", "/v1/usage", query=query)
|
|
203
|
+
return _answered(Usage.from_dict(data or {}), info)
|
|
204
|
+
|
|
205
|
+
def limits(self) -> UsageLimits:
|
|
206
|
+
"""Every limit for the month that holds the account, as you may see it, and how near each is."""
|
|
207
|
+
data, info = self._json("GET", "/v1/usage/limits")
|
|
208
|
+
return _answered(UsageLimits.from_dict(data or {}), info)
|
|
209
|
+
|
|
210
|
+
def set_month_limit(self, usd: Optional[float]) -> UsageLimits:
|
|
211
|
+
"""Set the account's own limit for the month, in dollars, or take it away with None: in your own account, with
|
|
212
|
+
a key that holds ``cloud:*``; in an organisation, an owner's."""
|
|
213
|
+
data, info = self._json("PUT", "/v1/usage/limits/account", payload={"month_cap_usd": usd})
|
|
214
|
+
return _answered(UsageLimits.from_dict(data or {}), info)
|
|
215
|
+
|
|
216
|
+
# --- team folders --------------------------------------------------------------------------------------------
|
|
217
|
+
|
|
218
|
+
def folders(self) -> FolderList:
|
|
219
|
+
"""The folders you see: your own (My files) first, then your teams'. ``on`` is False where folders are not set
|
|
220
|
+
up here yet."""
|
|
221
|
+
data, info = self._json("GET", "/v1/folders")
|
|
222
|
+
return _answered(FolderList.from_dict(data or {}), info)
|
|
223
|
+
|
|
224
|
+
def list_folder(self, folder: str, path: str = "", *, after: Optional[str] = None, limit: Optional[int] = None) -> FolderListing:
|
|
225
|
+
"""One page of what is in a folder of a folder (its top for ``""``): folders first, then files, each by name -
|
|
226
|
+
``limit`` of them at most (1 to 1000), from the ``next`` of the page before (``after``). ``folder_entries``
|
|
227
|
+
walks them all."""
|
|
228
|
+
query = [("path", path)]
|
|
229
|
+
if after:
|
|
230
|
+
query.append(("after", after))
|
|
231
|
+
if limit is not None:
|
|
232
|
+
query.append(("limit", str(limit)))
|
|
233
|
+
data, info = self._json("GET", "/v1/folders/" + quote(folder, safe="") + "/list", query=query)
|
|
234
|
+
return _answered(FolderListing.from_dict(data or {}), info)
|
|
235
|
+
|
|
236
|
+
def folder_entries(self, folder: str, path: str = "", *, limit: Optional[int] = None) -> PageIterator[FolderEntry]:
|
|
237
|
+
"""Everything in a folder of a folder, folders first, page after page as it is iterated."""
|
|
238
|
+
|
|
239
|
+
def page(after: Optional[str]) -> Tuple[List[FolderEntry], Optional[str]]:
|
|
240
|
+
listing = self.list_folder(folder, path, after=after, limit=limit)
|
|
241
|
+
return listing.entries, listing.next
|
|
242
|
+
|
|
243
|
+
return PageIterator(page)
|
|
244
|
+
|
|
245
|
+
def read_folder_file(self, folder: str, path: str, *, timeout: Optional[float] = None) -> bytes:
|
|
246
|
+
"""A file of a folder, its bytes."""
|
|
247
|
+
answer = self._send("GET", "/v1/folders/" + quote(folder, safe="") + "/files", query=[("path", path)],
|
|
248
|
+
accept="application/octet-stream", timeout=timeout, buffer=True)
|
|
249
|
+
return answer.content
|
|
250
|
+
|
|
251
|
+
def write_folder_file(self, folder: str, path: str, data: FileData, *, content_length: Optional[int] = None,
|
|
252
|
+
timeout: Optional[float] = None) -> FolderEntry:
|
|
253
|
+
"""Writes a file of a folder, in the place of the one there, making the folders on the way: ``data`` is bytes, a
|
|
254
|
+
``pathlib.Path`` to a file here, or a binary file object (one that cannot seek needs ``content_length``). At most
|
|
255
|
+
5 GB a file; the account keeps 25 GB in its folders for now, the Trash included."""
|
|
256
|
+
answer = self._send("PUT", "/v1/folders/" + quote(folder, safe="") + "/files", query=[("path", path)],
|
|
257
|
+
body=_file_body(data, content_length), timeout=timeout, buffer=True)
|
|
258
|
+
return FolderEntry.from_dict(json.loads(answer.content or b"{}"))
|
|
259
|
+
|
|
260
|
+
def delete_folder_file(self, folder: str, path: str) -> FolderTrash:
|
|
261
|
+
"""Puts a file or folder of a folder in its Trash, from which it can be put back for 30 days."""
|
|
262
|
+
data, info = self._json("DELETE", "/v1/folders/" + quote(folder, safe="") + "/files", query=[("path", path)])
|
|
263
|
+
return _answered(FolderTrash.from_dict(data or {}), info)
|
|
264
|
+
|
|
265
|
+
def move_folder_item(self, folder: str, src: str, dst: str, *, to_folder: Optional[str] = None) -> None:
|
|
266
|
+
"""Moves or renames a file or folder of a folder - into ``to_folder``, another folder you edit, too."""
|
|
267
|
+
payload: Dict[str, Any] = {"from": src, "to": dst}
|
|
268
|
+
if to_folder:
|
|
269
|
+
payload["to_folder"] = to_folder
|
|
270
|
+
self._json("POST", "/v1/folders/" + quote(folder, safe="") + "/move", payload=payload)
|
|
271
|
+
|
|
272
|
+
def give_team_folder(self, project: str, folder: str, access: str = "view") -> FolderGrant:
|
|
273
|
+
"""Gives a team a folder, to ``view`` or to ``edit``: its full managers, the organisation's owners and admins, or a
|
|
274
|
+
person in their own account."""
|
|
275
|
+
data, info = self._json("PUT", "/v1/projects/" + quote(project, safe="") + "/folders/" + quote(folder, safe=""),
|
|
276
|
+
payload={"access": access})
|
|
277
|
+
return _answered(FolderGrant.from_dict(data or {}), info)
|
|
278
|
+
|
|
279
|
+
def take_team_folder(self, project: str, folder: str) -> None:
|
|
280
|
+
"""Takes a folder from a team: its machines lose it at once."""
|
|
281
|
+
self._json("DELETE", "/v1/projects/" + quote(project, safe="") + "/folders/" + quote(folder, safe=""))
|
|
282
|
+
|
|
283
|
+
# --- code access ---------------------------------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
def code(self) -> Code:
|
|
286
|
+
"""The GitHub and GitLab the account connects, and the repositories its teams are given."""
|
|
287
|
+
data, info = self._json("GET", "/v1/code")
|
|
288
|
+
return _answered(Code.from_dict(data or {}), info)
|
|
289
|
+
|
|
290
|
+
def github_install_url(self) -> str:
|
|
291
|
+
"""Where to install Wheelmachine's GitHub App for the account: an owner's or an admin's (good for half an hour)."""
|
|
292
|
+
data, _ = self._json("POST", "/v1/code/github")
|
|
293
|
+
return str((data or {}).get("url", ""))
|
|
294
|
+
|
|
295
|
+
def connect_gitlab(self, token: str, address: Optional[str] = None) -> CodeConnection:
|
|
296
|
+
"""Connect a GitLab group by its access token (``read_repository``, ``write_repository``); ``address`` is the
|
|
297
|
+
GitLab's https address, gitlab.com when left out. The token is kept sealed and never shown again."""
|
|
298
|
+
payload: Dict[str, Any] = {"token": token}
|
|
299
|
+
if address:
|
|
300
|
+
payload["address"] = address
|
|
301
|
+
data, info = self._json("POST", "/v1/code/gitlab", payload=payload)
|
|
302
|
+
return _answered(CodeConnection.from_dict(data or {}), info)
|
|
303
|
+
|
|
304
|
+
def disconnect_code(self, connection: str) -> None:
|
|
305
|
+
"""Take a connection away, with every repository a team was given of it."""
|
|
306
|
+
self._json("DELETE", "/v1/code/connections/" + quote(connection, safe=""))
|
|
307
|
+
|
|
308
|
+
def code_repos(self, connection: str) -> List[str]:
|
|
309
|
+
"""The repositories a connection reaches, by their paths."""
|
|
310
|
+
data, _ = self._json("GET", "/v1/code/connections/" + quote(connection, safe="") + "/repos")
|
|
311
|
+
return [str(r.get("repo", "")) for r in _list_of(data)]
|
|
312
|
+
|
|
313
|
+
def team_code(self, project: str) -> List[CodeGrant]:
|
|
314
|
+
"""The repositories a team is given."""
|
|
315
|
+
data, _ = self._json("GET", "/v1/projects/" + quote(project, safe="") + "/code")
|
|
316
|
+
return [CodeGrant.from_dict(g) for g in _list_of(data)]
|
|
317
|
+
|
|
318
|
+
def grant_code(self, project: str, connection: str, repo: str, access: str = "read") -> CodeGrant:
|
|
319
|
+
"""Give a team a repository of a connection, to ``read`` or to ``write`` (push) too: its full managers', owners'
|
|
320
|
+
and admins'."""
|
|
321
|
+
data, info = self._json("PUT", "/v1/projects/" + quote(project, safe="") + "/code",
|
|
322
|
+
payload={"connection": connection, "repo": repo, "access": access})
|
|
323
|
+
return _answered(CodeGrant.from_dict(data or {}), info)
|
|
324
|
+
|
|
325
|
+
def revoke_code(self, project: str, grant: str) -> None:
|
|
326
|
+
"""Take a repository from a team."""
|
|
327
|
+
self._json("DELETE", "/v1/projects/" + quote(project, safe="") + "/code/" + quote(grant, safe=""))
|
|
328
|
+
|
|
329
|
+
# --- how a call is made ---------------------------------------------------------------------------------------
|
|
330
|
+
|
|
331
|
+
def _held(self, seconds: float) -> float:
|
|
332
|
+
"""The time a call the API may hold for `seconds` is given: that, and a margin, and never less than usual."""
|
|
333
|
+
return max(self.timeout, seconds + 15.0)
|
|
334
|
+
|
|
335
|
+
def _json(
|
|
336
|
+
self,
|
|
337
|
+
method: str,
|
|
338
|
+
path: str,
|
|
339
|
+
*,
|
|
340
|
+
query: Optional[Sequence[Tuple[str, str]]] = None,
|
|
341
|
+
payload: Any = None,
|
|
342
|
+
idempotency_key: Optional[str] = None,
|
|
343
|
+
repeatable: bool = False,
|
|
344
|
+
once: bool = False,
|
|
345
|
+
timeout: Optional[float] = None,
|
|
346
|
+
confirm_code: Optional[str] = None,
|
|
347
|
+
) -> Tuple[Any, ResponseInfo]:
|
|
348
|
+
"""A call with a JSON answer: the answer's JSON (None when it has no body) and how it came."""
|
|
349
|
+
body = None
|
|
350
|
+
if payload is not None:
|
|
351
|
+
raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
|
352
|
+
body = BytesBody(raw, "application/json")
|
|
353
|
+
answer = self._send(
|
|
354
|
+
method,
|
|
355
|
+
path,
|
|
356
|
+
query=query,
|
|
357
|
+
body=body,
|
|
358
|
+
idempotency_key=idempotency_key,
|
|
359
|
+
repeatable=repeatable,
|
|
360
|
+
once=once,
|
|
361
|
+
timeout=timeout,
|
|
362
|
+
confirm_code=confirm_code,
|
|
363
|
+
)
|
|
364
|
+
info = self._info(answer)
|
|
365
|
+
if not answer.content.strip():
|
|
366
|
+
return None, info
|
|
367
|
+
try:
|
|
368
|
+
return json.loads(answer.content.decode("utf-8")), info
|
|
369
|
+
except ValueError:
|
|
370
|
+
raise WheelmachineError(
|
|
371
|
+
"the answer could not be read as JSON", status=answer.status, request_id=info.request_id
|
|
372
|
+
) from None
|
|
373
|
+
|
|
374
|
+
def _send(
|
|
375
|
+
self,
|
|
376
|
+
method: str,
|
|
377
|
+
path: str,
|
|
378
|
+
*,
|
|
379
|
+
query: Optional[Sequence[Tuple[str, str]]] = None,
|
|
380
|
+
body: Optional[Body] = None,
|
|
381
|
+
idempotency_key: Optional[str] = None,
|
|
382
|
+
repeatable: bool = False,
|
|
383
|
+
once: bool = False,
|
|
384
|
+
timeout: Optional[float] = None,
|
|
385
|
+
accept: str = "application/json",
|
|
386
|
+
buffer: bool = True,
|
|
387
|
+
confirm_code: Optional[str] = None,
|
|
388
|
+
) -> Answer:
|
|
389
|
+
"""One call, tried again when it is safe and worth it. Returns a 2xx or 3xx answer, its body read into
|
|
390
|
+
`content` when `buffer`; raises the error for any other. `repeatable`: a POST that does the same however
|
|
391
|
+
many times it is sent, so it is sent again as a PUT would be. `once`: a call that makes something new each
|
|
392
|
+
time, shown once, so it is never sent again once it may have reached the API, not even when refused for a
|
|
393
|
+
moment; only when it could not go out at all."""
|
|
394
|
+
target = self._target.prefix + path
|
|
395
|
+
if query:
|
|
396
|
+
target += "?" + urlencode(list(query), quote_via=quote)
|
|
397
|
+
limit = timeout if timeout is not None else self.timeout
|
|
398
|
+
repeatable = not once and (repeatable or method in _REPEATABLE or idempotency_key is not None)
|
|
399
|
+
deadline = time.monotonic() + limit
|
|
400
|
+
attempt = 0
|
|
401
|
+
while True:
|
|
402
|
+
headers = [
|
|
403
|
+
("User-Agent", USER_AGENT),
|
|
404
|
+
("Accept", accept),
|
|
405
|
+
]
|
|
406
|
+
if self._key.value():
|
|
407
|
+
headers.insert(0, ("Authorization", "Bearer " + self._key.value()))
|
|
408
|
+
if idempotency_key is not None:
|
|
409
|
+
headers.append(("Idempotency-Key", idempotency_key))
|
|
410
|
+
if confirm_code is not None:
|
|
411
|
+
headers.append(("X-Confirm-Code", confirm_code))
|
|
412
|
+
opened: Optional[OpenBody] = None
|
|
413
|
+
if body is not None:
|
|
414
|
+
opened = body.open()
|
|
415
|
+
headers.append(("Content-Type", body.content_type))
|
|
416
|
+
headers.append(("Content-Length", str(opened.length)))
|
|
417
|
+
elif method in ("POST", "PUT", "PATCH"):
|
|
418
|
+
headers.append(("Content-Length", "0"))
|
|
419
|
+
try:
|
|
420
|
+
answer = self._transport.send(method, target, headers, opened, limit)
|
|
421
|
+
if buffer and answer.status < 400:
|
|
422
|
+
answer.content = answer.read()
|
|
423
|
+
except Failure as failure:
|
|
424
|
+
if (
|
|
425
|
+
attempt < self.max_retries
|
|
426
|
+
and (repeatable or not failure.sent)
|
|
427
|
+
and (body is None or body.can_repeat())
|
|
428
|
+
):
|
|
429
|
+
self._sleep(self._backoff(attempt))
|
|
430
|
+
attempt += 1
|
|
431
|
+
continue
|
|
432
|
+
raise self._no_answer(failure, limit, idempotency_key) from failure.cause
|
|
433
|
+
finally:
|
|
434
|
+
if opened is not None:
|
|
435
|
+
opened.close()
|
|
436
|
+
if answer.status < 400:
|
|
437
|
+
return answer
|
|
438
|
+
error = self._refusal(answer)
|
|
439
|
+
error.idempotency_key = idempotency_key
|
|
440
|
+
# an answer came, so the call reached the API: one that makes something new each time is not sent again
|
|
441
|
+
pause = None
|
|
442
|
+
if not once:
|
|
443
|
+
pause = self._pause_before_retry(error, attempt, repeatable, idempotency_key, body, deadline)
|
|
444
|
+
if pause is None:
|
|
445
|
+
raise error
|
|
446
|
+
if error.status != 409:
|
|
447
|
+
attempt += 1
|
|
448
|
+
self._sleep(pause)
|
|
449
|
+
|
|
450
|
+
def _pause_before_retry(
|
|
451
|
+
self,
|
|
452
|
+
error: WheelmachineError,
|
|
453
|
+
attempt: int,
|
|
454
|
+
repeatable: bool,
|
|
455
|
+
idempotency_key: Optional[str],
|
|
456
|
+
body: Optional[Body],
|
|
457
|
+
deadline: float,
|
|
458
|
+
) -> Optional[float]:
|
|
459
|
+
"""How long to wait before trying a refused call again, or None when it is not tried again."""
|
|
460
|
+
if body is not None and not body.can_repeat():
|
|
461
|
+
return None
|
|
462
|
+
after = error.retry_after
|
|
463
|
+
if after is not None and after > LONGEST_RETRY_AFTER_S:
|
|
464
|
+
return None
|
|
465
|
+
status = error.status
|
|
466
|
+
if status == 409:
|
|
467
|
+
# the first call with this Idempotency-Key is still running: its answer is waited for, within the call's
|
|
468
|
+
# own time, without spending a retry
|
|
469
|
+
if idempotency_key is None or after is None or time.monotonic() + after > deadline:
|
|
470
|
+
return None
|
|
471
|
+
return after
|
|
472
|
+
if attempt >= self.max_retries:
|
|
473
|
+
return None
|
|
474
|
+
if status == 429:
|
|
475
|
+
if error.code == "limit_reached" and after is None:
|
|
476
|
+
return None # a limit of the account's, not of the moment: trying again soon changes nothing
|
|
477
|
+
elif status in (502, 504):
|
|
478
|
+
# the call may have reached the API and run: sent again only when that is safe
|
|
479
|
+
if not repeatable:
|
|
480
|
+
return None
|
|
481
|
+
elif status != 503:
|
|
482
|
+
return None
|
|
483
|
+
elif error.code == "not_open":
|
|
484
|
+
# not served yet (web addresses, say), not busy: waiting does not change it, whatever Retry-After says
|
|
485
|
+
return None
|
|
486
|
+
return after if after is not None else self._backoff(attempt)
|
|
487
|
+
|
|
488
|
+
def _backoff(self, attempt: int) -> float:
|
|
489
|
+
"""Exponential backoff with full jitter: anywhere from 0 to 0.5 s, 1 s, 2 s... at most 8 s."""
|
|
490
|
+
return min(_BACKOFF_MOST_S, _BACKOFF_FIRST_S * 2.0 ** attempt) * self._jitter()
|
|
491
|
+
|
|
492
|
+
def _refusal(self, answer: Answer) -> WheelmachineError:
|
|
493
|
+
problem: Optional[Dict[str, Any]] = None
|
|
494
|
+
try:
|
|
495
|
+
raw = answer.read(1 << 16)
|
|
496
|
+
except Failure:
|
|
497
|
+
raw = b""
|
|
498
|
+
finally:
|
|
499
|
+
answer.close()
|
|
500
|
+
if raw:
|
|
501
|
+
try:
|
|
502
|
+
parsed = json.loads(raw.decode("utf-8"))
|
|
503
|
+
if isinstance(parsed, dict):
|
|
504
|
+
problem = parsed
|
|
505
|
+
except ValueError:
|
|
506
|
+
problem = None
|
|
507
|
+
return error_from_answer(
|
|
508
|
+
answer.status,
|
|
509
|
+
answer.reason,
|
|
510
|
+
problem,
|
|
511
|
+
answer.header("X-Request-Id"),
|
|
512
|
+
_retry_after(answer.header("Retry-After")),
|
|
513
|
+
)
|
|
514
|
+
|
|
515
|
+
def _no_answer(self, failure: Failure, limit: float, idempotency_key: Optional[str]) -> APIConnectionError:
|
|
516
|
+
cause = failure.cause
|
|
517
|
+
error: APIConnectionError
|
|
518
|
+
if is_timeout(cause):
|
|
519
|
+
error = APITimeoutError("no answer from {} within {:g} seconds".format(self.base_url, limit))
|
|
520
|
+
else:
|
|
521
|
+
words = "the connection to {} was lost before its answer".format(self.base_url) if failure.sent else (
|
|
522
|
+
"could not reach {}".format(self.base_url)
|
|
523
|
+
)
|
|
524
|
+
error = APIConnectionError("{}: {}".format(words, str(cause) or type(cause).__name__))
|
|
525
|
+
error.idempotency_key = idempotency_key
|
|
526
|
+
return error
|
|
527
|
+
|
|
528
|
+
def _info(self, answer: Answer) -> ResponseInfo:
|
|
529
|
+
return ResponseInfo(
|
|
530
|
+
status=answer.status,
|
|
531
|
+
request_id=answer.header("X-Request-Id"),
|
|
532
|
+
replayed=(answer.header("Idempotent-Replayed") or "").strip().lower() == "true",
|
|
533
|
+
rate_limit=_rate_limit(answer.header("RateLimit"), answer.header("RateLimit-Policy")),
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
def _drain(self, answer: Answer, out: BinaryIO) -> int:
|
|
537
|
+
"""Writes an answer's body into `out`, a piece at a time; gives how many bytes came."""
|
|
538
|
+
n = 0
|
|
539
|
+
try:
|
|
540
|
+
for piece in answer.chunks():
|
|
541
|
+
out.write(piece)
|
|
542
|
+
n += len(piece)
|
|
543
|
+
except Failure as failure:
|
|
544
|
+
raise self._no_answer(failure, self.timeout, None) from failure.cause
|
|
545
|
+
return n
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _list_of(data: Any) -> List[Dict[str, Any]]:
|
|
549
|
+
got = data.get("data") if isinstance(data, dict) else None
|
|
550
|
+
return [item for item in got if isinstance(item, dict)] if isinstance(got, list) else []
|