okdata 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- okdata/__init__.py +28 -0
- okdata/_async_client.py +119 -0
- okdata/_base.py +88 -0
- okdata/_client.py +135 -0
- okdata/_errors.py +88 -0
- okdata/_generated/__init__.py +1 -0
- okdata/_generated/_async.py +7150 -0
- okdata/_generated/_endpoints.py +227 -0
- okdata/_generated/_errors.py +36 -0
- okdata/_generated/_models.py +1329 -0
- okdata/_generated/_sync.py +7150 -0
- okdata/_money.py +44 -0
- okdata/_paging.py +208 -0
- okdata/_request.py +74 -0
- okdata/_result.py +119 -0
- okdata/_transport.py +103 -0
- okdata/_types.py +123 -0
- okdata/_version.py +1 -0
- okdata/py.typed +0 -0
- okdata-0.1.0.dist-info/METADATA +143 -0
- okdata-0.1.0.dist-info/RECORD +23 -0
- okdata-0.1.0.dist-info/WHEEL +4 -0
- okdata-0.1.0.dist-info/licenses/LICENSE +21 -0
okdata/_money.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""US dollars as decimal strings outside, Decimal inside: no float ever touches a price or budget."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from decimal import ROUND_HALF_UP, Decimal, InvalidOperation
|
|
6
|
+
from typing import Union
|
|
7
|
+
|
|
8
|
+
_USD = re.compile(r"\d{1,12}(\.\d{1,6})?")
|
|
9
|
+
MAX_PRICE = re.compile(r"\d{1,7}(\.\d{1,6})?")
|
|
10
|
+
"""What X-Max-Price accepts: up to 7 digits, then up to 6 decimals."""
|
|
11
|
+
_MICRO = Decimal("0.000001")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def usd(value: Union[str, Decimal, int, float], name: str) -> Decimal:
|
|
15
|
+
"""A budget or cap as a Decimal; text must be a plain decimal, numbers are rounded to 6 places."""
|
|
16
|
+
if isinstance(value, bool):
|
|
17
|
+
raise TypeError(f"{name} must be US dollars, not a bool")
|
|
18
|
+
if isinstance(value, str):
|
|
19
|
+
text = value.strip()
|
|
20
|
+
if not _USD.fullmatch(text):
|
|
21
|
+
raise ValueError(f"{name} must be US dollars as a decimal string, like '0.05'")
|
|
22
|
+
return Decimal(text)
|
|
23
|
+
try:
|
|
24
|
+
amount = Decimal(str(value)) if isinstance(value, float) else Decimal(value)
|
|
25
|
+
except (InvalidOperation, ValueError) as err:
|
|
26
|
+
raise ValueError(f"{name} must be US dollars") from err
|
|
27
|
+
if not amount.is_finite() or amount < 0:
|
|
28
|
+
raise ValueError(f"{name} must be 0 or more US dollars")
|
|
29
|
+
return amount.quantize(_MICRO, rounding=ROUND_HALF_UP)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def text(amount: Decimal) -> str:
|
|
33
|
+
"""Decimal("0.0400") -> "0.04"; Decimal("0") -> "0"."""
|
|
34
|
+
normalized = amount.quantize(_MICRO, rounding=ROUND_HALF_UP).normalize()
|
|
35
|
+
out = format(normalized, "f")
|
|
36
|
+
return out if out != "-0" else "0"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def max_price_header(value: Union[str, Decimal, int, float]) -> str:
|
|
40
|
+
"""The price cap as X-Max-Price wants it."""
|
|
41
|
+
out = text(usd(value, "max_price"))
|
|
42
|
+
if not MAX_PRICE.fullmatch(out):
|
|
43
|
+
raise ValueError("max_price must be at most 7 digits before the point, like '0.05'")
|
|
44
|
+
return out
|
okdata/_paging.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""pages() and items(): follow page.next_cursor with the same parameters, and stop before the next
|
|
2
|
+
call could take the total past max_spend (counted from meta.billing.amount of the pages so far)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from decimal import Decimal
|
|
6
|
+
from typing import Any, AsyncIterator, Awaitable, Callable, Dict, Generic, Iterator, Literal, Mapping, Optional, Tuple, TypeVar
|
|
7
|
+
|
|
8
|
+
from ._errors import OkDataError
|
|
9
|
+
from ._money import text, usd
|
|
10
|
+
from ._types import EndpointInfo, Page, Price, Result
|
|
11
|
+
|
|
12
|
+
T = TypeVar("T")
|
|
13
|
+
StopReason = Literal["end", "max_pages", "max_items", "max_spend", "balance"]
|
|
14
|
+
"""Why paging ended: no next page, a limit you set, or a balance that will not cover the next page."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _positive(n: Optional[int], name: str) -> Optional[int]:
|
|
18
|
+
if n is not None and (isinstance(n, bool) or not isinstance(n, int) or n < 1):
|
|
19
|
+
raise ValueError(f"{name} must be a whole number, 1 or more")
|
|
20
|
+
return n
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class _Budget:
|
|
24
|
+
"""The paging state both pagers share: pages, spend, the per-call cap, why it stopped."""
|
|
25
|
+
|
|
26
|
+
def __init__(self, endpoint_id: str, info: Optional[EndpointInfo], params: Mapping[str, Any],
|
|
27
|
+
max_pages: Optional[int], max_spend: Optional[Price]) -> None:
|
|
28
|
+
if info is not None and not info.list:
|
|
29
|
+
raise TypeError(f"{endpoint_id} returns one object, not pages")
|
|
30
|
+
self.endpoint_id = endpoint_id
|
|
31
|
+
self.worst = Decimal(info.max_charge) if info is not None else Decimal(0)
|
|
32
|
+
self.params: Dict[str, Any] = dict(params)
|
|
33
|
+
self.max_pages = _positive(max_pages, "max_pages")
|
|
34
|
+
self.max_spend = usd(max_spend, "max_spend") if max_spend is not None else None
|
|
35
|
+
cap = self.params.get("max_price")
|
|
36
|
+
self.user_cap = usd(cap, "max_price") if cap is not None else None
|
|
37
|
+
self.spent = Decimal(0)
|
|
38
|
+
self.calls = 0
|
|
39
|
+
self.stop_reason: Optional[StopReason] = None
|
|
40
|
+
self.started = False
|
|
41
|
+
|
|
42
|
+
def start(self) -> None:
|
|
43
|
+
if self.started:
|
|
44
|
+
raise RuntimeError("a pager can be iterated once; call pages() again for a new run")
|
|
45
|
+
self.started = True
|
|
46
|
+
|
|
47
|
+
def next_call(self) -> Optional[Tuple[Dict[str, Any], bool]]:
|
|
48
|
+
"""The params of the next call and whether our own cap is on it, or None to stop."""
|
|
49
|
+
if self.max_pages is not None and self.calls >= self.max_pages:
|
|
50
|
+
self.stop_reason = "max_pages"
|
|
51
|
+
return None
|
|
52
|
+
left = self.max_spend - self.spent if self.max_spend is not None else None
|
|
53
|
+
if left is not None and self.worst > left:
|
|
54
|
+
self.stop_reason = "max_spend"
|
|
55
|
+
return None
|
|
56
|
+
capped = left is not None and (self.user_cap is None or left < self.user_cap)
|
|
57
|
+
return ({**self.params, "max_price": text(left)} if capped and left is not None else self.params), capped
|
|
58
|
+
|
|
59
|
+
def refused(self, err: OkDataError, capped: bool) -> bool:
|
|
60
|
+
"""Our own cap refused a page the catalog priced lower than the server does: that is the budget."""
|
|
61
|
+
if capped and err.code == "max_price_exceeded":
|
|
62
|
+
self.stop_reason = "max_spend"
|
|
63
|
+
return True
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
def after(self, result: Result) -> bool:
|
|
67
|
+
"""Counts a page; True when there is a next one to fetch."""
|
|
68
|
+
if not isinstance(result, Page):
|
|
69
|
+
raise TypeError(f"{self.endpoint_id} answered with one object, not a page")
|
|
70
|
+
self.calls += 1
|
|
71
|
+
self.spent += Decimal(result.cost or "0")
|
|
72
|
+
if not result.next_cursor or not result.has_more:
|
|
73
|
+
self.stop_reason = "end"
|
|
74
|
+
return False
|
|
75
|
+
if result.meta.get("billing", {}).get("covers_next") is False:
|
|
76
|
+
self.stop_reason = "balance"
|
|
77
|
+
return False
|
|
78
|
+
self.params = {**self.params, "cursor": result.next_cursor}
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _Stats:
|
|
83
|
+
_budget: _Budget
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def spent(self) -> str:
|
|
87
|
+
"""US dollars charged by the pages fetched so far."""
|
|
88
|
+
return text(self._budget.spent)
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def calls(self) -> int:
|
|
92
|
+
return self._budget.calls
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class Pager(_Stats, Generic[T]):
|
|
96
|
+
"""Pages of one listing. Iterate it once; spent, calls and stop_reason tell how it went."""
|
|
97
|
+
|
|
98
|
+
def __init__(self, budget: _Budget, call: Callable[[Dict[str, Any]], Result]) -> None:
|
|
99
|
+
self._budget = budget
|
|
100
|
+
self._call = call
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def stop_reason(self) -> Optional[StopReason]:
|
|
104
|
+
return self._budget.stop_reason
|
|
105
|
+
|
|
106
|
+
def __iter__(self) -> Iterator[Page[T]]:
|
|
107
|
+
self._budget.start()
|
|
108
|
+
while True:
|
|
109
|
+
step = self._budget.next_call()
|
|
110
|
+
if step is None:
|
|
111
|
+
return
|
|
112
|
+
params, capped = step
|
|
113
|
+
try:
|
|
114
|
+
result = self._call(params)
|
|
115
|
+
except OkDataError as err:
|
|
116
|
+
if self._budget.refused(err, capped):
|
|
117
|
+
return
|
|
118
|
+
raise
|
|
119
|
+
more = self._budget.after(result)
|
|
120
|
+
yield result # type: ignore[misc]
|
|
121
|
+
if not more:
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ItemPager(_Stats, Generic[T]):
|
|
126
|
+
"""The items of one listing, page after page, up to max_items."""
|
|
127
|
+
|
|
128
|
+
def __init__(self, pager: Pager[T], max_items: Optional[int]) -> None:
|
|
129
|
+
self._pager = pager
|
|
130
|
+
self._budget = pager._budget
|
|
131
|
+
self._max_items = _positive(max_items, "max_items")
|
|
132
|
+
self._items = 0
|
|
133
|
+
self._hit = False
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def items(self) -> int:
|
|
137
|
+
return self._items
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def stop_reason(self) -> Optional[StopReason]:
|
|
141
|
+
return "max_items" if self._hit else self._pager.stop_reason
|
|
142
|
+
|
|
143
|
+
def __iter__(self) -> Iterator[T]:
|
|
144
|
+
for page in self._pager:
|
|
145
|
+
for item in page.items:
|
|
146
|
+
yield item
|
|
147
|
+
self._items += 1
|
|
148
|
+
if self._max_items is not None and self._items >= self._max_items:
|
|
149
|
+
self._hit = True
|
|
150
|
+
return
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class AsyncPager(_Stats, Generic[T]):
|
|
154
|
+
"""The async twin of Pager: ``async for page in ok.pages(...)``."""
|
|
155
|
+
|
|
156
|
+
def __init__(self, budget: _Budget, call: Callable[[Dict[str, Any]], Awaitable[Result]]) -> None:
|
|
157
|
+
self._budget = budget
|
|
158
|
+
self._call = call
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def stop_reason(self) -> Optional[StopReason]:
|
|
162
|
+
return self._budget.stop_reason
|
|
163
|
+
|
|
164
|
+
async def __aiter__(self) -> AsyncIterator[Page[T]]:
|
|
165
|
+
self._budget.start()
|
|
166
|
+
while True:
|
|
167
|
+
step = self._budget.next_call()
|
|
168
|
+
if step is None:
|
|
169
|
+
return
|
|
170
|
+
params, capped = step
|
|
171
|
+
try:
|
|
172
|
+
result = await self._call(params)
|
|
173
|
+
except OkDataError as err:
|
|
174
|
+
if self._budget.refused(err, capped):
|
|
175
|
+
return
|
|
176
|
+
raise
|
|
177
|
+
more = self._budget.after(result)
|
|
178
|
+
yield result # type: ignore[misc]
|
|
179
|
+
if not more:
|
|
180
|
+
return
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class AsyncItemPager(_Stats, Generic[T]):
|
|
184
|
+
"""The async twin of ItemPager."""
|
|
185
|
+
|
|
186
|
+
def __init__(self, pager: AsyncPager[T], max_items: Optional[int]) -> None:
|
|
187
|
+
self._pager = pager
|
|
188
|
+
self._budget = pager._budget
|
|
189
|
+
self._max_items = _positive(max_items, "max_items")
|
|
190
|
+
self._items = 0
|
|
191
|
+
self._hit = False
|
|
192
|
+
|
|
193
|
+
@property
|
|
194
|
+
def items(self) -> int:
|
|
195
|
+
return self._items
|
|
196
|
+
|
|
197
|
+
@property
|
|
198
|
+
def stop_reason(self) -> Optional[StopReason]:
|
|
199
|
+
return "max_items" if self._hit else self._pager.stop_reason
|
|
200
|
+
|
|
201
|
+
async def __aiter__(self) -> AsyncIterator[T]:
|
|
202
|
+
async for page in self._pager:
|
|
203
|
+
for item in page.items:
|
|
204
|
+
yield item
|
|
205
|
+
self._items += 1
|
|
206
|
+
if self._max_items is not None and self._items >= self._max_items:
|
|
207
|
+
self._hit = True
|
|
208
|
+
return
|
okdata/_request.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Turns an endpoint id and flat params into the HTTP request the contract wants: GET with query
|
|
2
|
+
parameters, or POST with a JSON body when a list parameter (ids, keywords) is given to an endpoint
|
|
3
|
+
that takes one. view, fields and max_price ride along; max_price becomes X-Max-Price."""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
|
|
9
|
+
|
|
10
|
+
from ._money import max_price_header
|
|
11
|
+
from ._types import EndpointInfo
|
|
12
|
+
|
|
13
|
+
_ID = re.compile(r"[a-z0-9_]+(\.[a-z0-9_]+)+")
|
|
14
|
+
_FIELD = re.compile(r"[A-Za-z0-9_]+(\.[A-Za-z0-9_]+)*")
|
|
15
|
+
VIEWS = ("compact", "standard", "full")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class HttpRequest:
|
|
20
|
+
method: str
|
|
21
|
+
path: str
|
|
22
|
+
query: List[Tuple[str, str]] = field(default_factory=list)
|
|
23
|
+
body: Optional[Dict[str, Any]] = None
|
|
24
|
+
max_price: Optional[str] = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def path_of_id(endpoint_id: str) -> str:
|
|
28
|
+
"""/v1/douyin/videos/search for douyin.videos.search."""
|
|
29
|
+
if not _ID.fullmatch(endpoint_id):
|
|
30
|
+
raise ValueError(f"not an endpoint id: {endpoint_id!r} (ids look like douyin.videos.search)")
|
|
31
|
+
return "/v1/" + endpoint_id.replace(".", "/")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def field_list(fields: Union[str, Sequence[str]]) -> List[str]:
|
|
35
|
+
parts = [f.strip() for f in (fields.split(",") if isinstance(fields, str) else list(fields))]
|
|
36
|
+
parts = [p for p in parts if p]
|
|
37
|
+
if not parts:
|
|
38
|
+
raise ValueError("fields is empty; leave it out to get every field")
|
|
39
|
+
bad = [p for p in parts if not _FIELD.fullmatch(p)]
|
|
40
|
+
if bad:
|
|
41
|
+
raise ValueError(f"fields are dotted names like metrics.likes, not {bad!r}")
|
|
42
|
+
return parts
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _query_value(value: Any) -> str:
|
|
46
|
+
if isinstance(value, bool):
|
|
47
|
+
return "true" if value else "false"
|
|
48
|
+
if isinstance(value, (list, tuple)):
|
|
49
|
+
return ",".join(str(v) for v in value)
|
|
50
|
+
return str(value)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _is_list(value: Any) -> bool:
|
|
54
|
+
return isinstance(value, (list, tuple))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def build_request(endpoint_id: str, info: Optional[EndpointInfo], params: Mapping[str, Any]) -> HttpRequest:
|
|
58
|
+
values: Dict[str, Any] = {k: v for k, v in params.items() if v is not None}
|
|
59
|
+
cap = values.pop("max_price", None)
|
|
60
|
+
view = values.pop("view", None)
|
|
61
|
+
fields = values.pop("fields", None)
|
|
62
|
+
if view is not None:
|
|
63
|
+
if view not in VIEWS:
|
|
64
|
+
raise ValueError(f"view must be one of {', '.join(VIEWS)}")
|
|
65
|
+
values["view"] = view
|
|
66
|
+
if fields is not None:
|
|
67
|
+
values["fields"] = field_list(fields)
|
|
68
|
+
path = info.path if info is not None else path_of_id(endpoint_id)
|
|
69
|
+
max_price = max_price_header(cap) if cap is not None else None
|
|
70
|
+
batch = info is not None and info.post and any(info.params.get(k) == "string[]" and _is_list(v) for k, v in values.items())
|
|
71
|
+
if batch:
|
|
72
|
+
body = {k: list(v) if _is_list(v) else v for k, v in values.items()}
|
|
73
|
+
return HttpRequest("POST", path, [], body, max_price)
|
|
74
|
+
return HttpRequest("GET", path, [(k, _query_value(v)) for k, v in values.items()], None, max_price)
|
okdata/_result.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Success envelopes as Page / Item / Batch, error bodies as OkDataError, and when to retry."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import random
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any, Callable, Dict, Mapping, Optional, cast
|
|
8
|
+
|
|
9
|
+
from ._errors import RETRY_ADVICE, OkDataError, RetryAdvice
|
|
10
|
+
from ._types import Batch, Item, Meta, Page, Result
|
|
11
|
+
|
|
12
|
+
_RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
|
|
13
|
+
_IN_PROGRESS_MIN_S = 1.0
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def seconds(value: Any) -> Optional[float]:
|
|
17
|
+
"""A non-negative number of seconds, or None."""
|
|
18
|
+
if isinstance(value, bool):
|
|
19
|
+
return None
|
|
20
|
+
try:
|
|
21
|
+
n = float(value)
|
|
22
|
+
except (TypeError, ValueError):
|
|
23
|
+
return None
|
|
24
|
+
return n if n >= 0 and n == n and n != float("inf") else None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def to_result(body: Mapping[str, Any], idempotency_key: Optional[str]) -> Result:
|
|
28
|
+
meta = cast(Meta, body.get("meta") or {})
|
|
29
|
+
billing = meta.get("billing") or {}
|
|
30
|
+
data = body.get("data") or {}
|
|
31
|
+
base: Dict[str, Any] = dict(
|
|
32
|
+
request_id=str(body.get("request_id", "")),
|
|
33
|
+
capability=str(meta.get("capability", "")),
|
|
34
|
+
result_status=str(meta.get("result_status", "complete")),
|
|
35
|
+
cost=str(billing.get("amount", "0")),
|
|
36
|
+
balance_after=billing.get("balance_after") or None,
|
|
37
|
+
replayed=meta.get("replayed") is True,
|
|
38
|
+
cached=meta.get("cached") is True,
|
|
39
|
+
unavailable=dict(meta.get("unavailable") or {}),
|
|
40
|
+
warnings=list(body.get("warnings") or []),
|
|
41
|
+
meta=meta,
|
|
42
|
+
idempotency_key=idempotency_key,
|
|
43
|
+
raw=dict(body),
|
|
44
|
+
)
|
|
45
|
+
items = data.get("items")
|
|
46
|
+
page = body.get("page")
|
|
47
|
+
if isinstance(items, list) and isinstance(page, dict):
|
|
48
|
+
cursor = page.get("next_cursor")
|
|
49
|
+
return Page(
|
|
50
|
+
**base, items=items, next_cursor=cursor, cursor_expires_at=page.get("cursor_expires_at"),
|
|
51
|
+
has_more=page.get("has_more") is True and cursor is not None, returned=int(page.get("returned", len(items))),
|
|
52
|
+
)
|
|
53
|
+
if isinstance(items, list):
|
|
54
|
+
return Batch(**base, items=items, missing=list(data.get("missing") or []))
|
|
55
|
+
return Item(**base, item=cast(Dict[str, Any], data.get("item")))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse_answer(status: int, headers: Mapping[str, str], text: str, idempotency_key: Optional[str]) -> Dict[str, Any]:
|
|
59
|
+
"""The success envelope of an answer, or raises the OkDataError it carries."""
|
|
60
|
+
try:
|
|
61
|
+
parsed = json.loads(text) if text else None
|
|
62
|
+
except ValueError:
|
|
63
|
+
parsed = None
|
|
64
|
+
body = parsed if isinstance(parsed, dict) else None
|
|
65
|
+
if body is not None and body.get("code") == "ok" and status < 400:
|
|
66
|
+
return body
|
|
67
|
+
retry_after_header = seconds(headers.get("retry-after"))
|
|
68
|
+
if body is not None and isinstance(body.get("code"), str):
|
|
69
|
+
advice = body.get("retry")
|
|
70
|
+
raise OkDataError(
|
|
71
|
+
body["code"],
|
|
72
|
+
body["message"] if isinstance(body.get("message"), str) else body["code"],
|
|
73
|
+
status=status,
|
|
74
|
+
request_id=body["request_id"] if isinstance(body.get("request_id"), str) else headers.get("x-request-id"),
|
|
75
|
+
charged=body.get("charged", False),
|
|
76
|
+
retry=cast(RetryAdvice, advice) if advice in RETRY_ADVICE else "no",
|
|
77
|
+
retry_after_seconds=seconds(body.get("retry_after_seconds")) if body.get("retry_after_seconds") is not None else retry_after_header,
|
|
78
|
+
idempotency_key=idempotency_key,
|
|
79
|
+
body=body,
|
|
80
|
+
)
|
|
81
|
+
raise OkDataError(
|
|
82
|
+
"bad_response",
|
|
83
|
+
f"HTTP {status} without an okdata answer",
|
|
84
|
+
status=status,
|
|
85
|
+
request_id=headers.get("x-request-id"),
|
|
86
|
+
charged=None if status >= 500 or status < 300 else False,
|
|
87
|
+
retry="same_key" if status < 300 else "yes" if status in _RETRY_STATUSES else "no",
|
|
88
|
+
retry_after_seconds=retry_after_header,
|
|
89
|
+
idempotency_key=idempotency_key,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass(frozen=True)
|
|
94
|
+
class RetryPolicy:
|
|
95
|
+
max_retries: int
|
|
96
|
+
base_delay: float
|
|
97
|
+
"""First backoff step, seconds, when the server names no wait."""
|
|
98
|
+
max_delay: float
|
|
99
|
+
"""A server-named wait longer than this is not slept through: the error is raised."""
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def retry_delay(err: OkDataError, attempt: int, policy: RetryPolicy, rand: Callable[[], float] = random.random) -> Optional[float]:
|
|
103
|
+
"""Seconds to wait before the next attempt, or None to stop. ``attempt`` counts retries made so far.
|
|
104
|
+
|
|
105
|
+
yes / same_key: retry (same Idempotency-Key); once: one more try at most; no / after_fix: never.
|
|
106
|
+
"""
|
|
107
|
+
if attempt >= policy.max_retries or err.retry in ("no", "after_fix"):
|
|
108
|
+
return None
|
|
109
|
+
if err.retry == "once" and attempt > 0:
|
|
110
|
+
return None
|
|
111
|
+
if err.retry_after_seconds is not None:
|
|
112
|
+
return err.retry_after_seconds if err.retry_after_seconds <= policy.max_delay else None
|
|
113
|
+
backoff = min(policy.max_delay, policy.base_delay * 2.0 ** attempt) * (0.5 + rand() / 2)
|
|
114
|
+
return max(_IN_PROGRESS_MIN_S, backoff) if err.code == "call_in_progress" else backoff
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def leaves_outcome_open(err: OkDataError) -> bool:
|
|
118
|
+
"""Errors after which an earlier attempt with the same key may still have run (and been charged)."""
|
|
119
|
+
return err.charged is None or err.code == "call_in_progress"
|
okdata/_transport.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""One logical request, sync or async: the same Idempotency-Key on every attempt, retries as the
|
|
2
|
+
error body says, a per-attempt timeout, and the charge marked unknown when an earlier attempt's fate is."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
|
|
10
|
+
from ._errors import OkDataConnectionError, OkDataError
|
|
11
|
+
from ._result import RetryPolicy, leaves_outcome_open, parse_answer, retry_delay
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class Config:
|
|
16
|
+
base_url: str
|
|
17
|
+
api_key: Optional[str] = field(repr=False) # never in a repr, a log line or a traceback
|
|
18
|
+
timeout: float = field(default=65.0)
|
|
19
|
+
policy: RetryPolicy = field(default_factory=lambda: RetryPolicy(2, 0.5, 20.0))
|
|
20
|
+
headers: Dict[str, str] = field(default_factory=dict)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Send:
|
|
25
|
+
method: str
|
|
26
|
+
path: str
|
|
27
|
+
query: Sequence[Tuple[str, str]] = ()
|
|
28
|
+
body: Optional[Dict[str, Any]] = None
|
|
29
|
+
idempotency_key: Optional[str] = None
|
|
30
|
+
max_price: Optional[str] = None
|
|
31
|
+
auth: bool = True
|
|
32
|
+
"""False: send no Authorization header (the free catalog routes, the CLI link)."""
|
|
33
|
+
timeout: Optional[float] = None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _headers(cfg: Config, req: Send) -> Dict[str, str]:
|
|
37
|
+
headers = {"Accept": "application/json", **cfg.headers}
|
|
38
|
+
if req.auth and cfg.api_key:
|
|
39
|
+
headers["Authorization"] = f"Bearer {cfg.api_key}"
|
|
40
|
+
if req.idempotency_key:
|
|
41
|
+
headers["Idempotency-Key"] = req.idempotency_key
|
|
42
|
+
if req.max_price is not None:
|
|
43
|
+
headers["X-Max-Price"] = req.max_price
|
|
44
|
+
return headers
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _lost(cfg: Config, req: Send, err: Exception) -> OkDataConnectionError:
|
|
48
|
+
if isinstance(err, httpx.TimeoutException):
|
|
49
|
+
limit = req.timeout if req.timeout is not None else cfg.timeout
|
|
50
|
+
return OkDataConnectionError(f"no answer from {cfg.base_url} within {limit} s", "client_timeout", req.idempotency_key)
|
|
51
|
+
return OkDataConnectionError(f"could not reach {cfg.base_url}: {err}", "connection_error", req.idempotency_key)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _give_up(err: OkDataError, open_outcome: bool) -> OkDataError:
|
|
55
|
+
return err.with_charged(None) if open_outcome and err.charged is False else err
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def send_sync(cfg: Config, http: httpx.Client, sleep: Callable[[float], None], req: Send) -> Dict[str, Any]:
|
|
59
|
+
"""Sends one request (with its retries); returns the success envelope or raises OkDataError."""
|
|
60
|
+
open_outcome = False
|
|
61
|
+
retries = 0
|
|
62
|
+
while True:
|
|
63
|
+
try:
|
|
64
|
+
try:
|
|
65
|
+
res = http.request(req.method, cfg.base_url + req.path, params=list(req.query), json=req.body,
|
|
66
|
+
headers=_headers(cfg, req), timeout=req.timeout if req.timeout is not None else cfg.timeout)
|
|
67
|
+
except httpx.TransportError as err:
|
|
68
|
+
raise _lost(cfg, req, err) from err
|
|
69
|
+
return parse_answer(res.status_code, res.headers, res.text, req.idempotency_key)
|
|
70
|
+
except OkDataError as err:
|
|
71
|
+
open_outcome = open_outcome or leaves_outcome_open(err)
|
|
72
|
+
delay = retry_delay(err, retries, cfg.policy)
|
|
73
|
+
if delay is None:
|
|
74
|
+
final = _give_up(err, open_outcome)
|
|
75
|
+
if final is err:
|
|
76
|
+
raise
|
|
77
|
+
raise final from err
|
|
78
|
+
sleep(delay)
|
|
79
|
+
retries += 1
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
async def send_async(cfg: Config, http: httpx.AsyncClient, sleep: Callable[[float], Awaitable[None]], req: Send) -> Dict[str, Any]:
|
|
83
|
+
"""The async twin of send_sync."""
|
|
84
|
+
open_outcome = False
|
|
85
|
+
retries = 0
|
|
86
|
+
while True:
|
|
87
|
+
try:
|
|
88
|
+
try:
|
|
89
|
+
res = await http.request(req.method, cfg.base_url + req.path, params=list(req.query), json=req.body,
|
|
90
|
+
headers=_headers(cfg, req), timeout=req.timeout if req.timeout is not None else cfg.timeout)
|
|
91
|
+
except httpx.TransportError as err:
|
|
92
|
+
raise _lost(cfg, req, err) from err
|
|
93
|
+
return parse_answer(res.status_code, res.headers, res.text, req.idempotency_key)
|
|
94
|
+
except OkDataError as err:
|
|
95
|
+
open_outcome = open_outcome or leaves_outcome_open(err)
|
|
96
|
+
delay = retry_delay(err, retries, cfg.policy)
|
|
97
|
+
if delay is None:
|
|
98
|
+
final = _give_up(err, open_outcome)
|
|
99
|
+
if final is err:
|
|
100
|
+
raise
|
|
101
|
+
raise final from err
|
|
102
|
+
await sleep(delay)
|
|
103
|
+
retries += 1
|
okdata/_types.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""What a call returns (Page, Item, Batch) and the shapes shared with the generated methods."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from decimal import Decimal
|
|
6
|
+
from typing import Any, Dict, Generic, List, Literal, Mapping, Optional, Sequence, TypedDict, TypeVar, Union
|
|
7
|
+
|
|
8
|
+
T = TypeVar("T")
|
|
9
|
+
|
|
10
|
+
View = Literal["compact", "standard", "full"]
|
|
11
|
+
"""compact (shortest; long text cut at 500 characters), standard (default) or full (adds ``ext``)."""
|
|
12
|
+
|
|
13
|
+
Fields = Union[str, Sequence[str]]
|
|
14
|
+
"""Dotted paths to keep: ``["id", "metrics.likes"]`` or ``"id,metrics.likes"``."""
|
|
15
|
+
|
|
16
|
+
Price = Union[str, Decimal, int, float]
|
|
17
|
+
"""US dollars. Text (``"0.05"``) or Decimal is exact; floats are rounded to 6 places."""
|
|
18
|
+
|
|
19
|
+
ResultStatus = Literal["complete", "partial", "empty"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Billing(TypedDict, total=False):
|
|
23
|
+
amount: str
|
|
24
|
+
"""What this call cost, US dollars as a decimal string ("0" for a free replay)."""
|
|
25
|
+
currency: str
|
|
26
|
+
balance_after: str
|
|
27
|
+
covers_next: bool
|
|
28
|
+
"""Sent (as false) only when what is left does not cover another call like this one."""
|
|
29
|
+
top_up_url: str
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Meta(TypedDict, total=False):
|
|
33
|
+
capability: str
|
|
34
|
+
result_status: ResultStatus
|
|
35
|
+
billing: Billing
|
|
36
|
+
unavailable: Dict[str, str]
|
|
37
|
+
"""Fields this endpoint cannot give, and why ("platform" or "capability")."""
|
|
38
|
+
cached: bool
|
|
39
|
+
replayed: bool
|
|
40
|
+
fetched_at: str
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class CallWarning(TypedDict, total=False):
|
|
44
|
+
code: str
|
|
45
|
+
"""items_dropped, text_truncated, items_capped_by_balance or note."""
|
|
46
|
+
message: str
|
|
47
|
+
count: int
|
|
48
|
+
item: int
|
|
49
|
+
fields: List[str]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Missing(TypedDict):
|
|
53
|
+
id: str
|
|
54
|
+
reason: str
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class EndpointInfo:
|
|
59
|
+
"""One endpoint as the SDK needs it at run time (generated from the catalog)."""
|
|
60
|
+
|
|
61
|
+
id: str
|
|
62
|
+
path: str
|
|
63
|
+
list: bool
|
|
64
|
+
post: bool
|
|
65
|
+
"""True: a list parameter (ids, keywords) sends the call as POST with a JSON body."""
|
|
66
|
+
batch_max: Optional[int]
|
|
67
|
+
unit: Literal["call", "item"]
|
|
68
|
+
price: str
|
|
69
|
+
max_charge: str
|
|
70
|
+
page_size: Optional[int]
|
|
71
|
+
params: Mapping[str, str]
|
|
72
|
+
object: str
|
|
73
|
+
title: str
|
|
74
|
+
paused: bool
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class _Result:
|
|
79
|
+
request_id: str
|
|
80
|
+
capability: str
|
|
81
|
+
result_status: str
|
|
82
|
+
cost: str
|
|
83
|
+
"""What this call cost, US dollars as a decimal string."""
|
|
84
|
+
balance_after: Optional[str]
|
|
85
|
+
replayed: bool
|
|
86
|
+
"""True when an Idempotency-Key replayed a stored result at no charge."""
|
|
87
|
+
cached: bool
|
|
88
|
+
unavailable: Dict[str, str]
|
|
89
|
+
warnings: List[CallWarning]
|
|
90
|
+
meta: Meta
|
|
91
|
+
idempotency_key: Optional[str]
|
|
92
|
+
"""The key this call was sent with; resending with it never charges twice."""
|
|
93
|
+
raw: Dict[str, Any]
|
|
94
|
+
"""The success envelope exactly as the API sent it."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@dataclass(frozen=True)
|
|
98
|
+
class Page(_Result, Generic[T]):
|
|
99
|
+
"""One page of a list endpoint. The next page is a new call, charged on its own."""
|
|
100
|
+
|
|
101
|
+
items: List[T]
|
|
102
|
+
next_cursor: Optional[str]
|
|
103
|
+
cursor_expires_at: Optional[str]
|
|
104
|
+
has_more: bool
|
|
105
|
+
returned: int
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass(frozen=True)
|
|
109
|
+
class Item(_Result, Generic[T]):
|
|
110
|
+
"""The answer of a single-object endpoint."""
|
|
111
|
+
|
|
112
|
+
item: T
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass(frozen=True)
|
|
116
|
+
class Batch(_Result, Generic[T]):
|
|
117
|
+
"""The batch form of a single-object endpoint (ids given): found objects and what was missing."""
|
|
118
|
+
|
|
119
|
+
items: List[T]
|
|
120
|
+
missing: List[Missing]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
Result = Union[Page[Dict[str, Any]], Item[Dict[str, Any]], Batch[Dict[str, Any]]]
|
okdata/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
okdata/py.typed
ADDED
|
File without changes
|