sdo-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdo/__init__.py +74 -0
- sdo/_decompress.py +253 -0
- sdo/_encoding.py +739 -0
- sdo/_json.py +50 -0
- sdo/_logging.py +93 -0
- sdo/_secret.py +74 -0
- sdo/_transport.py +639 -0
- sdo/_urls.py +80 -0
- sdo/_version.py +8 -0
- sdo/_wire.py +126 -0
- sdo/client.py +459 -0
- sdo/config.py +452 -0
- sdo/errors.py +247 -0
- sdo/jobs.py +490 -0
- sdo/plugins.py +143 -0
- sdo/py.typed +0 -0
- sdo/response.py +177 -0
- sdo/usage.py +62 -0
- sdo_client-0.1.0.dist-info/METADATA +377 -0
- sdo_client-0.1.0.dist-info/RECORD +23 -0
- sdo_client-0.1.0.dist-info/WHEEL +4 -0
- sdo_client-0.1.0.dist-info/licenses/LICENSE +201 -0
- sdo_client-0.1.0.dist-info/licenses/NOTICE +8 -0
sdo/__init__.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""sdo — the official Python SDK for the Scrape.do web-scraping API.
|
|
3
|
+
|
|
4
|
+
Quickstart::
|
|
5
|
+
|
|
6
|
+
from sdo import Client, ScrapeConfig
|
|
7
|
+
|
|
8
|
+
with Client(token="YOUR_TOKEN") as client: # or set SDO_TOKEN
|
|
9
|
+
result = client.scrape("https://example.com", render=True)
|
|
10
|
+
print(result.status_code, len(result.bytes()))
|
|
11
|
+
print(result.text[:200])
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from ._version import __version__
|
|
17
|
+
from .client import Client
|
|
18
|
+
from .config import ScrapeConfig
|
|
19
|
+
from .errors import (
|
|
20
|
+
SdoAPIError,
|
|
21
|
+
SdoAuthError,
|
|
22
|
+
SdoConcurrencyError,
|
|
23
|
+
SdoConfigError,
|
|
24
|
+
SdoConnectionError,
|
|
25
|
+
SdoError,
|
|
26
|
+
SdoGatewayError,
|
|
27
|
+
SdoResponseTooLargeError,
|
|
28
|
+
SdoTimeoutError,
|
|
29
|
+
SdoTransportError,
|
|
30
|
+
SdoValidationError,
|
|
31
|
+
)
|
|
32
|
+
from .jobs import (
|
|
33
|
+
JOB_SORTS,
|
|
34
|
+
JOB_STATUSES,
|
|
35
|
+
MAX_PLUGIN_PARAMS,
|
|
36
|
+
Account,
|
|
37
|
+
AsyncJobs,
|
|
38
|
+
Job,
|
|
39
|
+
JobPage,
|
|
40
|
+
JobStatus,
|
|
41
|
+
Task,
|
|
42
|
+
TaskResult,
|
|
43
|
+
)
|
|
44
|
+
from .response import ScrapeResult
|
|
45
|
+
from .usage import Usage
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"__version__",
|
|
49
|
+
"Client",
|
|
50
|
+
"ScrapeConfig",
|
|
51
|
+
"ScrapeResult",
|
|
52
|
+
"Usage",
|
|
53
|
+
"AsyncJobs",
|
|
54
|
+
"Job",
|
|
55
|
+
"JobPage",
|
|
56
|
+
"JobStatus",
|
|
57
|
+
"JOB_SORTS",
|
|
58
|
+
"JOB_STATUSES",
|
|
59
|
+
"MAX_PLUGIN_PARAMS",
|
|
60
|
+
"Task",
|
|
61
|
+
"TaskResult",
|
|
62
|
+
"Account",
|
|
63
|
+
"SdoError",
|
|
64
|
+
"SdoConfigError",
|
|
65
|
+
"SdoTransportError",
|
|
66
|
+
"SdoConnectionError",
|
|
67
|
+
"SdoTimeoutError",
|
|
68
|
+
"SdoResponseTooLargeError",
|
|
69
|
+
"SdoAPIError",
|
|
70
|
+
"SdoAuthError",
|
|
71
|
+
"SdoValidationError",
|
|
72
|
+
"SdoConcurrencyError",
|
|
73
|
+
"SdoGatewayError",
|
|
74
|
+
]
|
sdo/_decompress.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Bounded, incremental ``Content-Encoding`` decoding.
|
|
3
|
+
|
|
4
|
+
The scraped target controls the response body, so a few kilobytes of gzip or
|
|
5
|
+
brotli can inflate to gigabytes. httpx's own decoders inflate each network
|
|
6
|
+
chunk in one unbounded call (brotli without an output limit), so the SDK reads
|
|
7
|
+
the raw bytes and decodes them here: every decoder call is capped at ``_STEP``
|
|
8
|
+
bytes of output and the running total is checked after each call. Memory
|
|
9
|
+
therefore never exceeds the ceiling plus one step, whatever the ratio.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import zlib
|
|
15
|
+
from collections.abc import Iterator
|
|
16
|
+
|
|
17
|
+
import brotli
|
|
18
|
+
|
|
19
|
+
#: Output produced per decoder call.
|
|
20
|
+
_STEP = 64 * 1024
|
|
21
|
+
#: The most codings one body is decoded through (urllib3's limit too). Each
|
|
22
|
+
#: stage holds its own decoder state and buffers, and the target controls the
|
|
23
|
+
#: header, so a longer chain is a :class:`DecodeError`, whatever the body.
|
|
24
|
+
MAX_CODINGS = 5
|
|
25
|
+
_GZIP_MAGIC = b"\x1f\x8b"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DecodeError(Exception):
|
|
29
|
+
"""The body is not valid for its declared ``Content-Encoding``."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TooLarge(Exception):
|
|
33
|
+
"""A decoding stage produced more than the ceiling."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _Zlib:
|
|
37
|
+
"""gzip (including concatenated members) and deflate (zlib-wrapped or raw,
|
|
38
|
+
since servers send both)."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, coding: str) -> None:
|
|
41
|
+
self._gzip = coding == "gzip"
|
|
42
|
+
self._d = zlib.decompressobj(16 + zlib.MAX_WBITS) if self._gzip else None
|
|
43
|
+
self._carry = b"" # input held back until there is enough to decide on
|
|
44
|
+
self._fed = False
|
|
45
|
+
|
|
46
|
+
def feed(self, data: bytes, final: bool) -> Iterator[bytes]:
|
|
47
|
+
data = self._carry + data
|
|
48
|
+
self._carry = b""
|
|
49
|
+
self._fed = self._fed or bool(data)
|
|
50
|
+
yield from self._inflate(data, final)
|
|
51
|
+
if final and self._fed and (self._d is None or not self._d.eof):
|
|
52
|
+
# A stream cut short is an error, not a shorter body: the framing
|
|
53
|
+
# said the body was complete, so the payload itself is damaged.
|
|
54
|
+
raise zlib.error("the compressed stream ended before its end marker")
|
|
55
|
+
|
|
56
|
+
def _inflate(self, data: bytes, final: bool) -> Iterator[bytes]:
|
|
57
|
+
if self._d is None:
|
|
58
|
+
# Two bytes tell zlib-wrapped deflate (CM=8, header a multiple of
|
|
59
|
+
# 31) from raw deflate.
|
|
60
|
+
if len(data) < 2 and not final:
|
|
61
|
+
self._carry = data
|
|
62
|
+
return
|
|
63
|
+
wrapped = len(data) >= 2 and data[0] & 0x0F == 8 and (data[0] << 8 | data[1]) % 31 == 0
|
|
64
|
+
self._d = zlib.decompressobj(zlib.MAX_WBITS if wrapped else -zlib.MAX_WBITS)
|
|
65
|
+
d = self._d
|
|
66
|
+
while data:
|
|
67
|
+
if d.eof:
|
|
68
|
+
# After the end of the stream only another gzip member is
|
|
69
|
+
# decoded; anything else is ignored, as httpx and browsers do.
|
|
70
|
+
if not self._gzip:
|
|
71
|
+
return
|
|
72
|
+
if len(data) < 2 and not final:
|
|
73
|
+
self._carry = data
|
|
74
|
+
return
|
|
75
|
+
if data[:2] != _GZIP_MAGIC:
|
|
76
|
+
return
|
|
77
|
+
d = self._d = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
|
78
|
+
out = d.decompress(data, _STEP)
|
|
79
|
+
yield out
|
|
80
|
+
if d.eof:
|
|
81
|
+
data = d.unused_data
|
|
82
|
+
continue
|
|
83
|
+
data = d.unconsumed_tail
|
|
84
|
+
# Output capped with every input byte consumed: zlib may still hold
|
|
85
|
+
# output (e.g. a long back-reference), so drain it in steps too.
|
|
86
|
+
while not data and len(out) == _STEP and not d.eof:
|
|
87
|
+
out = d.decompress(b"", _STEP)
|
|
88
|
+
yield out
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _drain(d: brotli.Decompressor, data: bytes) -> Iterator[bytes]:
|
|
92
|
+
"""Feed ``data`` and yield the output in ``_STEP``-sized pieces."""
|
|
93
|
+
yield d.process(data, output_buffer_limit=_STEP)
|
|
94
|
+
# The limit stops the output buffer from growing; what did not fit stays
|
|
95
|
+
# inside the decoder, so drain it with empty input before the next chunk
|
|
96
|
+
# (can_accept_more_data() alone does not signal it).
|
|
97
|
+
while not d.is_finished():
|
|
98
|
+
out = d.process(b"", output_buffer_limit=_STEP)
|
|
99
|
+
if not out and d.can_accept_more_data():
|
|
100
|
+
break
|
|
101
|
+
yield out
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class _Brotli:
|
|
105
|
+
"""brotli. Bytes after the end of a complete stream are ignored, as for
|
|
106
|
+
gzip. The decoder rejects them when they arrive in the same call as the
|
|
107
|
+
end, and cannot be asked where the stream ended, so that case is replayed
|
|
108
|
+
from the input kept so far (see :meth:`_replay`)."""
|
|
109
|
+
|
|
110
|
+
def __init__(self) -> None:
|
|
111
|
+
self._d = brotli.Decompressor()
|
|
112
|
+
self._seen: list[bytes] = [] # input so far, until the stream ends
|
|
113
|
+
self._emitted = 0 # output yielded so far
|
|
114
|
+
self._done = False
|
|
115
|
+
|
|
116
|
+
def feed(self, data: bytes, final: bool) -> Iterator[bytes]:
|
|
117
|
+
if self._done:
|
|
118
|
+
return # data after the end of a complete stream is ignored
|
|
119
|
+
if final and self._seen and not self._d.is_finished():
|
|
120
|
+
raise brotli.error("the brotli stream ended before its end marker")
|
|
121
|
+
if not data:
|
|
122
|
+
return
|
|
123
|
+
self._seen.append(data)
|
|
124
|
+
try:
|
|
125
|
+
for out in _drain(self._d, data):
|
|
126
|
+
self._emitted += len(out)
|
|
127
|
+
yield out
|
|
128
|
+
except brotli.error:
|
|
129
|
+
yield from self._replay()
|
|
130
|
+
if self._d.is_finished():
|
|
131
|
+
self._done = True
|
|
132
|
+
self._seen = []
|
|
133
|
+
|
|
134
|
+
@staticmethod
|
|
135
|
+
def _state(data: bytes) -> str:
|
|
136
|
+
"""``"open"`` when ``data`` is an unfinished prefix of a valid stream,
|
|
137
|
+
else ``"closed"`` (it ended or is corrupt). Monotonic in the prefix
|
|
138
|
+
length, so the end can be bisected. Bounded: the decoder only rejects
|
|
139
|
+
trailing bytes after producing all its output, which the caller had
|
|
140
|
+
already counted against the cap, and a prefix produces no more."""
|
|
141
|
+
d = brotli.Decompressor()
|
|
142
|
+
try:
|
|
143
|
+
for _ in _drain(d, data):
|
|
144
|
+
pass
|
|
145
|
+
except brotli.error:
|
|
146
|
+
return "closed"
|
|
147
|
+
return "closed" if d.is_finished() else "open"
|
|
148
|
+
|
|
149
|
+
def _replay(self) -> Iterator[bytes]:
|
|
150
|
+
"""The last chunk failed: find the shortest prefix of it after which
|
|
151
|
+
the stream is closed, decode up to there with a fresh decoder (output
|
|
152
|
+
already yielded is skipped), and ignore the rest if the stream ended
|
|
153
|
+
there. A stream that is corrupt instead raises again."""
|
|
154
|
+
head, chunk = b"".join(self._seen[:-1]), self._seen[-1]
|
|
155
|
+
lo, hi = 0, len(chunk)
|
|
156
|
+
while lo < hi:
|
|
157
|
+
mid = (lo + hi) // 2
|
|
158
|
+
if self._state(head + chunk[:mid]) == "open":
|
|
159
|
+
lo = mid + 1
|
|
160
|
+
else:
|
|
161
|
+
hi = mid
|
|
162
|
+
d = brotli.Decompressor()
|
|
163
|
+
skip = self._emitted
|
|
164
|
+
for out in _drain(d, head + chunk[:lo]):
|
|
165
|
+
if skip >= len(out):
|
|
166
|
+
skip -= len(out)
|
|
167
|
+
continue
|
|
168
|
+
out, skip = out[skip:], 0
|
|
169
|
+
self._emitted += len(out)
|
|
170
|
+
yield out
|
|
171
|
+
# Here the stream ended at ``lo``: a corrupt one raised in _drain.
|
|
172
|
+
self._d = d
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
_ALIASES = {"x-gzip": "gzip"}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def codings(header: str | None) -> list[str]:
|
|
179
|
+
"""The codings to undo, in undo order. The header lists them in the order
|
|
180
|
+
they were applied, so they are undone from the last one; ``identity`` is
|
|
181
|
+
skipped, and an unsupported coding stops decoding there (the bytes are then
|
|
182
|
+
returned as they are at that point)."""
|
|
183
|
+
out = []
|
|
184
|
+
for token in reversed((header or "").split(",")):
|
|
185
|
+
coding = token.strip().lower()
|
|
186
|
+
coding = _ALIASES.get(coding, coding)
|
|
187
|
+
if coding in ("", "identity"):
|
|
188
|
+
continue
|
|
189
|
+
if coding not in ("gzip", "deflate", "br"):
|
|
190
|
+
break
|
|
191
|
+
out.append(coding)
|
|
192
|
+
return out
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class BodyDecoder:
|
|
196
|
+
"""Feed raw chunks, get the decoded body. Raises :class:`TooLarge` as soon
|
|
197
|
+
as any stage's output crosses ``limit``, and :class:`DecodeError` on a
|
|
198
|
+
corrupt stream or a chain of more than :data:`MAX_CODINGS` codings."""
|
|
199
|
+
|
|
200
|
+
def __init__(self, content_encoding: str | None, limit: int) -> None:
|
|
201
|
+
self._limit = limit
|
|
202
|
+
chain = codings(content_encoding)
|
|
203
|
+
# Raised on the first feed or finish, where the caller handles it.
|
|
204
|
+
self._error: str | None = None
|
|
205
|
+
if len(chain) > MAX_CODINGS:
|
|
206
|
+
self._error, chain = f"more than {MAX_CODINGS} content codings", []
|
|
207
|
+
self._stages = [_Brotli() if c == "br" else _Zlib(c) for c in chain]
|
|
208
|
+
self._totals = [0] * (len(self._stages) + 1)
|
|
209
|
+
self._parts: list[bytes] = []
|
|
210
|
+
|
|
211
|
+
def _fit(self, index: int, piece: bytes) -> bytes:
|
|
212
|
+
"""``piece`` as far as it fits under the cap for stage ``index``; the
|
|
213
|
+
caller raises :class:`TooLarge` when it was cut, after passing the
|
|
214
|
+
prefix on, so a capped read still holds everything up to the cap."""
|
|
215
|
+
room = self._limit - self._totals[index]
|
|
216
|
+
piece = piece[:room] if len(piece) > room else piece
|
|
217
|
+
self._totals[index] += len(piece)
|
|
218
|
+
return piece
|
|
219
|
+
|
|
220
|
+
def _run(self, index: int, data: bytes, final: bool) -> None:
|
|
221
|
+
if self._error:
|
|
222
|
+
raise DecodeError(self._error)
|
|
223
|
+
if index == len(self._stages):
|
|
224
|
+
kept = self._fit(index, data)
|
|
225
|
+
self._parts.append(kept)
|
|
226
|
+
if len(kept) < len(data):
|
|
227
|
+
raise TooLarge
|
|
228
|
+
return
|
|
229
|
+
try:
|
|
230
|
+
for piece in self._stages[index].feed(data, final):
|
|
231
|
+
kept = self._fit(index, piece)
|
|
232
|
+
if kept:
|
|
233
|
+
self._run(index + 1, kept, False)
|
|
234
|
+
if len(kept) < len(piece):
|
|
235
|
+
raise TooLarge
|
|
236
|
+
except (zlib.error, brotli.error) as exc:
|
|
237
|
+
raise DecodeError(str(exc)) from None
|
|
238
|
+
if final:
|
|
239
|
+
self._run(index + 1, b"", True)
|
|
240
|
+
|
|
241
|
+
def feed(self, chunk: bytes) -> None:
|
|
242
|
+
self._run(0, chunk, False)
|
|
243
|
+
|
|
244
|
+
def finish(self) -> bytes:
|
|
245
|
+
self._run(0, b"", True)
|
|
246
|
+
return self.partial()
|
|
247
|
+
|
|
248
|
+
def partial(self) -> bytes:
|
|
249
|
+
"""The decoded bytes so far (at most ``limit``)."""
|
|
250
|
+
return b"".join(self._parts)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
__all__ = ["BodyDecoder", "DecodeError", "TooLarge", "codings"]
|