sdo-client 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdo/__init__.py ADDED
@@ -0,0 +1,74 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """sdo — the official Python SDK for the Scrape.do web-scraping API.
3
+
4
+ Quickstart::
5
+
6
+ from sdo import Client, ScrapeConfig
7
+
8
+ with Client(token="YOUR_TOKEN") as client: # or set SDO_TOKEN
9
+ result = client.scrape("https://example.com", render=True)
10
+ print(result.status_code, len(result.bytes()))
11
+ print(result.text[:200])
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from ._version import __version__
17
+ from .client import Client
18
+ from .config import ScrapeConfig
19
+ from .errors import (
20
+ SdoAPIError,
21
+ SdoAuthError,
22
+ SdoConcurrencyError,
23
+ SdoConfigError,
24
+ SdoConnectionError,
25
+ SdoError,
26
+ SdoGatewayError,
27
+ SdoResponseTooLargeError,
28
+ SdoTimeoutError,
29
+ SdoTransportError,
30
+ SdoValidationError,
31
+ )
32
+ from .jobs import (
33
+ JOB_SORTS,
34
+ JOB_STATUSES,
35
+ MAX_PLUGIN_PARAMS,
36
+ Account,
37
+ AsyncJobs,
38
+ Job,
39
+ JobPage,
40
+ JobStatus,
41
+ Task,
42
+ TaskResult,
43
+ )
44
+ from .response import ScrapeResult
45
+ from .usage import Usage
46
+
47
+ __all__ = [
48
+ "__version__",
49
+ "Client",
50
+ "ScrapeConfig",
51
+ "ScrapeResult",
52
+ "Usage",
53
+ "AsyncJobs",
54
+ "Job",
55
+ "JobPage",
56
+ "JobStatus",
57
+ "JOB_SORTS",
58
+ "JOB_STATUSES",
59
+ "MAX_PLUGIN_PARAMS",
60
+ "Task",
61
+ "TaskResult",
62
+ "Account",
63
+ "SdoError",
64
+ "SdoConfigError",
65
+ "SdoTransportError",
66
+ "SdoConnectionError",
67
+ "SdoTimeoutError",
68
+ "SdoResponseTooLargeError",
69
+ "SdoAPIError",
70
+ "SdoAuthError",
71
+ "SdoValidationError",
72
+ "SdoConcurrencyError",
73
+ "SdoGatewayError",
74
+ ]
sdo/_decompress.py ADDED
@@ -0,0 +1,253 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Bounded, incremental ``Content-Encoding`` decoding.
3
+
4
+ The scraped target controls the response body, so a few kilobytes of gzip or
5
+ brotli can inflate to gigabytes. httpx's own decoders inflate each network
6
+ chunk in one unbounded call (brotli without an output limit), so the SDK reads
7
+ the raw bytes and decodes them here: every decoder call is capped at ``_STEP``
8
+ bytes of output and the running total is checked after each call. Memory
9
+ therefore never exceeds the ceiling plus one step, whatever the ratio.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import zlib
15
+ from collections.abc import Iterator
16
+
17
+ import brotli
18
+
19
+ #: Output produced per decoder call.
20
+ _STEP = 64 * 1024
21
+ #: The most codings one body is decoded through (urllib3's limit too). Each
22
+ #: stage holds its own decoder state and buffers, and the target controls the
23
+ #: header, so a longer chain is a :class:`DecodeError`, whatever the body.
24
+ MAX_CODINGS = 5
25
+ _GZIP_MAGIC = b"\x1f\x8b"
26
+
27
+
28
+ class DecodeError(Exception):
29
+ """The body is not valid for its declared ``Content-Encoding``."""
30
+
31
+
32
+ class TooLarge(Exception):
33
+ """A decoding stage produced more than the ceiling."""
34
+
35
+
36
+ class _Zlib:
37
+ """gzip (including concatenated members) and deflate (zlib-wrapped or raw,
38
+ since servers send both)."""
39
+
40
+ def __init__(self, coding: str) -> None:
41
+ self._gzip = coding == "gzip"
42
+ self._d = zlib.decompressobj(16 + zlib.MAX_WBITS) if self._gzip else None
43
+ self._carry = b"" # input held back until there is enough to decide on
44
+ self._fed = False
45
+
46
+ def feed(self, data: bytes, final: bool) -> Iterator[bytes]:
47
+ data = self._carry + data
48
+ self._carry = b""
49
+ self._fed = self._fed or bool(data)
50
+ yield from self._inflate(data, final)
51
+ if final and self._fed and (self._d is None or not self._d.eof):
52
+ # A stream cut short is an error, not a shorter body: the framing
53
+ # said the body was complete, so the payload itself is damaged.
54
+ raise zlib.error("the compressed stream ended before its end marker")
55
+
56
+ def _inflate(self, data: bytes, final: bool) -> Iterator[bytes]:
57
+ if self._d is None:
58
+ # Two bytes tell zlib-wrapped deflate (CM=8, header a multiple of
59
+ # 31) from raw deflate.
60
+ if len(data) < 2 and not final:
61
+ self._carry = data
62
+ return
63
+ wrapped = len(data) >= 2 and data[0] & 0x0F == 8 and (data[0] << 8 | data[1]) % 31 == 0
64
+ self._d = zlib.decompressobj(zlib.MAX_WBITS if wrapped else -zlib.MAX_WBITS)
65
+ d = self._d
66
+ while data:
67
+ if d.eof:
68
+ # After the end of the stream only another gzip member is
69
+ # decoded; anything else is ignored, as httpx and browsers do.
70
+ if not self._gzip:
71
+ return
72
+ if len(data) < 2 and not final:
73
+ self._carry = data
74
+ return
75
+ if data[:2] != _GZIP_MAGIC:
76
+ return
77
+ d = self._d = zlib.decompressobj(16 + zlib.MAX_WBITS)
78
+ out = d.decompress(data, _STEP)
79
+ yield out
80
+ if d.eof:
81
+ data = d.unused_data
82
+ continue
83
+ data = d.unconsumed_tail
84
+ # Output capped with every input byte consumed: zlib may still hold
85
+ # output (e.g. a long back-reference), so drain it in steps too.
86
+ while not data and len(out) == _STEP and not d.eof:
87
+ out = d.decompress(b"", _STEP)
88
+ yield out
89
+
90
+
91
+ def _drain(d: brotli.Decompressor, data: bytes) -> Iterator[bytes]:
92
+ """Feed ``data`` and yield the output in ``_STEP``-sized pieces."""
93
+ yield d.process(data, output_buffer_limit=_STEP)
94
+ # The limit stops the output buffer from growing; what did not fit stays
95
+ # inside the decoder, so drain it with empty input before the next chunk
96
+ # (can_accept_more_data() alone does not signal it).
97
+ while not d.is_finished():
98
+ out = d.process(b"", output_buffer_limit=_STEP)
99
+ if not out and d.can_accept_more_data():
100
+ break
101
+ yield out
102
+
103
+
104
+ class _Brotli:
105
+ """brotli. Bytes after the end of a complete stream are ignored, as for
106
+ gzip. The decoder rejects them when they arrive in the same call as the
107
+ end, and cannot be asked where the stream ended, so that case is replayed
108
+ from the input kept so far (see :meth:`_replay`)."""
109
+
110
+ def __init__(self) -> None:
111
+ self._d = brotli.Decompressor()
112
+ self._seen: list[bytes] = [] # input so far, until the stream ends
113
+ self._emitted = 0 # output yielded so far
114
+ self._done = False
115
+
116
+ def feed(self, data: bytes, final: bool) -> Iterator[bytes]:
117
+ if self._done:
118
+ return # data after the end of a complete stream is ignored
119
+ if final and self._seen and not self._d.is_finished():
120
+ raise brotli.error("the brotli stream ended before its end marker")
121
+ if not data:
122
+ return
123
+ self._seen.append(data)
124
+ try:
125
+ for out in _drain(self._d, data):
126
+ self._emitted += len(out)
127
+ yield out
128
+ except brotli.error:
129
+ yield from self._replay()
130
+ if self._d.is_finished():
131
+ self._done = True
132
+ self._seen = []
133
+
134
+ @staticmethod
135
+ def _state(data: bytes) -> str:
136
+ """``"open"`` when ``data`` is an unfinished prefix of a valid stream,
137
+ else ``"closed"`` (it ended or is corrupt). Monotonic in the prefix
138
+ length, so the end can be bisected. Bounded: the decoder only rejects
139
+ trailing bytes after producing all its output, which the caller had
140
+ already counted against the cap, and a prefix produces no more."""
141
+ d = brotli.Decompressor()
142
+ try:
143
+ for _ in _drain(d, data):
144
+ pass
145
+ except brotli.error:
146
+ return "closed"
147
+ return "closed" if d.is_finished() else "open"
148
+
149
+ def _replay(self) -> Iterator[bytes]:
150
+ """The last chunk failed: find the shortest prefix of it after which
151
+ the stream is closed, decode up to there with a fresh decoder (output
152
+ already yielded is skipped), and ignore the rest if the stream ended
153
+ there. A stream that is corrupt instead raises again."""
154
+ head, chunk = b"".join(self._seen[:-1]), self._seen[-1]
155
+ lo, hi = 0, len(chunk)
156
+ while lo < hi:
157
+ mid = (lo + hi) // 2
158
+ if self._state(head + chunk[:mid]) == "open":
159
+ lo = mid + 1
160
+ else:
161
+ hi = mid
162
+ d = brotli.Decompressor()
163
+ skip = self._emitted
164
+ for out in _drain(d, head + chunk[:lo]):
165
+ if skip >= len(out):
166
+ skip -= len(out)
167
+ continue
168
+ out, skip = out[skip:], 0
169
+ self._emitted += len(out)
170
+ yield out
171
+ # Here the stream ended at ``lo``: a corrupt one raised in _drain.
172
+ self._d = d
173
+
174
+
175
+ _ALIASES = {"x-gzip": "gzip"}
176
+
177
+
178
+ def codings(header: str | None) -> list[str]:
179
+ """The codings to undo, in undo order. The header lists them in the order
180
+ they were applied, so they are undone from the last one; ``identity`` is
181
+ skipped, and an unsupported coding stops decoding there (the bytes are then
182
+ returned as they are at that point)."""
183
+ out = []
184
+ for token in reversed((header or "").split(",")):
185
+ coding = token.strip().lower()
186
+ coding = _ALIASES.get(coding, coding)
187
+ if coding in ("", "identity"):
188
+ continue
189
+ if coding not in ("gzip", "deflate", "br"):
190
+ break
191
+ out.append(coding)
192
+ return out
193
+
194
+
195
+ class BodyDecoder:
196
+ """Feed raw chunks, get the decoded body. Raises :class:`TooLarge` as soon
197
+ as any stage's output crosses ``limit``, and :class:`DecodeError` on a
198
+ corrupt stream or a chain of more than :data:`MAX_CODINGS` codings."""
199
+
200
+ def __init__(self, content_encoding: str | None, limit: int) -> None:
201
+ self._limit = limit
202
+ chain = codings(content_encoding)
203
+ # Raised on the first feed or finish, where the caller handles it.
204
+ self._error: str | None = None
205
+ if len(chain) > MAX_CODINGS:
206
+ self._error, chain = f"more than {MAX_CODINGS} content codings", []
207
+ self._stages = [_Brotli() if c == "br" else _Zlib(c) for c in chain]
208
+ self._totals = [0] * (len(self._stages) + 1)
209
+ self._parts: list[bytes] = []
210
+
211
+ def _fit(self, index: int, piece: bytes) -> bytes:
212
+ """``piece`` as far as it fits under the cap for stage ``index``; the
213
+ caller raises :class:`TooLarge` when it was cut, after passing the
214
+ prefix on, so a capped read still holds everything up to the cap."""
215
+ room = self._limit - self._totals[index]
216
+ piece = piece[:room] if len(piece) > room else piece
217
+ self._totals[index] += len(piece)
218
+ return piece
219
+
220
+ def _run(self, index: int, data: bytes, final: bool) -> None:
221
+ if self._error:
222
+ raise DecodeError(self._error)
223
+ if index == len(self._stages):
224
+ kept = self._fit(index, data)
225
+ self._parts.append(kept)
226
+ if len(kept) < len(data):
227
+ raise TooLarge
228
+ return
229
+ try:
230
+ for piece in self._stages[index].feed(data, final):
231
+ kept = self._fit(index, piece)
232
+ if kept:
233
+ self._run(index + 1, kept, False)
234
+ if len(kept) < len(piece):
235
+ raise TooLarge
236
+ except (zlib.error, brotli.error) as exc:
237
+ raise DecodeError(str(exc)) from None
238
+ if final:
239
+ self._run(index + 1, b"", True)
240
+
241
+ def feed(self, chunk: bytes) -> None:
242
+ self._run(0, chunk, False)
243
+
244
+ def finish(self) -> bytes:
245
+ self._run(0, b"", True)
246
+ return self.partial()
247
+
248
+ def partial(self) -> bytes:
249
+ """The decoded bytes so far (at most ``limit``)."""
250
+ return b"".join(self._parts)
251
+
252
+
253
+ __all__ = ["BodyDecoder", "DecodeError", "TooLarge", "codings"]