sdo-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdo/__init__.py +74 -0
- sdo/_decompress.py +253 -0
- sdo/_encoding.py +739 -0
- sdo/_json.py +50 -0
- sdo/_logging.py +93 -0
- sdo/_secret.py +74 -0
- sdo/_transport.py +639 -0
- sdo/_urls.py +80 -0
- sdo/_version.py +8 -0
- sdo/_wire.py +126 -0
- sdo/client.py +459 -0
- sdo/config.py +452 -0
- sdo/errors.py +247 -0
- sdo/jobs.py +490 -0
- sdo/plugins.py +143 -0
- sdo/py.typed +0 -0
- sdo/response.py +177 -0
- sdo/usage.py +62 -0
- sdo_client-0.1.0.dist-info/METADATA +377 -0
- sdo_client-0.1.0.dist-info/RECORD +23 -0
- sdo_client-0.1.0.dist-info/WHEEL +4 -0
- sdo_client-0.1.0.dist-info/licenses/LICENSE +201 -0
- sdo_client-0.1.0.dist-info/licenses/NOTICE +8 -0
sdo/_json.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""The one JSON reader for bodies the SDK parses: ``ScrapeResult.json()``,
|
|
3
|
+
plugin responses, async-job responses and error envelopes.
|
|
4
|
+
|
|
5
|
+
UTF-8 with invalid bytes replaced by U+FFFD, a leading BOM ignored; ``NaN``,
|
|
6
|
+
``Infinity`` and numbers that overflow a double are rejected (they are not JSON,
|
|
7
|
+
and the JS and Go SDKs refuse them); any failure, nesting too deep for the
|
|
8
|
+
parser included, raises :class:`ValueError`.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _no_constant(name: str) -> Any:
|
|
19
|
+
raise ValueError(f"{name} is not a JSON number")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _float(text: str) -> float:
|
|
23
|
+
value = float(text)
|
|
24
|
+
if not math.isfinite(value):
|
|
25
|
+
raise ValueError(f"{text} overflows a double")
|
|
26
|
+
return value
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _int(text: str) -> int:
|
|
30
|
+
value = int(text)
|
|
31
|
+
float(value) # OverflowError when it cannot be a double
|
|
32
|
+
return value
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def loads(body: bytes, **kwargs: Any) -> Any:
|
|
36
|
+
"""Parse ``body``; raise :class:`ValueError` when it is not valid JSON."""
|
|
37
|
+
text = body.decode("utf-8", errors="replace").removeprefix("")
|
|
38
|
+
options: dict[str, Any] = {
|
|
39
|
+
"parse_constant": _no_constant,
|
|
40
|
+
"parse_float": _float,
|
|
41
|
+
"parse_int": _int,
|
|
42
|
+
}
|
|
43
|
+
options.update(kwargs)
|
|
44
|
+
try:
|
|
45
|
+
return json.loads(text, **options)
|
|
46
|
+
except (RecursionError, OverflowError) as exc:
|
|
47
|
+
raise ValueError(f"{type(exc).__name__}: {exc}") from None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
__all__ = ["loads"]
|
sdo/_logging.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Keep the token out of httpx/httpcore log records.
|
|
3
|
+
|
|
4
|
+
httpx logs every request at INFO as ``HTTP Request: GET <full URL>``, and the
|
|
5
|
+
sync API carries the token in that URL's query string, so any application that
|
|
6
|
+
logs at INFO (``logging.basicConfig(level=logging.INFO)``, Celery ``-l info``,
|
|
7
|
+
Scrapy's DEBUG default) would write the live token to its logs. A filter on
|
|
8
|
+
those loggers rewrites each record through the same redaction as error
|
|
9
|
+
messages, plus a literal scrub of every token a live client holds.
|
|
10
|
+
|
|
11
|
+
Filters on a logger do not apply to its children, so each logger httpcore
|
|
12
|
+
(and hpack, for HTTP/2) creates is covered by name.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
import threading
|
|
19
|
+
|
|
20
|
+
from .errors import redact_secrets
|
|
21
|
+
|
|
22
|
+
_LOGGERS = (
|
|
23
|
+
"httpx",
|
|
24
|
+
"httpcore",
|
|
25
|
+
"httpcore.connection",
|
|
26
|
+
"httpcore.http11",
|
|
27
|
+
"httpcore.http2",
|
|
28
|
+
"httpcore.proxy",
|
|
29
|
+
"httpcore.socks",
|
|
30
|
+
# HTTP/2 (httpx[http2]): hpack logs header blocks and evicted header-table
|
|
31
|
+
# entries, X-Token among them, at DEBUG. h2 itself logs through a no-op
|
|
32
|
+
# logger unless one is configured, and httpcore configures none.
|
|
33
|
+
"hpack",
|
|
34
|
+
"hpack.hpack",
|
|
35
|
+
"hpack.table",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
_lock = threading.Lock()
|
|
39
|
+
_tokens: dict[str, int] = {}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def register_token(token: str) -> None:
|
|
43
|
+
"""Scrub ``token`` literally from log records while a client uses it."""
|
|
44
|
+
with _lock:
|
|
45
|
+
_tokens[token] = _tokens.get(token, 0) + 1
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def unregister_token(token: str) -> None:
|
|
49
|
+
with _lock:
|
|
50
|
+
count = _tokens.get(token, 0) - 1
|
|
51
|
+
if count > 0:
|
|
52
|
+
_tokens[token] = count
|
|
53
|
+
else:
|
|
54
|
+
_tokens.pop(token, None)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def redact_known(text: str) -> str:
|
|
58
|
+
"""Redact credential shapes and every registered token from ``text``."""
|
|
59
|
+
with _lock:
|
|
60
|
+
tokens = sorted(_tokens, key=len, reverse=True)
|
|
61
|
+
for token in tokens:
|
|
62
|
+
text = redact_secrets(text, token)
|
|
63
|
+
return redact_secrets(text)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class RedactingFilter(logging.Filter):
|
|
67
|
+
"""Rewrites a record's message with credentials masked."""
|
|
68
|
+
|
|
69
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
70
|
+
try:
|
|
71
|
+
message = record.getMessage()
|
|
72
|
+
broken = False
|
|
73
|
+
except Exception: # a malformed record: keep its parts, masked
|
|
74
|
+
message = f"{record.msg} {record.args!r}"
|
|
75
|
+
broken = True
|
|
76
|
+
masked = redact_known(message)
|
|
77
|
+
if broken or masked != message:
|
|
78
|
+
record.msg = masked
|
|
79
|
+
record.args = None
|
|
80
|
+
return True
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def install() -> None:
|
|
84
|
+
"""Attach the filter once per logger (idempotent across re-imports)."""
|
|
85
|
+
for name in _LOGGERS:
|
|
86
|
+
logger = logging.getLogger(name)
|
|
87
|
+
if not any(type(f).__name__ == "RedactingFilter" for f in logger.filters):
|
|
88
|
+
logger.addFilter(RedactingFilter())
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
install()
|
|
92
|
+
|
|
93
|
+
__all__ = ["RedactingFilter", "install", "redact_known", "register_token", "unregister_token"]
|
sdo/_secret.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""The token, held so that no stack frame shows it.
|
|
3
|
+
|
|
4
|
+
Error trackers (Sentry's ``include_local_variables``, pytest ``--showlocals``,
|
|
5
|
+
rich tracebacks, the Django debug page) record the ``repr()`` of every local
|
|
6
|
+
variable of every frame an exception passes through. So the token never sits in
|
|
7
|
+
a plain local on a path that can raise: it lives in a :class:`Secret` whose
|
|
8
|
+
``repr`` and ``str`` are masked, and the raw value is only read inside small
|
|
9
|
+
helpers that return before anything is raised.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
#: Tokens are trimmed of ASCII whitespace (WHATWG: tab, LF, FF, CR, space).
|
|
18
|
+
_ASCII_WHITESPACE = " \t\n\f\r"
|
|
19
|
+
#: What is left must be visible ASCII: it travels in a query string and a header.
|
|
20
|
+
_TOKEN_RE = re.compile(r"[\x21-\x7e]+")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Secret:
|
|
24
|
+
"""A credential whose ``repr`` and ``str`` never show it."""
|
|
25
|
+
|
|
26
|
+
__slots__ = ("_value",)
|
|
27
|
+
|
|
28
|
+
def __init__(self, value: object) -> None:
|
|
29
|
+
# Any caller argument is wrapped as-is, before it is validated, so even
|
|
30
|
+
# an invalid token never sits in a plain local.
|
|
31
|
+
self._value = value
|
|
32
|
+
|
|
33
|
+
def reveal(self) -> str:
|
|
34
|
+
value = self._value
|
|
35
|
+
return value if isinstance(value, str) else ""
|
|
36
|
+
|
|
37
|
+
def __repr__(self) -> str:
|
|
38
|
+
return "Secret('***')"
|
|
39
|
+
|
|
40
|
+
__str__ = __repr__
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _clean(raw: object) -> str | None:
|
|
44
|
+
"""The trimmed token, ``""`` when blank, or ``None`` when it is invalid."""
|
|
45
|
+
if not isinstance(raw, str):
|
|
46
|
+
return None
|
|
47
|
+
value = raw.strip(_ASCII_WHITESPACE)
|
|
48
|
+
if value and not _TOKEN_RE.fullmatch(value):
|
|
49
|
+
return None
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_token(arg: Secret) -> Secret | str:
|
|
54
|
+
"""The token from the argument, else ``SDO_TOKEN``, wrapped; or the text of
|
|
55
|
+
the configuration error to raise. A blank argument means "not given".
|
|
56
|
+
|
|
57
|
+
Returns rather than raises, so the frame that raises holds no raw value.
|
|
58
|
+
"""
|
|
59
|
+
raw = arg._value
|
|
60
|
+
if raw is not None:
|
|
61
|
+
value = _clean(raw)
|
|
62
|
+
if value is None:
|
|
63
|
+
return "The token must be a string of visible ASCII characters."
|
|
64
|
+
if value:
|
|
65
|
+
return Secret(value)
|
|
66
|
+
env = _clean(os.environ.get("SDO_TOKEN", ""))
|
|
67
|
+
if env is None:
|
|
68
|
+
return "SDO_TOKEN must be a string of visible ASCII characters."
|
|
69
|
+
if env:
|
|
70
|
+
return Secret(env)
|
|
71
|
+
return "No token provided. Pass Client(token=...) or set SDO_TOKEN."
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
__all__ = ["Secret", "resolve_token"]
|