constile-gate 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- constile_gate/__init__.py +23 -0
- constile_gate/_adapter.py +177 -0
- constile_gate/_charset.py +191 -0
- constile_gate/_js.py +123 -0
- constile_gate/agents.py +101 -0
- constile_gate/asgi.py +318 -0
- constile_gate/core.py +784 -0
- constile_gate/django.py +154 -0
- constile_gate/driver.py +196 -0
- constile_gate/fastapi.py +14 -0
- constile_gate/flask.py +52 -0
- constile_gate/page_meta.py +88 -0
- constile_gate/py.typed +0 -0
- constile_gate/starlette.py +15 -0
- constile_gate/transport.py +43 -0
- constile_gate/wsgi.py +262 -0
- constile_gate-0.6.0.dist-info/METADATA +257 -0
- constile_gate-0.6.0.dist-info/RECORD +21 -0
- constile_gate-0.6.0.dist-info/WHEEL +5 -0
- constile_gate-0.6.0.dist-info/licenses/LICENSE +21 -0
- constile_gate-0.6.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The Constile gate for Python sites: a port of the reference gate with
|
|
3
|
+
WSGI and ASGI middleware, and wiring for Django, Flask, and FastAPI.
|
|
4
|
+
"""
|
|
5
|
+
from .agents import classify_client, signature_agent_origin
|
|
6
|
+
from .core import GATE_VERSION, GateOptions, GateRequest, Headers, clear_gate_cache, handle_gate, ip_prefix
|
|
7
|
+
from .page_meta import extract_page_meta
|
|
8
|
+
|
|
9
|
+
__version__ = "0.6.0"
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"GATE_VERSION",
|
|
13
|
+
"GateOptions",
|
|
14
|
+
"GateRequest",
|
|
15
|
+
"Headers",
|
|
16
|
+
"__version__",
|
|
17
|
+
"classify_client",
|
|
18
|
+
"clear_gate_cache",
|
|
19
|
+
"extract_page_meta",
|
|
20
|
+
"handle_gate",
|
|
21
|
+
"ip_prefix",
|
|
22
|
+
"signature_agent_origin",
|
|
23
|
+
]
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""What every adapter shares: settings, the visitor's address, and URL building."""
|
|
2
|
+
import ipaddress
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from http import HTTPStatus
|
|
7
|
+
from typing import Tuple
|
|
8
|
+
from urllib.parse import quote
|
|
9
|
+
|
|
10
|
+
from .core import CONFIG_RETRY_S, GATE_VERSION, GateOptions, ip_prefix
|
|
11
|
+
from .transport import DEFAULT_TIMEOUT
|
|
12
|
+
|
|
13
|
+
log = logging.getLogger("constile_gate")
|
|
14
|
+
|
|
15
|
+
MISSING_SETTINGS = "%s: CONSTILE_SITE_ID, CONSTILE_SITE_SECRET, or CONSTILE_NETWORK_URL is missing; passing everything through" % GATE_VERSION
|
|
16
|
+
|
|
17
|
+
# Characters a URL path keeps as they are (everything the WHATWG URL parser doesn't percent-encode in a path).
|
|
18
|
+
_PATH_SAFE = "/!$&'()*+,;=:@-._~[]^|"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class GateSettings:
|
|
23
|
+
site_id: str
|
|
24
|
+
# Kept out of the repr, so an error tracker that records local variables never captures it.
|
|
25
|
+
secret: str = field(repr=False)
|
|
26
|
+
network_url: str
|
|
27
|
+
trusted_proxies: Tuple[object, ...] = ()
|
|
28
|
+
timeout: float = DEFAULT_TIMEOUT
|
|
29
|
+
retry_after: float = CONFIG_RETRY_S
|
|
30
|
+
|
|
31
|
+
def options(self, platform):
|
|
32
|
+
return GateOptions(
|
|
33
|
+
site_id=self.site_id,
|
|
34
|
+
secret=self.secret,
|
|
35
|
+
network_url=self.network_url,
|
|
36
|
+
platform=platform,
|
|
37
|
+
config_retry_s=self.retry_after,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def parse_trusted_proxies(value):
|
|
42
|
+
"""A list of addresses and networks (CIDR), or a string of them separated by commas or spaces."""
|
|
43
|
+
if value is None:
|
|
44
|
+
return ()
|
|
45
|
+
items = value.replace(",", " ").split() if isinstance(value, str) else list(value)
|
|
46
|
+
networks = []
|
|
47
|
+
for item in items:
|
|
48
|
+
try:
|
|
49
|
+
networks.append(ipaddress.ip_network(str(item).strip(), strict=False))
|
|
50
|
+
except ValueError:
|
|
51
|
+
raise ValueError("constile gate: %r in the trusted proxies isn't an address or a network" % (item,)) from None
|
|
52
|
+
return tuple(networks)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def resolve_settings(site_id=None, secret=None, network_url=None, trusted_proxies=None, timeout=None, retry_after=None, source=None):
|
|
56
|
+
"""
|
|
57
|
+
Settings from the arguments, falling back to CONSTILE_SITE_ID, CONSTILE_SITE_SECRET,
|
|
58
|
+
CONSTILE_NETWORK_URL, CONSTILE_TRUSTED_PROXIES, CONSTILE_TIMEOUT, and CONSTILE_RETRY_AFTER.
|
|
59
|
+
Returns None when the site id, secret, or network URL is missing.
|
|
60
|
+
"""
|
|
61
|
+
env = os.environ if source is None else source
|
|
62
|
+
# Read the settings that can be malformed before the secret, so the secret is never a local
|
|
63
|
+
# variable of a frame that raises.
|
|
64
|
+
proxies = parse_trusted_proxies(trusted_proxies if trusted_proxies is not None else env.get("CONSTILE_TRUSTED_PROXIES"))
|
|
65
|
+
timeout = float(timeout if timeout is not None else env.get("CONSTILE_TIMEOUT") or DEFAULT_TIMEOUT)
|
|
66
|
+
retry_after = float(retry_after if retry_after is not None else env.get("CONSTILE_RETRY_AFTER") or CONFIG_RETRY_S)
|
|
67
|
+
site_id = site_id or env.get("CONSTILE_SITE_ID")
|
|
68
|
+
secret = secret or env.get("CONSTILE_SITE_SECRET")
|
|
69
|
+
network_url = network_url or env.get("CONSTILE_NETWORK_URL")
|
|
70
|
+
if not site_id or not secret or not network_url:
|
|
71
|
+
return None
|
|
72
|
+
return GateSettings(
|
|
73
|
+
site_id=str(site_id),
|
|
74
|
+
secret=str(secret),
|
|
75
|
+
network_url=str(network_url),
|
|
76
|
+
trusted_proxies=proxies,
|
|
77
|
+
timeout=timeout,
|
|
78
|
+
retry_after=retry_after,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _address(text):
|
|
83
|
+
"""An address as ipaddress reads it, without a port, or None. IPv4-mapped IPv6 addresses become IPv4."""
|
|
84
|
+
text = (text or "").strip()
|
|
85
|
+
if text.startswith("[") and "]" in text:
|
|
86
|
+
text = text[1 : text.index("]")]
|
|
87
|
+
elif text.count(":") == 1:
|
|
88
|
+
text = text.split(":", 1)[0]
|
|
89
|
+
try:
|
|
90
|
+
addr = ipaddress.ip_address(text)
|
|
91
|
+
except ValueError:
|
|
92
|
+
return None
|
|
93
|
+
if addr.version == 6 and addr.ipv4_mapped is not None:
|
|
94
|
+
return addr.ipv4_mapped
|
|
95
|
+
return addr
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _trusted(addr, networks):
|
|
99
|
+
return addr is not None and any(addr.version == n.version and addr in n for n in networks)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def visitor_address(peer, forwarded_for, trusted_proxies):
|
|
103
|
+
"""
|
|
104
|
+
The visitor's address. It's the connection's peer, unless the peer is a
|
|
105
|
+
trusted proxy: then X-Forwarded-For is read from the right, skipping trusted
|
|
106
|
+
proxies, and the first address that isn't one is the visitor. Entries to the
|
|
107
|
+
left of that (which the visitor could have written) are never read.
|
|
108
|
+
"""
|
|
109
|
+
addr = _address(peer)
|
|
110
|
+
if addr is None:
|
|
111
|
+
return None
|
|
112
|
+
if not trusted_proxies or not _trusted(addr, trusted_proxies):
|
|
113
|
+
return str(addr)
|
|
114
|
+
hops = [h for h in (forwarded_for or "").split(",") if h.strip()]
|
|
115
|
+
while hops and _trusted(addr, trusted_proxies):
|
|
116
|
+
addr = _address(hops.pop())
|
|
117
|
+
if addr is None:
|
|
118
|
+
return None
|
|
119
|
+
return str(addr)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def signals_for(peer, forwarded_for, trusted_proxies):
|
|
123
|
+
"""Off Cloudflare the gate can't see the visitor's network number or country, only the address."""
|
|
124
|
+
ip = visitor_address(peer, forwarded_for, trusted_proxies)
|
|
125
|
+
return {"asn": None, "as_org": None, "country": None, "ip_prefix": ip_prefix(ip), "ip": ip}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def quote_path(raw):
|
|
129
|
+
"""A path as a URL carries it, from the raw bytes a server decoded."""
|
|
130
|
+
return quote(raw, safe=_PATH_SAFE) or "/"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def host_with_port(host, port, scheme):
|
|
134
|
+
default = {"http": "80", "https": "443"}.get(scheme)
|
|
135
|
+
port = str(port) if port is not None else None
|
|
136
|
+
if ":" in host and not host.startswith("["):
|
|
137
|
+
host = "[%s]" % host
|
|
138
|
+
return host if not port or port == default else "%s:%s" % (host, port)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def status_line(status):
|
|
142
|
+
try:
|
|
143
|
+
phrase = HTTPStatus(status).phrase
|
|
144
|
+
except ValueError:
|
|
145
|
+
phrase = "Unknown"
|
|
146
|
+
return "%d %s" % (status, phrase)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
_ON = ("1", "true", "yes", "on")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def is_on(value):
|
|
153
|
+
"""True for True, or for 1, true, yes, or on in any case."""
|
|
154
|
+
if isinstance(value, bool):
|
|
155
|
+
return value
|
|
156
|
+
return str(value or "").strip().lower() in _ON
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def disabled(option=None, source=None):
|
|
160
|
+
"""
|
|
161
|
+
Whether the gate is turned off: the disabled option when given, otherwise
|
|
162
|
+
CONSTILE_DISABLED (1, true, yes, or on). Read on every request, so it takes
|
|
163
|
+
effect without a restart wherever the environment can change.
|
|
164
|
+
"""
|
|
165
|
+
if option is not None:
|
|
166
|
+
return is_on(option)
|
|
167
|
+
env = os.environ if source is None else source
|
|
168
|
+
return is_on(env.get("CONSTILE_DISABLED"))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def warn_missing():
|
|
172
|
+
log.warning(MISSING_SETTINGS)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def head_only(method):
|
|
176
|
+
return method.upper() == "HEAD"
|
|
177
|
+
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Reads a page's bytes as text the way the reference gate does with
|
|
3
|
+
`new TextDecoder(charset)`: in the charset its Content-Type names, by the
|
|
4
|
+
labels of the WHATWG Encoding Standard, and as UTF-8 when it names none or one
|
|
5
|
+
the standard doesn't know. Bytes that don't decode become U+FFFD.
|
|
6
|
+
|
|
7
|
+
- UTF-8, UTF-16, x-user-defined, and the 27 single-byte encodings decode
|
|
8
|
+
exactly as the standard says. The single-byte tables are Python's codecs with
|
|
9
|
+
the standard's differences applied, checked against its index files.
|
|
10
|
+
- The Chinese, Japanese, and Korean encodings use Python's closest codecs.
|
|
11
|
+
Well-formed text decodes the same; malformed bytes may not. JavaScript
|
|
12
|
+
runtimes differ from the standard there too.
|
|
13
|
+
|
|
14
|
+
LABELS is the standard's label table (https://encoding.spec.whatwg.org/encodings.json).
|
|
15
|
+
"""
|
|
16
|
+
import codecs
|
|
17
|
+
import re
|
|
18
|
+
|
|
19
|
+
from ._js import WS_CLASS
|
|
20
|
+
|
|
21
|
+
_CHARSET = re.compile(r"charset" + WS_CLASS + r"*=" + WS_CLASS + r'*"?([\w.:-]+)', re.IGNORECASE | re.ASCII)
|
|
22
|
+
|
|
23
|
+
LABELS = { # noqa: E501 - generated from encodings.json
|
|
24
|
+
'unicode-1-1-utf-8': 'UTF-8', 'unicode11utf8': 'UTF-8', 'unicode20utf8': 'UTF-8', 'utf-8': 'UTF-8',
|
|
25
|
+
'utf8': 'UTF-8', 'x-unicode20utf8': 'UTF-8', '866': 'IBM866', 'cp866': 'IBM866', 'csibm866': 'IBM866',
|
|
26
|
+
'ibm866': 'IBM866', 'csisolatin2': 'ISO-8859-2', 'iso-8859-2': 'ISO-8859-2', 'iso-ir-101': 'ISO-8859-2',
|
|
27
|
+
'iso8859-2': 'ISO-8859-2', 'iso88592': 'ISO-8859-2', 'iso_8859-2': 'ISO-8859-2', 'iso_8859-2:1987': 'ISO-8859-2',
|
|
28
|
+
'l2': 'ISO-8859-2', 'latin2': 'ISO-8859-2', 'csisolatin3': 'ISO-8859-3', 'iso-8859-3': 'ISO-8859-3',
|
|
29
|
+
'iso-ir-109': 'ISO-8859-3', 'iso8859-3': 'ISO-8859-3', 'iso88593': 'ISO-8859-3', 'iso_8859-3': 'ISO-8859-3',
|
|
30
|
+
'iso_8859-3:1988': 'ISO-8859-3', 'l3': 'ISO-8859-3', 'latin3': 'ISO-8859-3', 'csisolatin4': 'ISO-8859-4',
|
|
31
|
+
'iso-8859-4': 'ISO-8859-4', 'iso-ir-110': 'ISO-8859-4', 'iso8859-4': 'ISO-8859-4', 'iso88594': 'ISO-8859-4',
|
|
32
|
+
'iso_8859-4': 'ISO-8859-4', 'iso_8859-4:1988': 'ISO-8859-4', 'l4': 'ISO-8859-4', 'latin4': 'ISO-8859-4',
|
|
33
|
+
'csisolatincyrillic': 'ISO-8859-5', 'cyrillic': 'ISO-8859-5', 'iso-8859-5': 'ISO-8859-5',
|
|
34
|
+
'iso-ir-144': 'ISO-8859-5', 'iso8859-5': 'ISO-8859-5', 'iso88595': 'ISO-8859-5', 'iso_8859-5': 'ISO-8859-5',
|
|
35
|
+
'iso_8859-5:1988': 'ISO-8859-5', 'arabic': 'ISO-8859-6', 'asmo-708': 'ISO-8859-6', 'csiso88596e': 'ISO-8859-6',
|
|
36
|
+
'csiso88596i': 'ISO-8859-6', 'csisolatinarabic': 'ISO-8859-6', 'ecma-114': 'ISO-8859-6',
|
|
37
|
+
'iso-8859-6': 'ISO-8859-6', 'iso-8859-6-e': 'ISO-8859-6', 'iso-8859-6-i': 'ISO-8859-6',
|
|
38
|
+
'iso-ir-127': 'ISO-8859-6', 'iso8859-6': 'ISO-8859-6', 'iso88596': 'ISO-8859-6', 'iso_8859-6': 'ISO-8859-6',
|
|
39
|
+
'iso_8859-6:1987': 'ISO-8859-6', 'csisolatingreek': 'ISO-8859-7', 'ecma-118': 'ISO-8859-7',
|
|
40
|
+
'elot_928': 'ISO-8859-7', 'greek': 'ISO-8859-7', 'greek8': 'ISO-8859-7', 'iso-8859-7': 'ISO-8859-7',
|
|
41
|
+
'iso-ir-126': 'ISO-8859-7', 'iso8859-7': 'ISO-8859-7', 'iso88597': 'ISO-8859-7', 'iso_8859-7': 'ISO-8859-7',
|
|
42
|
+
'iso_8859-7:1987': 'ISO-8859-7', 'sun_eu_greek': 'ISO-8859-7', 'csiso88598e': 'ISO-8859-8',
|
|
43
|
+
'csisolatinhebrew': 'ISO-8859-8', 'hebrew': 'ISO-8859-8', 'iso-8859-8': 'ISO-8859-8',
|
|
44
|
+
'iso-8859-8-e': 'ISO-8859-8', 'iso-ir-138': 'ISO-8859-8', 'iso8859-8': 'ISO-8859-8', 'iso88598': 'ISO-8859-8',
|
|
45
|
+
'iso_8859-8': 'ISO-8859-8', 'iso_8859-8:1988': 'ISO-8859-8', 'visual': 'ISO-8859-8',
|
|
46
|
+
'csiso88598i': 'ISO-8859-8-I', 'iso-8859-8-i': 'ISO-8859-8-I', 'logical': 'ISO-8859-8-I',
|
|
47
|
+
'csisolatin6': 'ISO-8859-10', 'iso-8859-10': 'ISO-8859-10', 'iso-ir-157': 'ISO-8859-10',
|
|
48
|
+
'iso8859-10': 'ISO-8859-10', 'iso885910': 'ISO-8859-10', 'l6': 'ISO-8859-10', 'latin6': 'ISO-8859-10',
|
|
49
|
+
'iso-8859-13': 'ISO-8859-13', 'iso8859-13': 'ISO-8859-13', 'iso885913': 'ISO-8859-13',
|
|
50
|
+
'iso-8859-14': 'ISO-8859-14', 'iso8859-14': 'ISO-8859-14', 'iso885914': 'ISO-8859-14',
|
|
51
|
+
'csisolatin9': 'ISO-8859-15', 'iso-8859-15': 'ISO-8859-15', 'iso8859-15': 'ISO-8859-15',
|
|
52
|
+
'iso885915': 'ISO-8859-15', 'iso_8859-15': 'ISO-8859-15', 'l9': 'ISO-8859-15', 'iso-8859-16': 'ISO-8859-16',
|
|
53
|
+
'cskoi8r': 'KOI8-R', 'koi': 'KOI8-R', 'koi8': 'KOI8-R', 'koi8-r': 'KOI8-R', 'koi8_r': 'KOI8-R',
|
|
54
|
+
'koi8-ru': 'KOI8-U', 'koi8-u': 'KOI8-U', 'csmacintosh': 'macintosh', 'mac': 'macintosh', 'macintosh': 'macintosh',
|
|
55
|
+
'x-mac-roman': 'macintosh', 'dos-874': 'windows-874', 'iso-8859-11': 'windows-874', 'iso8859-11': 'windows-874',
|
|
56
|
+
'iso885911': 'windows-874', 'tis-620': 'windows-874', 'windows-874': 'windows-874', 'cp1250': 'windows-1250',
|
|
57
|
+
'windows-1250': 'windows-1250', 'x-cp1250': 'windows-1250', 'cp1251': 'windows-1251',
|
|
58
|
+
'windows-1251': 'windows-1251', 'x-cp1251': 'windows-1251', 'ansi_x3.4-1968': 'windows-1252',
|
|
59
|
+
'ascii': 'windows-1252', 'cp1252': 'windows-1252', 'cp819': 'windows-1252', 'csisolatin1': 'windows-1252',
|
|
60
|
+
'ibm819': 'windows-1252', 'iso-8859-1': 'windows-1252', 'iso-ir-100': 'windows-1252', 'iso8859-1': 'windows-1252',
|
|
61
|
+
'iso88591': 'windows-1252', 'iso_8859-1': 'windows-1252', 'iso_8859-1:1987': 'windows-1252', 'l1': 'windows-1252',
|
|
62
|
+
'latin1': 'windows-1252', 'us-ascii': 'windows-1252', 'windows-1252': 'windows-1252', 'x-cp1252': 'windows-1252',
|
|
63
|
+
'cp1253': 'windows-1253', 'windows-1253': 'windows-1253', 'x-cp1253': 'windows-1253', 'cp1254': 'windows-1254',
|
|
64
|
+
'csisolatin5': 'windows-1254', 'iso-8859-9': 'windows-1254', 'iso-ir-148': 'windows-1254',
|
|
65
|
+
'iso8859-9': 'windows-1254', 'iso88599': 'windows-1254', 'iso_8859-9': 'windows-1254',
|
|
66
|
+
'iso_8859-9:1989': 'windows-1254', 'l5': 'windows-1254', 'latin5': 'windows-1254', 'windows-1254': 'windows-1254',
|
|
67
|
+
'x-cp1254': 'windows-1254', 'cp1255': 'windows-1255', 'windows-1255': 'windows-1255', 'x-cp1255': 'windows-1255',
|
|
68
|
+
'cp1256': 'windows-1256', 'windows-1256': 'windows-1256', 'x-cp1256': 'windows-1256', 'cp1257': 'windows-1257',
|
|
69
|
+
'windows-1257': 'windows-1257', 'x-cp1257': 'windows-1257', 'cp1258': 'windows-1258',
|
|
70
|
+
'windows-1258': 'windows-1258', 'x-cp1258': 'windows-1258', 'x-mac-cyrillic': 'x-mac-cyrillic',
|
|
71
|
+
'x-mac-ukrainian': 'x-mac-cyrillic', 'chinese': 'GBK', 'csgb2312': 'GBK', 'csiso58gb231280': 'GBK',
|
|
72
|
+
'gb2312': 'GBK', 'gb_2312': 'GBK', 'gb_2312-80': 'GBK', 'gbk': 'GBK', 'iso-ir-58': 'GBK', 'x-gbk': 'GBK',
|
|
73
|
+
'gb18030': 'gb18030', 'big5': 'Big5', 'big5-hkscs': 'Big5', 'cn-big5': 'Big5', 'csbig5': 'Big5',
|
|
74
|
+
'x-x-big5': 'Big5', 'cseucpkdfmtjapanese': 'EUC-JP', 'euc-jp': 'EUC-JP', 'x-euc-jp': 'EUC-JP',
|
|
75
|
+
'csiso2022jp': 'ISO-2022-JP', 'iso-2022-jp': 'ISO-2022-JP', 'csshiftjis': 'Shift_JIS', 'ms932': 'Shift_JIS',
|
|
76
|
+
'ms_kanji': 'Shift_JIS', 'shift-jis': 'Shift_JIS', 'shift_jis': 'Shift_JIS', 'sjis': 'Shift_JIS',
|
|
77
|
+
'windows-31j': 'Shift_JIS', 'x-sjis': 'Shift_JIS', 'cseuckr': 'EUC-KR', 'csksc56011987': 'EUC-KR',
|
|
78
|
+
'euc-kr': 'EUC-KR', 'iso-ir-149': 'EUC-KR', 'korean': 'EUC-KR', 'ks_c_5601-1987': 'EUC-KR',
|
|
79
|
+
'ks_c_5601-1989': 'EUC-KR', 'ksc5601': 'EUC-KR', 'ksc_5601': 'EUC-KR', 'windows-949': 'EUC-KR',
|
|
80
|
+
'csiso2022kr': 'replacement', 'hz-gb-2312': 'replacement', 'iso-2022-cn': 'replacement',
|
|
81
|
+
'iso-2022-cn-ext': 'replacement', 'iso-2022-kr': 'replacement', 'replacement': 'replacement',
|
|
82
|
+
'unicodefffe': 'UTF-16BE', 'utf-16be': 'UTF-16BE', 'csunicode': 'UTF-16LE', 'iso-10646-ucs-2': 'UTF-16LE',
|
|
83
|
+
'ucs-2': 'UTF-16LE', 'unicode': 'UTF-16LE', 'unicodefeff': 'UTF-16LE', 'utf-16': 'UTF-16LE',
|
|
84
|
+
'utf-16le': 'UTF-16LE', 'x-user-defined': 'x-user-defined',
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
_SINGLE_BYTE = {
|
|
88
|
+
"IBM866": "cp866",
|
|
89
|
+
"ISO-8859-2": "iso8859_2",
|
|
90
|
+
"ISO-8859-3": "iso8859_3",
|
|
91
|
+
"ISO-8859-4": "iso8859_4",
|
|
92
|
+
"ISO-8859-5": "iso8859_5",
|
|
93
|
+
"ISO-8859-6": "iso8859_6",
|
|
94
|
+
"ISO-8859-7": "iso8859_7",
|
|
95
|
+
"ISO-8859-8": "iso8859_8",
|
|
96
|
+
"ISO-8859-8-I": "iso8859_8",
|
|
97
|
+
"ISO-8859-10": "iso8859_10",
|
|
98
|
+
"ISO-8859-13": "iso8859_13",
|
|
99
|
+
"ISO-8859-14": "iso8859_14",
|
|
100
|
+
"ISO-8859-15": "iso8859_15",
|
|
101
|
+
"ISO-8859-16": "iso8859_16",
|
|
102
|
+
"KOI8-R": "koi8_r",
|
|
103
|
+
"KOI8-U": "koi8_u",
|
|
104
|
+
"macintosh": "mac_roman",
|
|
105
|
+
"windows-874": "cp874",
|
|
106
|
+
"windows-1250": "cp1250",
|
|
107
|
+
"windows-1251": "cp1251",
|
|
108
|
+
"windows-1252": "cp1252",
|
|
109
|
+
"windows-1253": "cp1253",
|
|
110
|
+
"windows-1254": "cp1254",
|
|
111
|
+
"windows-1255": "cp1255",
|
|
112
|
+
"windows-1256": "cp1256",
|
|
113
|
+
"windows-1257": "cp1257",
|
|
114
|
+
"windows-1258": "cp1258",
|
|
115
|
+
"x-mac-cyrillic": "mac_cyrillic",
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
# Where the standard's single-byte indexes differ from Python's codecs.
|
|
119
|
+
_SINGLE_BYTE_FIXES = {"KOI8-U": {0xAE: "\u045e", 0xBE: "\u040e"}, "windows-1255": {0xCA: "\u05ba"}}
|
|
120
|
+
|
|
121
|
+
# The standard decodes GBK with the gb18030 decoder.
|
|
122
|
+
_MULTI_BYTE = {
|
|
123
|
+
"GBK": "gb18030",
|
|
124
|
+
"gb18030": "gb18030",
|
|
125
|
+
"Big5": "big5hkscs",
|
|
126
|
+
"EUC-JP": "euc_jp",
|
|
127
|
+
"ISO-2022-JP": "iso2022_jp",
|
|
128
|
+
"Shift_JIS": "cp932",
|
|
129
|
+
"EUC-KR": "cp949",
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
# Unicode encodings, with the byte order mark TextDecoder drops for each.
|
|
133
|
+
_UNICODE = {"UTF-8": ("utf-8", b"\xef\xbb\xbf"), "UTF-16BE": ("utf-16-be", b"\xfe\xff"), "UTF-16LE": ("utf-16-le", b"\xff\xfe")}
|
|
134
|
+
|
|
135
|
+
_tables = {}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _single_byte_table(encoding):
|
|
139
|
+
table = _tables.get(encoding)
|
|
140
|
+
if table is None:
|
|
141
|
+
codec = _SINGLE_BYTE[encoding]
|
|
142
|
+
windows = encoding.startswith("windows-")
|
|
143
|
+
table = {}
|
|
144
|
+
for b in range(0x80, 0x100):
|
|
145
|
+
ch = bytes([b]).decode(codec, "replace")
|
|
146
|
+
if ch == "\ufffd" and windows and b <= 0x9F:
|
|
147
|
+
# The standard's windows-* indexes give unassigned C1 bytes their own code point.
|
|
148
|
+
ch = chr(b)
|
|
149
|
+
table[b] = ch
|
|
150
|
+
table.update(_SINGLE_BYTE_FIXES.get(encoding, {}))
|
|
151
|
+
_tables[encoding] = table
|
|
152
|
+
return table
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _gb18030_errors(err):
|
|
156
|
+
# The standard's gb18030 decoder reads a lone 0x80 as the euro sign.
|
|
157
|
+
if err.object[err.start] == 0x80:
|
|
158
|
+
return "\u20ac", err.start + 1
|
|
159
|
+
return "\ufffd", err.end
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
codecs.register_error("constile_gate.gb18030", _gb18030_errors)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def encoding_for(content_type):
|
|
166
|
+
"""The standard's name for the encoding a Content-Type value declares, or UTF-8."""
|
|
167
|
+
m = _CHARSET.search(content_type or "")
|
|
168
|
+
name = LABELS.get(m.group(1).lower()) if m else None
|
|
169
|
+
if name in _UNICODE or name in _SINGLE_BYTE or name in _MULTI_BYTE or name == "x-user-defined":
|
|
170
|
+
return name
|
|
171
|
+
# No charset, a label the standard doesn't know, or "replacement", which TextDecoder refuses.
|
|
172
|
+
return "UTF-8"
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def decode(data, encoding):
|
|
176
|
+
if encoding in _UNICODE:
|
|
177
|
+
codec, bom = _UNICODE[encoding]
|
|
178
|
+
if data.startswith(bom):
|
|
179
|
+
data = data[len(bom):]
|
|
180
|
+
return data.decode(codec, "replace")
|
|
181
|
+
if encoding in _SINGLE_BYTE:
|
|
182
|
+
# Latin-1 maps each byte to the code point of the same number, so translate() can map bytes.
|
|
183
|
+
return data.decode("latin-1").translate(_single_byte_table(encoding))
|
|
184
|
+
if encoding == "x-user-defined":
|
|
185
|
+
return "".join(chr(b) if b < 0x80 else chr(0xF780 + b - 0x80) for b in data)
|
|
186
|
+
codec = _MULTI_BYTE[encoding]
|
|
187
|
+
return data.decode(codec, "constile_gate.gb18030" if codec == "gb18030" else "replace")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def decode_page(data, content_type):
|
|
191
|
+
return decode(data, encoding_for(content_type))
|
constile_gate/_js.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The JavaScript string and JSON behavior the reference gate (TypeScript) relies on,
|
|
3
|
+
so the port gives the same answers down to the code unit:
|
|
4
|
+
|
|
5
|
+
- JavaScript strings count UTF-16 code units, so `slice(0, n)` can split an
|
|
6
|
+
astral character and leave a lone surrogate. `js_slice` does the same.
|
|
7
|
+
- JavaScript's `\\s` and `trim()` use the ECMAScript whitespace set, which
|
|
8
|
+
differs from Python's `str.isspace()`. `WS_CLASS` and `trim` use it.
|
|
9
|
+
- `JSON.stringify` writes compact JSON, keeps non-ASCII characters, and
|
|
10
|
+
escapes lone surrogates. `dumps` does the same.
|
|
11
|
+
"""
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
# ECMAScript WhiteSpace and LineTerminator code points: what \s matches and trim() removes.
|
|
17
|
+
WS_CHARS = "\t\n\x0b\x0c\r \xa0
"
|
|
18
|
+
WS_CLASS = "[\\t\\n\\x0b\\x0c\\r \\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff]"
|
|
19
|
+
|
|
20
|
+
_ASTRAL = re.compile("[\U00010000-\U0010ffff]")
|
|
21
|
+
_SURROGATE = re.compile("[\ud800-\udfff]")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def trim(s):
|
|
25
|
+
return s.strip(WS_CHARS)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def js_len(s):
|
|
29
|
+
"""The length JavaScript reports: UTF-16 code units."""
|
|
30
|
+
return len(s) + len(_ASTRAL.findall(s))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def js_slice(s, n):
|
|
34
|
+
"""`s.slice(0, n)` in JavaScript, counting UTF-16 code units."""
|
|
35
|
+
head = s[:n]
|
|
36
|
+
if not _ASTRAL.search(head):
|
|
37
|
+
return head
|
|
38
|
+
units = s.encode("utf-16-le", "surrogatepass")[: 2 * n]
|
|
39
|
+
return units.decode("utf-16-le", "surrogatepass")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def combine_surrogates(s):
|
|
43
|
+
"""Joins adjacent high and low surrogates into one character, as a JavaScript string would read them."""
|
|
44
|
+
if not _SURROGATE.search(s):
|
|
45
|
+
return s
|
|
46
|
+
return s.encode("utf-16-le", "surrogatepass").decode("utf-16-le", "surrogatepass")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def dumps(value):
|
|
50
|
+
"""`JSON.stringify(value)` as UTF-8 bytes."""
|
|
51
|
+
text = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
|
|
52
|
+
text = _SURROGATE.sub(lambda m: "\\u%04x" % ord(m.group()), text)
|
|
53
|
+
return text.encode("utf-8")
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def utf8(s):
|
|
57
|
+
"""A string as the UTF-8 bytes a Response body gets: lone surrogates become U+FFFD."""
|
|
58
|
+
return _SURROGATE.sub("\ufffd", combine_surrogates(s)).encode("utf-8")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _no_constants(name):
|
|
62
|
+
raise ValueError("invalid JSON constant %s" % name)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse(body):
|
|
66
|
+
"""`await response.json()`: UTF-8 with an optional byte order mark, strict JSON."""
|
|
67
|
+
return json.loads(body.decode("utf-8-sig", "replace"), parse_constant=_no_constants)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def truthy(value):
|
|
71
|
+
"""JavaScript truthiness: empty strings, 0, NaN, null, and false are falsy; empty arrays and objects aren't."""
|
|
72
|
+
if value is None or value is False:
|
|
73
|
+
return False
|
|
74
|
+
if isinstance(value, (int, float)):
|
|
75
|
+
return value != 0 and not (isinstance(value, float) and math.isnan(value))
|
|
76
|
+
if isinstance(value, str):
|
|
77
|
+
return value != ""
|
|
78
|
+
return True
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
_MISSING = object()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def number(value=_MISSING):
|
|
85
|
+
"""`Number(value)` for the values JSON can hold; a missing value is NaN, as undefined is."""
|
|
86
|
+
if value is _MISSING:
|
|
87
|
+
return math.nan
|
|
88
|
+
if value is None:
|
|
89
|
+
return 0
|
|
90
|
+
if isinstance(value, (bool, int, float)):
|
|
91
|
+
return value
|
|
92
|
+
if isinstance(value, str):
|
|
93
|
+
text = trim(value)
|
|
94
|
+
if text == "":
|
|
95
|
+
return 0
|
|
96
|
+
try:
|
|
97
|
+
return float(text)
|
|
98
|
+
except ValueError:
|
|
99
|
+
return math.nan
|
|
100
|
+
return math.nan
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def string(value):
|
|
104
|
+
"""`String(value)` for the values JSON can hold."""
|
|
105
|
+
if isinstance(value, str):
|
|
106
|
+
return value
|
|
107
|
+
if value is None:
|
|
108
|
+
return "null"
|
|
109
|
+
if value is True:
|
|
110
|
+
return "true"
|
|
111
|
+
if value is False:
|
|
112
|
+
return "false"
|
|
113
|
+
if isinstance(value, int):
|
|
114
|
+
return str(value)
|
|
115
|
+
if isinstance(value, float):
|
|
116
|
+
if math.isnan(value):
|
|
117
|
+
return "NaN"
|
|
118
|
+
if value.is_integer() and abs(value) < 1e21:
|
|
119
|
+
return str(int(value))
|
|
120
|
+
return repr(value)
|
|
121
|
+
if isinstance(value, list):
|
|
122
|
+
return ",".join("" if v is None else string(v) for v in value)
|
|
123
|
+
return "[object Object]"
|
constile_gate/agents.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Who is asking for a page. A port of classifyClient and signatureAgentOrigin in
|
|
3
|
+
packages/core/src/agents.ts. The gate makes an offer only to agents fetching for
|
|
4
|
+
a person; people, search engines, and training crawlers pass through.
|
|
5
|
+
"""
|
|
6
|
+
import re
|
|
7
|
+
from typing import Any, Dict, NamedTuple, Optional
|
|
8
|
+
from urllib.parse import unquote
|
|
9
|
+
|
|
10
|
+
from ._js import js_len, truthy
|
|
11
|
+
|
|
12
|
+
GENERIC_BOT = re.compile(r"bot\b|crawler|spider|headless|scraper", re.ASCII)
|
|
13
|
+
_QUOTED_HTTPS = re.compile(r'"(https://[^"]+)"')
|
|
14
|
+
# Characters a URL host can't contain (the WHATWG URL standard's forbidden domain code points).
|
|
15
|
+
_FORBIDDEN_HOST = re.compile(r"[\x00-\x20#%/:<>?@\[\\\]^|\x7f]")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ClientClass(NamedTuple):
|
|
19
|
+
"""kind is "agent", "crawler", "search", "browser", or "unknown_client"; agent is the registry entry, if any."""
|
|
20
|
+
|
|
21
|
+
kind: str
|
|
22
|
+
agent: Optional[Dict[str, Any]] = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _https_origin(url):
|
|
26
|
+
"""`new URL(url).origin` for an https URL, or None where the URL constructor would throw."""
|
|
27
|
+
rest = url[len("https:"):].lstrip("/\\")
|
|
28
|
+
end = len(rest)
|
|
29
|
+
for c in "/\\?#":
|
|
30
|
+
i = rest.find(c)
|
|
31
|
+
if 0 <= i < end:
|
|
32
|
+
end = i
|
|
33
|
+
authority = rest[:end].rpartition("@")[2]
|
|
34
|
+
if authority.startswith("["):
|
|
35
|
+
close = authority.find("]")
|
|
36
|
+
if close < 0:
|
|
37
|
+
return None
|
|
38
|
+
host, port = authority[: close + 1].lower(), authority[close + 1:]
|
|
39
|
+
if port and not port.startswith(":"):
|
|
40
|
+
return None
|
|
41
|
+
port = port[1:]
|
|
42
|
+
else:
|
|
43
|
+
host, _, port = authority.partition(":")
|
|
44
|
+
if ":" in port:
|
|
45
|
+
return None
|
|
46
|
+
host = unquote(host)
|
|
47
|
+
if not host or _FORBIDDEN_HOST.search(host):
|
|
48
|
+
return None
|
|
49
|
+
try:
|
|
50
|
+
host = host.encode("idna").decode("ascii").lower()
|
|
51
|
+
except UnicodeError:
|
|
52
|
+
return None
|
|
53
|
+
if port:
|
|
54
|
+
if not port.isdigit() or not port.isascii() or int(port) > 65535:
|
|
55
|
+
return None
|
|
56
|
+
if int(port) != 443:
|
|
57
|
+
return "https://%s:%d" % (host, int(port))
|
|
58
|
+
return "https://" + host
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def signature_agent_origin(header):
|
|
62
|
+
"""The origin named in a Signature-Agent header, in either the dictionary or the quoted-string form."""
|
|
63
|
+
m = _QUOTED_HTTPS.search(header or "")
|
|
64
|
+
if not m:
|
|
65
|
+
return None
|
|
66
|
+
return _https_origin(m.group(1))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def classify_client(headers, registry):
|
|
70
|
+
"""headers: anything with get(name) and has(name), case-insensitive. registry: the config's agents."""
|
|
71
|
+
ua = (headers.get("user-agent") or "").lower()
|
|
72
|
+
# Longest token first, so "claude-searchbot" wins over a shorter token it contains.
|
|
73
|
+
by_length = sorted(registry, key=lambda a: -js_len(a["token"]))
|
|
74
|
+
hit = next((a for a in by_length if a["token"] and a["token"].lower() in ua), None)
|
|
75
|
+
if hit:
|
|
76
|
+
if hit["kind"] == "user_agent":
|
|
77
|
+
return ClientClass("agent", hit)
|
|
78
|
+
return ClientClass(hit["kind"], hit)
|
|
79
|
+
# A signed request names its agent in Signature-Agent. The network checks the signature;
|
|
80
|
+
# an agent recognized only this way gets no offer unless the signature verifies.
|
|
81
|
+
origin = signature_agent_origin(headers.get("signature-agent")) if headers.has("signature") else None
|
|
82
|
+
signer = None
|
|
83
|
+
if origin:
|
|
84
|
+
signer = next(
|
|
85
|
+
(
|
|
86
|
+
a
|
|
87
|
+
for a in registry
|
|
88
|
+
if a.get("kind") == "user_agent"
|
|
89
|
+
and truthy(a.get("signature_agent"))
|
|
90
|
+
and signature_agent_origin('"%s"' % a["signature_agent"]) == origin
|
|
91
|
+
),
|
|
92
|
+
None,
|
|
93
|
+
)
|
|
94
|
+
if signer:
|
|
95
|
+
return ClientClass("agent", signer)
|
|
96
|
+
if not ua:
|
|
97
|
+
return ClientClass("unknown_client")
|
|
98
|
+
# Never show an offer to a person: any browser-looking client without a bot marker is a browser.
|
|
99
|
+
if ua.startswith("mozilla/") and not GENERIC_BOT.search(ua):
|
|
100
|
+
return ClientClass("browser")
|
|
101
|
+
return ClientClass("unknown_client")
|