torquests 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- torquests/__init__.py +98 -0
- torquests/__main__.py +10 -0
- torquests/_client/__init__.py +1 -0
- torquests/_client/bootstrap.py +560 -0
- torquests/_client/config.py +21 -0
- torquests/_client/torclient.py +499 -0
- torquests/_crypto/__init__.py +6 -0
- torquests/_crypto/ed25519_blind.py +115 -0
- torquests/_crypto/primitives.py +147 -0
- torquests/_dir/__init__.py +6 -0
- torquests/_dir/authorities.py +50 -0
- torquests/_dir/consensus.py +155 -0
- torquests/_dir/dirhttp.py +136 -0
- torquests/_dir/guards.py +91 -0
- torquests/_dir/keycerts.py +102 -0
- torquests/_dir/microdesc.py +28 -0
- torquests/_dir/models.py +126 -0
- torquests/_dir/parsers.py +365 -0
- torquests/_dir/pathselect.py +238 -0
- torquests/_http/__init__.py +5 -0
- torquests/_http/connection.py +63 -0
- torquests/_http/response.py +76 -0
- torquests/_http/streamsocket.py +109 -0
- torquests/_http/tlssocket.py +109 -0
- torquests/_net/__init__.py +5 -0
- torquests/_net/channel.py +144 -0
- torquests/_net/circuit.py +368 -0
- torquests/_net/flowcontrol.py +115 -0
- torquests/_net/hop.py +40 -0
- torquests/_net/link.py +133 -0
- torquests/_net/sink.py +25 -0
- torquests/_net/stream.py +176 -0
- torquests/_net/transport.py +135 -0
- torquests/_onion/__init__.py +1 -0
- torquests/_onion/address.py +105 -0
- torquests/_onion/descriptor.py +538 -0
- torquests/_onion/hsdir.py +160 -0
- torquests/_onion/rendezvous.py +164 -0
- torquests/_proto/__init__.py +6 -0
- torquests/_proto/cells.py +229 -0
- torquests/_proto/certs.py +106 -0
- torquests/_proto/constants.py +132 -0
- torquests/_proto/handshake/__init__.py +11 -0
- torquests/_proto/handshake/hs_ntor.py +131 -0
- torquests/_proto/handshake/ntor.py +89 -0
- torquests/_proto/linkspec.py +99 -0
- torquests/_proto/relay.py +214 -0
- torquests/_proto/relay_crypto.py +110 -0
- torquests/adapter.py +176 -0
- torquests/api.py +90 -0
- torquests/cli.py +105 -0
- torquests/client.py +17 -0
- torquests/exceptions.py +184 -0
- torquests/py.typed +0 -0
- torquests/sessions.py +166 -0
- torquests/socks.py +205 -0
- torquests/stealth.py +245 -0
- torquests-1.0.0.dist-info/METADATA +183 -0
- torquests-1.0.0.dist-info/RECORD +64 -0
- torquests-1.0.0.dist-info/WHEEL +5 -0
- torquests-1.0.0.dist-info/entry_points.txt +2 -0
- torquests-1.0.0.dist-info/licenses/LICENSE +674 -0
- torquests-1.0.0.dist-info/licenses/NOTICE +8 -0
- torquests-1.0.0.dist-info/top_level.txt +1 -0
torquests/__init__.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""torquests: a pure-Python Tor client with a requests-compatible API.
|
|
2
|
+
|
|
3
|
+
Route HTTP requests through the Tor network, clearnet or v3 ``.onion``, with the
|
|
4
|
+
API you already know from ``requests``::
|
|
5
|
+
|
|
6
|
+
import torquests
|
|
7
|
+
|
|
8
|
+
r = torquests.get("https://check.torproject.org/api/ip")
|
|
9
|
+
|
|
10
|
+
with torquests.Session() as session:
|
|
11
|
+
session.get("http://example.onion")
|
|
12
|
+
|
|
13
|
+
The first call bootstraps a verified consensus of the Tor network and reuses it
|
|
14
|
+
across the process. To reuse a client explicitly, construct a :class:`TorClient`
|
|
15
|
+
and pass ``Session(tor=client)``.
|
|
16
|
+
|
|
17
|
+
Common ``requests`` names are re-exported so ``import torquests as requests``
|
|
18
|
+
works for the usual idioms (``codes``, ``Response``, ``HTTPError``, ...).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
|
|
25
|
+
from requests import PreparedRequest, Request, Response, codes
|
|
26
|
+
from requests.exceptions import (
|
|
27
|
+
ConnectionError,
|
|
28
|
+
ConnectTimeout,
|
|
29
|
+
HTTPError,
|
|
30
|
+
JSONDecodeError,
|
|
31
|
+
ReadTimeout,
|
|
32
|
+
RequestException,
|
|
33
|
+
Timeout,
|
|
34
|
+
TooManyRedirects,
|
|
35
|
+
URLRequired,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
from . import exceptions
|
|
39
|
+
from .adapter import IsolationPolicy, TorAdapter
|
|
40
|
+
from .api import (
|
|
41
|
+
close,
|
|
42
|
+
delete,
|
|
43
|
+
get,
|
|
44
|
+
head,
|
|
45
|
+
new_identity,
|
|
46
|
+
options,
|
|
47
|
+
patch,
|
|
48
|
+
post,
|
|
49
|
+
put,
|
|
50
|
+
request,
|
|
51
|
+
)
|
|
52
|
+
from .client import TorClient, TorConfig
|
|
53
|
+
from .exceptions import TorError
|
|
54
|
+
from .sessions import MixedSession, Session
|
|
55
|
+
from .stealth import StealthTorAdapter, stealth_session
|
|
56
|
+
|
|
57
|
+
__version__ = "1.0.0"
|
|
58
|
+
|
|
59
|
+
# A library should not configure logging; attach a no-op handler so emitting a
|
|
60
|
+
# record without a configured handler does not warn.
|
|
61
|
+
logging.getLogger(__name__).addHandler(logging.NullHandler())
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"ConnectTimeout",
|
|
65
|
+
"ConnectionError",
|
|
66
|
+
"HTTPError",
|
|
67
|
+
"IsolationPolicy",
|
|
68
|
+
"JSONDecodeError",
|
|
69
|
+
"MixedSession",
|
|
70
|
+
"PreparedRequest",
|
|
71
|
+
"ReadTimeout",
|
|
72
|
+
"Request",
|
|
73
|
+
"RequestException",
|
|
74
|
+
"Response",
|
|
75
|
+
"Session",
|
|
76
|
+
"StealthTorAdapter",
|
|
77
|
+
"Timeout",
|
|
78
|
+
"TooManyRedirects",
|
|
79
|
+
"TorAdapter",
|
|
80
|
+
"TorClient",
|
|
81
|
+
"TorConfig",
|
|
82
|
+
"TorError",
|
|
83
|
+
"URLRequired",
|
|
84
|
+
"__version__",
|
|
85
|
+
"close",
|
|
86
|
+
"codes",
|
|
87
|
+
"delete",
|
|
88
|
+
"exceptions",
|
|
89
|
+
"get",
|
|
90
|
+
"head",
|
|
91
|
+
"new_identity",
|
|
92
|
+
"options",
|
|
93
|
+
"patch",
|
|
94
|
+
"post",
|
|
95
|
+
"put",
|
|
96
|
+
"request",
|
|
97
|
+
"stealth_session",
|
|
98
|
+
]
|
torquests/__main__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Client orchestration: configuration, circuit pooling, and lifecycle."""
|
|
@@ -0,0 +1,560 @@
|
|
|
1
|
+
"""Bootstrapping a live view of the Tor network.
|
|
2
|
+
|
|
3
|
+
The client needs a verified consensus before it can build a circuit, but it has
|
|
4
|
+
no circuit yet, a chicken-and-egg the network solves by letting anyone fetch
|
|
5
|
+
directory documents from a directory over plain HTTP. Those documents are signed,
|
|
6
|
+
so the transport does not have to be trusted: the authority signing keys are
|
|
7
|
+
verified against the hardcoded identity fingerprints, and the consensus is checked
|
|
8
|
+
against a majority of them. Only the relay selection that follows is anonymized.
|
|
9
|
+
|
|
10
|
+
This fetches over HTTP with the standard library (no extra dependency) and hands
|
|
11
|
+
out full circuit paths, fetching each relay's microdescriptor (its ntor key and
|
|
12
|
+
exit policy) on demand. Those on-demand fetches would otherwise leak, in
|
|
13
|
+
cleartext from the client's real IP, which relay a circuit is about to use; once
|
|
14
|
+
a :attr:`~LiveDirectory.set_dir_tunnel` hook is installed they are carried over a
|
|
15
|
+
Tor directory circuit (BEGIN_DIR) instead, and only fall back to cleartext if the
|
|
16
|
+
tunnel fails.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import base64
|
|
22
|
+
import logging
|
|
23
|
+
import random
|
|
24
|
+
import threading
|
|
25
|
+
import urllib.request
|
|
26
|
+
from collections.abc import Callable, Iterable, Mapping, Sequence
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from itertools import islice
|
|
29
|
+
from typing import TYPE_CHECKING
|
|
30
|
+
|
|
31
|
+
from .._dir.authorities import DirectoryAuthority
|
|
32
|
+
from .._dir.microdesc import usable_ed_identity
|
|
33
|
+
from .._dir.models import Consensus, Microdescriptor, RouterStatus
|
|
34
|
+
from .._dir.pathselect import family_conflict, normalize_family
|
|
35
|
+
from .._net.hop import RelayInfo
|
|
36
|
+
from ..exceptions import (
|
|
37
|
+
ChannelError,
|
|
38
|
+
CircuitError,
|
|
39
|
+
DirectoryError,
|
|
40
|
+
StreamError,
|
|
41
|
+
TorBootstrapError,
|
|
42
|
+
TorReadTimeout,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
#: A hook that fetches a directory document (given its request path) over Tor.
|
|
46
|
+
DirTunnel = Callable[[str], str]
|
|
47
|
+
|
|
48
|
+
if TYPE_CHECKING:
|
|
49
|
+
from .._dir.keycerts import KeyCertificate
|
|
50
|
+
|
|
51
|
+
_MD_BATCH = 50
|
|
52
|
+
|
|
53
|
+
#: How many times a path is re-drawn to avoid two same-family relays before the
|
|
54
|
+
#: last draw is accepted anyway. Family conflicts among a stable guard, a random
|
|
55
|
+
#: middle, and a random exit are rare, so a handful of attempts converges.
|
|
56
|
+
_FAMILY_SELECT_ATTEMPTS = 8
|
|
57
|
+
|
|
58
|
+
logger = logging.getLogger(__name__)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _http_get(host: str, port: int, path: str, timeout: float) -> bytes:
|
|
62
|
+
# No tool-identifying User-Agent to the directory over cleartext. urllib adds
|
|
63
|
+
# "User-Agent: Python-urllib/<ver>" when none is set, which fingerprints the
|
|
64
|
+
# client, so set an explicit empty header: urllib then sends "User-Agent:"
|
|
65
|
+
# with no value and does not substitute its default.
|
|
66
|
+
url = f"http://{host}:{port}{path}"
|
|
67
|
+
request = urllib.request.Request(url)
|
|
68
|
+
request.add_header("User-Agent", "")
|
|
69
|
+
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
70
|
+
return bytes(response.read())
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _fetch(
|
|
74
|
+
path: str,
|
|
75
|
+
mirrors: Sequence[tuple[str, int]],
|
|
76
|
+
timeout: float,
|
|
77
|
+
rng: random.Random,
|
|
78
|
+
) -> str:
|
|
79
|
+
# Shuffle the mirror order per fetch so no single authority is always the
|
|
80
|
+
# first (or only) directory contacted for a client.
|
|
81
|
+
order = list(mirrors)
|
|
82
|
+
rng.shuffle(order)
|
|
83
|
+
failures = []
|
|
84
|
+
for host, port in order:
|
|
85
|
+
try:
|
|
86
|
+
return _http_get(host, port, path, timeout).decode("ascii", "replace")
|
|
87
|
+
except Exception as exc:
|
|
88
|
+
failures.append(f"{host}:{port} {exc}")
|
|
89
|
+
raise DirectoryError(f"all directory mirrors failed for {path}: {'; '.join(failures)}")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _batched(items: Sequence[bytes], size: int) -> Iterable[list[bytes]]:
|
|
93
|
+
iterator = iter(items)
|
|
94
|
+
while chunk := list(islice(iterator, size)):
|
|
95
|
+
yield chunk
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _authorities_with_keys(
|
|
99
|
+
authority_set: Sequence[DirectoryAuthority],
|
|
100
|
+
certs: Mapping[bytes, KeyCertificate],
|
|
101
|
+
) -> list[DirectoryAuthority]:
|
|
102
|
+
"""Return every authority in the full set, keyed where a cert was fetched.
|
|
103
|
+
|
|
104
|
+
The returned list always has ``len(authority_set)`` entries: each authority
|
|
105
|
+
carries its fetched signing key when one is available, and is kept bare
|
|
106
|
+
otherwise. A bare authority can never be counted toward the consensus quorum
|
|
107
|
+
(it has no key to verify a signature with) yet still counts toward the
|
|
108
|
+
denominator, so an on-path attacker who truncates ``/tor/keys/all`` cannot
|
|
109
|
+
shrink the authority set the majority threshold is computed against.
|
|
110
|
+
"""
|
|
111
|
+
result: list[DirectoryAuthority] = []
|
|
112
|
+
for authority in authority_set:
|
|
113
|
+
cert = certs.get(authority.v3ident)
|
|
114
|
+
if cert is not None:
|
|
115
|
+
result.append(authority.with_signing_key(cert.signing_key_pem))
|
|
116
|
+
else:
|
|
117
|
+
result.append(authority)
|
|
118
|
+
return result
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _microdesc_path(digests: Sequence[bytes]) -> str:
|
|
122
|
+
"""The ``/tor/micro/d/`` request path for a batch of microdescriptor digests."""
|
|
123
|
+
encoded = [base64.b64encode(d).decode("ascii").rstrip("=") for d in digests]
|
|
124
|
+
return "/tor/micro/d/" + "-".join(encoded)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class LiveDirectory:
|
|
128
|
+
"""A verified consensus plus on-demand microdescriptor fetching."""
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
consensus: Consensus,
|
|
133
|
+
mirrors: Sequence[tuple[str, int]],
|
|
134
|
+
*,
|
|
135
|
+
timeout: float,
|
|
136
|
+
rng: random.Random | None = None,
|
|
137
|
+
) -> None:
|
|
138
|
+
from .._dir.guards import GuardManager
|
|
139
|
+
from .._dir.pathselect import DEFAULT_BWWEIGHTSCALE, BandwidthWeights, PathSelector
|
|
140
|
+
|
|
141
|
+
self.consensus = consensus
|
|
142
|
+
# Only relays whose ed25519 identity the consensus agrees on are usable:
|
|
143
|
+
# this client pins the ed identity on every channel, so a NoEdConsensus
|
|
144
|
+
# relay (disputed identity) cannot serve as a hop. Filtering here keeps it
|
|
145
|
+
# out of guard, middle, exit, HSDir, and directory-cache selection alike.
|
|
146
|
+
self._routers = [r for r in consensus.routers if "NoEdConsensus" not in r.flags]
|
|
147
|
+
self._mirrors = list(mirrors)
|
|
148
|
+
self._timeout = timeout
|
|
149
|
+
self._rng: random.Random = rng if rng is not None else random.SystemRandom()
|
|
150
|
+
scale = consensus.params.get("bwweightscale", DEFAULT_BWWEIGHTSCALE)
|
|
151
|
+
self._weights = (
|
|
152
|
+
BandwidthWeights(consensus.bandwidth_weights, scale)
|
|
153
|
+
if consensus.bandwidth_weights
|
|
154
|
+
else None
|
|
155
|
+
)
|
|
156
|
+
self._selector = PathSelector(self._routers, rng=self._rng, weights=self._weights)
|
|
157
|
+
# A stable, process-lifetime primary-guard set: every path built by this
|
|
158
|
+
# directory enters through the same guard instead of a fresh random one.
|
|
159
|
+
self._guards = GuardManager(sample_size=3, rng=self._rng)
|
|
160
|
+
self._guards.update(self._routers, weights=self._weights)
|
|
161
|
+
self._microdescriptors: dict[bytes, Microdescriptor] = {}
|
|
162
|
+
# When set, directory documents are fetched over a Tor circuit instead of
|
|
163
|
+
# in cleartext from the client's real IP. Left unset (cleartext) until a
|
|
164
|
+
# bootstrapped client installs a tunnel; see :meth:`set_dir_tunnel`. This
|
|
165
|
+
# directory is shared across every client in the process, so it holds the
|
|
166
|
+
# installed tunnels in an ordered registry and points ``_dir_tunnel`` at the
|
|
167
|
+
# newest one still installed. Closing a client in any order then re-points it
|
|
168
|
+
# at a live sibling, never a dead circuit. The lock guards the registry,
|
|
169
|
+
# since clients install and close concurrently.
|
|
170
|
+
self._dir_tunnel: DirTunnel | None = None
|
|
171
|
+
self._dir_tunnels: list[DirTunnel] = []
|
|
172
|
+
self._tunnel_lock = threading.Lock()
|
|
173
|
+
|
|
174
|
+
def set_dir_tunnel(self, fetch: DirTunnel | None) -> None:
|
|
175
|
+
"""Install (or clear) a hook that fetches directory documents over Tor.
|
|
176
|
+
|
|
177
|
+
When set, the on-demand microdescriptor fetches this directory performs
|
|
178
|
+
go over a Tor circuit (a BEGIN_DIR directory stream) rather than being
|
|
179
|
+
requested in cleartext from the client's real IP, so an on-path observer
|
|
180
|
+
of the directory traffic no longer learns which middle relay a circuit is
|
|
181
|
+
about to use. Fetching is best-effort: a tunnel failure falls back to the
|
|
182
|
+
cleartext fetch, so installing a tunnel can only remove the leak, never
|
|
183
|
+
break the client. Pass ``None`` to restore cleartext fetching.
|
|
184
|
+
"""
|
|
185
|
+
with self._tunnel_lock:
|
|
186
|
+
self._dir_tunnel = fetch
|
|
187
|
+
|
|
188
|
+
def install_dir_tunnel(self, fetch: DirTunnel) -> None:
|
|
189
|
+
"""Register ``fetch`` as the active directory tunnel.
|
|
190
|
+
|
|
191
|
+
This directory is process-global (see :func:`get_directory`), so several
|
|
192
|
+
clients can hold a tunnel at once. Each call appends ``fetch`` to an ordered
|
|
193
|
+
registry and makes it the active tunnel; :meth:`restore_dir_tunnel` drops it
|
|
194
|
+
again when the client closes. The registry records every installed tunnel, so
|
|
195
|
+
any close order leaves the active tunnel on a live circuit and never reverts a
|
|
196
|
+
surviving client to cleartext directory fetches from the real IP.
|
|
197
|
+
"""
|
|
198
|
+
with self._tunnel_lock:
|
|
199
|
+
self._dir_tunnels.append(fetch)
|
|
200
|
+
self._dir_tunnel = self._dir_tunnels[-1]
|
|
201
|
+
|
|
202
|
+
def restore_dir_tunnel(self, fetch: DirTunnel) -> None:
|
|
203
|
+
"""Undo an :meth:`install_dir_tunnel`: drop ``fetch`` from the registry.
|
|
204
|
+
|
|
205
|
+
Removes ``fetch`` wherever it sits in the registry and re-points the active
|
|
206
|
+
tunnel at the newest one still installed, or clears it when none remain.
|
|
207
|
+
Closing a client that a later one shadowed leaves the active tunnel alone;
|
|
208
|
+
closing the active one falls back to a live sibling, never a dead circuit.
|
|
209
|
+
"""
|
|
210
|
+
with self._tunnel_lock:
|
|
211
|
+
for index, installed in enumerate(self._dir_tunnels):
|
|
212
|
+
if installed is fetch:
|
|
213
|
+
del self._dir_tunnels[index]
|
|
214
|
+
break
|
|
215
|
+
self._dir_tunnel = self._dir_tunnels[-1] if self._dir_tunnels else None
|
|
216
|
+
|
|
217
|
+
def _primary_guards(self) -> list[RouterStatus]:
|
|
218
|
+
"""The client's stable primary guards, resolved in the current consensus."""
|
|
219
|
+
return self._guards.primary_guards(self._routers)
|
|
220
|
+
|
|
221
|
+
def _fetch_dir(self, path: str, *, allow_tunnel: bool = True) -> str:
|
|
222
|
+
"""Fetch a directory document, over the Tor tunnel when one is installed.
|
|
223
|
+
|
|
224
|
+
With a tunnel installed and ``allow_tunnel`` true, the fetch is carried
|
|
225
|
+
over a Tor circuit (BEGIN_DIR); any transport or protocol failure on that
|
|
226
|
+
circuit falls back to the cleartext fetch. ``allow_tunnel=False`` forces
|
|
227
|
+
cleartext, which is how the tunnel's own circuit is bootstrapped without
|
|
228
|
+
recursing into a not-yet-built tunnel.
|
|
229
|
+
"""
|
|
230
|
+
tunnel = self._dir_tunnel
|
|
231
|
+
if tunnel is not None and allow_tunnel:
|
|
232
|
+
try:
|
|
233
|
+
return tunnel(path)
|
|
234
|
+
except (
|
|
235
|
+
TorBootstrapError,
|
|
236
|
+
ChannelError,
|
|
237
|
+
CircuitError,
|
|
238
|
+
StreamError,
|
|
239
|
+
TorReadTimeout,
|
|
240
|
+
) as exc:
|
|
241
|
+
# Best-effort: routing over Tor can only remove the metadata leak,
|
|
242
|
+
# never remove functionality, so a failed tunnel fetch falls back to
|
|
243
|
+
# a cleartext fetch. Warn: that fallback re-exposes, in cleartext
|
|
244
|
+
# from the real IP, which relay a circuit is about to use, so an
|
|
245
|
+
# operator relying on the tunnel needs the degradation to be visible.
|
|
246
|
+
logger.warning(
|
|
247
|
+
"directory tunnel fetch failed (%s); falling back to a cleartext "
|
|
248
|
+
"fetch of %s from the real IP",
|
|
249
|
+
exc,
|
|
250
|
+
path,
|
|
251
|
+
)
|
|
252
|
+
return _fetch(path, self._mirrors, self._timeout, self._rng)
|
|
253
|
+
|
|
254
|
+
def _missing_digests(self, routers: Sequence[RouterStatus]) -> list[bytes]:
|
|
255
|
+
"""The microdescriptor digests of ``routers`` not already cached."""
|
|
256
|
+
return [
|
|
257
|
+
r.microdescriptor_digest
|
|
258
|
+
for r in routers
|
|
259
|
+
if r.microdescriptor_digest and r.microdescriptor_digest not in self._microdescriptors
|
|
260
|
+
]
|
|
261
|
+
|
|
262
|
+
def _store_microdescriptors(self, text: str) -> None:
|
|
263
|
+
from .._dir.parsers import parse_microdescriptors
|
|
264
|
+
|
|
265
|
+
for md in parse_microdescriptors(text):
|
|
266
|
+
self._microdescriptors[md.digest] = md
|
|
267
|
+
|
|
268
|
+
def _fetch_microdescriptors(
|
|
269
|
+
self, routers: Sequence[RouterStatus], *, allow_tunnel: bool = True
|
|
270
|
+
) -> None:
|
|
271
|
+
for chunk in _batched(self._missing_digests(routers), _MD_BATCH):
|
|
272
|
+
self._store_microdescriptors(
|
|
273
|
+
self._fetch_dir(_microdesc_path(chunk), allow_tunnel=allow_tunnel)
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
def _relay_info(self, router: RouterStatus) -> RelayInfo:
|
|
277
|
+
md = self._microdescriptors.get(router.microdescriptor_digest or b"")
|
|
278
|
+
if md is None:
|
|
279
|
+
raise DirectoryError(f"missing microdescriptor for relay {router.nickname}")
|
|
280
|
+
ed_identity = usable_ed_identity(router.flags, md)
|
|
281
|
+
if ed_identity is None:
|
|
282
|
+
raise DirectoryError(f"relay {router.nickname} has no usable ed25519 identity")
|
|
283
|
+
return RelayInfo(
|
|
284
|
+
address=(router.address, router.or_port),
|
|
285
|
+
ntor_onion_key=md.ntor_onion_key,
|
|
286
|
+
identity_digest=router.fingerprint,
|
|
287
|
+
ed_identity=ed_identity,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
def _select_exit(self, port: int) -> RouterStatus:
|
|
291
|
+
from .._dir.pathselect import bandwidth_weighted_choice
|
|
292
|
+
|
|
293
|
+
# Consider every exit whose policy allows the port, not a raw-bandwidth
|
|
294
|
+
# top-N slice: capping by bandwidth first would shrink the exit anonymity
|
|
295
|
+
# set toward a handful of high-capacity relays. Position weighting then
|
|
296
|
+
# applies over the full policy-allowed set.
|
|
297
|
+
candidates = [r for r in self._routers if r.is_exit]
|
|
298
|
+
self._fetch_microdescriptors_parallel(candidates)
|
|
299
|
+
allowed = []
|
|
300
|
+
for router in candidates:
|
|
301
|
+
md = self._microdescriptors.get(router.microdescriptor_digest or b"")
|
|
302
|
+
if md is not None and md.exit_policy is not None and md.exit_policy.allows(port):
|
|
303
|
+
allowed.append(router)
|
|
304
|
+
if not allowed:
|
|
305
|
+
raise DirectoryError(f"no exit relay allows port {port}")
|
|
306
|
+
return bandwidth_weighted_choice(self._rng, allowed, position="exit", weights=self._weights)
|
|
307
|
+
|
|
308
|
+
def _family_of(self, router: RouterStatus) -> frozenset[str]:
|
|
309
|
+
"""The relay's declared family fingerprints, from its cached microdescriptor."""
|
|
310
|
+
md = self._microdescriptors.get(router.microdescriptor_digest or b"")
|
|
311
|
+
return normalize_family(md.family) if md is not None else frozenset()
|
|
312
|
+
|
|
313
|
+
def _has_family_conflict(self, relays: Sequence[RouterStatus]) -> bool:
|
|
314
|
+
"""Whether any two of ``relays`` declare each other in family."""
|
|
315
|
+
families = [(r.fingerprint, self._family_of(r)) for r in relays]
|
|
316
|
+
for index, (fp_a, family_a) in enumerate(families):
|
|
317
|
+
for fp_b, family_b in families[index + 1 :]:
|
|
318
|
+
if family_conflict(fp_a, family_a, fp_b, family_b):
|
|
319
|
+
return True
|
|
320
|
+
return False
|
|
321
|
+
|
|
322
|
+
def _select_family_disjoint(
|
|
323
|
+
self, select: Callable[[], list[RouterStatus]], *, allow_tunnel: bool = True
|
|
324
|
+
) -> list[RouterStatus]:
|
|
325
|
+
"""Draw a path with ``select`` until its relays share no declared family.
|
|
326
|
+
|
|
327
|
+
Family membership lives in the microdescriptors, which this directory
|
|
328
|
+
fetches on demand, so it can only be checked after selection: each attempt
|
|
329
|
+
draws relays, fetches their microdescriptors, and accepts the draw when no
|
|
330
|
+
two are family. A stable guard, a random middle, and a random exit rarely
|
|
331
|
+
collide, so a few attempts converge; the last draw is returned regardless
|
|
332
|
+
so a pathological consensus never fails the request (identity and /16
|
|
333
|
+
separation are already enforced during selection).
|
|
334
|
+
"""
|
|
335
|
+
relays = select()
|
|
336
|
+
for _ in range(_FAMILY_SELECT_ATTEMPTS - 1):
|
|
337
|
+
self._fetch_microdescriptors(relays, allow_tunnel=allow_tunnel)
|
|
338
|
+
if not self._has_family_conflict(relays):
|
|
339
|
+
return relays
|
|
340
|
+
relays = select()
|
|
341
|
+
self._fetch_microdescriptors(relays, allow_tunnel=allow_tunnel)
|
|
342
|
+
return relays
|
|
343
|
+
|
|
344
|
+
def path_provider(self, host: str, port: int) -> list[RelayInfo]:
|
|
345
|
+
"""Select a guard -> middle -> exit path that allows ``port``.
|
|
346
|
+
|
|
347
|
+
The guard is the client's stable primary guard (guard-spec); only the
|
|
348
|
+
middle and exit vary between circuits. No two hops share an identity or a
|
|
349
|
+
/16 (enforced during selection); family separation is best-effort, applied
|
|
350
|
+
by :meth:`_select_family_disjoint`.
|
|
351
|
+
"""
|
|
352
|
+
|
|
353
|
+
def select() -> list[RouterStatus]:
|
|
354
|
+
exit_relay = self._select_exit(port)
|
|
355
|
+
guard = self._selector.select_guard(exclude=[exit_relay], prefer=self._primary_guards())
|
|
356
|
+
middle = self._selector.select_middle(exclude=[guard, exit_relay])
|
|
357
|
+
return [guard, middle, exit_relay]
|
|
358
|
+
|
|
359
|
+
guard, middle, exit_relay = self._select_family_disjoint(select)
|
|
360
|
+
return [self._relay_info(guard), self._relay_info(middle), self._relay_info(exit_relay)]
|
|
361
|
+
|
|
362
|
+
def path_to(self, target: RouterStatus) -> list[RelayInfo]:
|
|
363
|
+
"""A guard -> middle -> ``target`` path (used to reach an HSDir, intro, or RP)."""
|
|
364
|
+
|
|
365
|
+
def select() -> list[RouterStatus]:
|
|
366
|
+
guard = self._selector.select_guard(exclude=[target], prefer=self._primary_guards())
|
|
367
|
+
middle = self._selector.select_middle(exclude=[guard, target])
|
|
368
|
+
return [guard, middle, target]
|
|
369
|
+
|
|
370
|
+
guard, middle, target = self._select_family_disjoint(select)
|
|
371
|
+
return [self._relay_info(guard), self._relay_info(middle), self._relay_info(target)]
|
|
372
|
+
|
|
373
|
+
def rendezvous_path(self) -> list[RelayInfo]:
|
|
374
|
+
"""A three-hop path whose last hop is usable as a rendezvous point."""
|
|
375
|
+
|
|
376
|
+
def select() -> list[RouterStatus]:
|
|
377
|
+
guard = self._selector.select_guard(prefer=self._primary_guards())
|
|
378
|
+
middle = self._selector.select_middle(exclude=[guard])
|
|
379
|
+
rp = self._selector.select_middle(exclude=[guard, middle])
|
|
380
|
+
return [guard, middle, rp]
|
|
381
|
+
|
|
382
|
+
guard, middle, rp = self._select_family_disjoint(select)
|
|
383
|
+
return [self._relay_info(guard), self._relay_info(middle), self._relay_info(rp)]
|
|
384
|
+
|
|
385
|
+
def path_ending_at(self, target: RelayInfo) -> list[RelayInfo]:
|
|
386
|
+
"""A guard -> middle -> ``target`` path where the last hop is given directly.
|
|
387
|
+
|
|
388
|
+
Used to reach an introduction point named by an onion descriptor rather
|
|
389
|
+
than by the consensus. The guard and middle are drawn family-disjoint from
|
|
390
|
+
each other (best-effort); the target comes from the descriptor, not the
|
|
391
|
+
consensus, so it carries no declared family to check here.
|
|
392
|
+
"""
|
|
393
|
+
|
|
394
|
+
def select() -> list[RouterStatus]:
|
|
395
|
+
guard = self._selector.select_guard(prefer=self._primary_guards())
|
|
396
|
+
middle = self._selector.select_middle(exclude=[guard])
|
|
397
|
+
return [guard, middle]
|
|
398
|
+
|
|
399
|
+
guard, middle = self._select_family_disjoint(select)
|
|
400
|
+
return [self._relay_info(guard), self._relay_info(middle), target]
|
|
401
|
+
|
|
402
|
+
# --- directory tunnel -------------------------------------------------- #
|
|
403
|
+
|
|
404
|
+
def dir_circuit_path(self) -> list[RelayInfo]:
|
|
405
|
+
"""Select a guard -> middle -> directory-cache path for the directory tunnel.
|
|
406
|
+
|
|
407
|
+
The last hop carries the ``V2Dir`` flag, so it answers tunneled BEGIN_DIR
|
|
408
|
+
directory requests over its OR port (dir-spec/assigning-flags-vote.md).
|
|
409
|
+
The three relays on this path have their microdescriptors fetched in
|
|
410
|
+
*cleartext* (``allow_tunnel=False``): this is the single, unavoidable
|
|
411
|
+
residual cleartext fetch, done once when the tunnel is built, that lets
|
|
412
|
+
every later microdescriptor fetch be tunneled without recursing back into
|
|
413
|
+
a not-yet-built tunnel. The guard is the client's stable primary guard,
|
|
414
|
+
which an on-path observer already learns from the TLS connection, so only
|
|
415
|
+
the middle and the cache are newly exposed, once.
|
|
416
|
+
"""
|
|
417
|
+
from .._dir.pathselect import bandwidth_weighted_choice
|
|
418
|
+
|
|
419
|
+
caches = [r for r in self._routers if r.is_v2dir]
|
|
420
|
+
if not caches:
|
|
421
|
+
raise DirectoryError("consensus has no V2Dir directory cache to tunnel through")
|
|
422
|
+
|
|
423
|
+
def select() -> list[RouterStatus]:
|
|
424
|
+
cache = bandwidth_weighted_choice(
|
|
425
|
+
self._rng, caches, position="middle", weights=self._weights
|
|
426
|
+
)
|
|
427
|
+
guard = self._selector.select_guard(exclude=[cache], prefer=self._primary_guards())
|
|
428
|
+
middle = self._selector.select_middle(exclude=[guard, cache])
|
|
429
|
+
return [guard, middle, cache]
|
|
430
|
+
|
|
431
|
+
guard, middle, cache = self._select_family_disjoint(select, allow_tunnel=False)
|
|
432
|
+
return [self._relay_info(guard), self._relay_info(middle), self._relay_info(cache)]
|
|
433
|
+
|
|
434
|
+
# --- onion-service directory support ----------------------------------- #
|
|
435
|
+
|
|
436
|
+
def _fetch_microdescriptors_parallel(
|
|
437
|
+
self, routers: Sequence[RouterStatus], *, allow_tunnel: bool = True
|
|
438
|
+
) -> None:
|
|
439
|
+
import concurrent.futures
|
|
440
|
+
|
|
441
|
+
batches = list(_batched(self._missing_digests(routers), _MD_BATCH))
|
|
442
|
+
if not batches:
|
|
443
|
+
return
|
|
444
|
+
|
|
445
|
+
def fetch(chunk: list[bytes]) -> str:
|
|
446
|
+
return self._fetch_dir(_microdesc_path(chunk), allow_tunnel=allow_tunnel)
|
|
447
|
+
|
|
448
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
|
|
449
|
+
# pool.map yields on this thread, so the microdescriptor store is
|
|
450
|
+
# only ever mutated here, never concurrently from the worker threads.
|
|
451
|
+
for text in pool.map(fetch, batches):
|
|
452
|
+
self._store_microdescriptors(text)
|
|
453
|
+
|
|
454
|
+
def _hsdir_nodes(self) -> list[_HsDirNode]:
|
|
455
|
+
hsdirs = [r for r in self._routers if r.is_hsdir]
|
|
456
|
+
self._fetch_microdescriptors_parallel(hsdirs)
|
|
457
|
+
nodes = []
|
|
458
|
+
for router in hsdirs:
|
|
459
|
+
md = self._microdescriptors.get(router.microdescriptor_digest or b"")
|
|
460
|
+
if md is None:
|
|
461
|
+
continue
|
|
462
|
+
ed_identity = usable_ed_identity(router.flags, md)
|
|
463
|
+
if ed_identity is not None:
|
|
464
|
+
nodes.append(_HsDirNode(ed_identity, router))
|
|
465
|
+
return nodes
|
|
466
|
+
|
|
467
|
+
def responsible_hsdirs(
|
|
468
|
+
self, blinded_pubkey: bytes, *, use_previous_srv: bool
|
|
469
|
+
) -> list[RouterStatus]:
|
|
470
|
+
"""The HSDirs that should hold a descriptor, for the current time period."""
|
|
471
|
+
from .._onion.hsdir import disaster_srv, responsible_hsdirs
|
|
472
|
+
|
|
473
|
+
period = self.time_period()
|
|
474
|
+
period_length = self.period_length()
|
|
475
|
+
srv = (
|
|
476
|
+
self.consensus.shared_random_previous
|
|
477
|
+
if use_previous_srv
|
|
478
|
+
else self.consensus.shared_random_current
|
|
479
|
+
)
|
|
480
|
+
if srv is None:
|
|
481
|
+
srv = disaster_srv(period_length, period)
|
|
482
|
+
chosen = responsible_hsdirs(blinded_pubkey, self._hsdir_nodes(), srv, period, period_length)
|
|
483
|
+
return [node.router for node in chosen]
|
|
484
|
+
|
|
485
|
+
def period_length(self) -> int:
|
|
486
|
+
"""The time-period length in minutes (consensus ``hsdir_interval``)."""
|
|
487
|
+
from .._crypto.ed25519_blind import DEFAULT_PERIOD_LENGTH_MINUTES
|
|
488
|
+
|
|
489
|
+
return self.consensus.params.get("hsdir_interval", DEFAULT_PERIOD_LENGTH_MINUTES)
|
|
490
|
+
|
|
491
|
+
def time_period(self) -> int:
|
|
492
|
+
from .._crypto.ed25519_blind import time_period
|
|
493
|
+
|
|
494
|
+
return time_period(int(self.consensus.valid_after.timestamp()), self.period_length())
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
class _HsDirNode:
|
|
498
|
+
"""Adapts a RouterStatus to the hash-ring node interface (``.ed_identity``)."""
|
|
499
|
+
|
|
500
|
+
def __init__(self, ed_identity: bytes, router: RouterStatus) -> None:
|
|
501
|
+
self.ed_identity = ed_identity
|
|
502
|
+
self.router = router
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def bootstrap(
|
|
506
|
+
*,
|
|
507
|
+
authorities: Sequence[DirectoryAuthority] | None = None,
|
|
508
|
+
timeout: float = 60.0,
|
|
509
|
+
rng: random.Random | None = None,
|
|
510
|
+
) -> LiveDirectory:
|
|
511
|
+
"""Fetch and verify the consensus, returning a live directory."""
|
|
512
|
+
from .._dir.authorities import DEFAULT_AUTHORITIES
|
|
513
|
+
from .._dir.consensus import verify_consensus
|
|
514
|
+
from .._dir.keycerts import parse_key_certificates
|
|
515
|
+
|
|
516
|
+
resolved_rng: random.Random = rng if rng is not None else random.SystemRandom()
|
|
517
|
+
authority_set = list(authorities) if authorities is not None else list(DEFAULT_AUTHORITIES)
|
|
518
|
+
mirrors = [a.dir_address for a in authority_set]
|
|
519
|
+
|
|
520
|
+
cert_text = _fetch("/tor/keys/all", mirrors, timeout, resolved_rng)
|
|
521
|
+
certs = {c.v3ident: c for c in parse_key_certificates(cert_text)}
|
|
522
|
+
fetched = sum(1 for a in authority_set if a.v3ident in certs)
|
|
523
|
+
if fetched * 2 <= len(authority_set):
|
|
524
|
+
raise TorBootstrapError(
|
|
525
|
+
f"only {fetched} of {len(authority_set)} authority signing keys verified"
|
|
526
|
+
)
|
|
527
|
+
# Verify the consensus against the FULL authority set (keyed where possible),
|
|
528
|
+
# not just the authorities whose cert we happened to fetch: the majority
|
|
529
|
+
# threshold must be computed over every authority, so withholding certs cannot
|
|
530
|
+
# lower the quorum.
|
|
531
|
+
authorities_with_keys = _authorities_with_keys(authority_set, certs)
|
|
532
|
+
|
|
533
|
+
consensus_text = _fetch(
|
|
534
|
+
"/tor/status-vote/current/consensus-microdesc", mirrors, timeout, resolved_rng
|
|
535
|
+
)
|
|
536
|
+
consensus = verify_consensus(
|
|
537
|
+
consensus_text, authorities_with_keys, now=datetime.now(timezone.utc)
|
|
538
|
+
)
|
|
539
|
+
return LiveDirectory(consensus, mirrors, timeout=timeout, rng=resolved_rng)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
_cache_lock = threading.Lock()
|
|
543
|
+
_cached_directory: LiveDirectory | None = None
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def get_directory(*, timeout: float = 60.0, refresh: bool = False) -> LiveDirectory:
|
|
547
|
+
"""Return a process-global live directory, bootstrapping once and reusing it.
|
|
548
|
+
|
|
549
|
+
Re-bootstraps when ``refresh`` is set or the cached consensus is no longer live,
|
|
550
|
+
so a client created after the consensus expires (a fresh session, or the module
|
|
551
|
+
verbs) selects paths from a current network view rather than a stale one.
|
|
552
|
+
"""
|
|
553
|
+
global _cached_directory
|
|
554
|
+
with _cache_lock:
|
|
555
|
+
cached = _cached_directory
|
|
556
|
+
stale = cached is not None and not cached.consensus.is_live(datetime.now(timezone.utc))
|
|
557
|
+
if cached is None or refresh or stale:
|
|
558
|
+
cached = bootstrap(timeout=timeout)
|
|
559
|
+
_cached_directory = cached
|
|
560
|
+
return cached
|