pyfunda 3.1.2__tar.gz → 3.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyfunda-3.1.2 → pyfunda-3.1.4}/PKG-INFO +1 -1
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_transport.py +75 -19
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/funda.py +1 -4
- {pyfunda-3.1.2 → pyfunda-3.1.4}/pyproject.toml +1 -1
- pyfunda-3.1.4/tests/test_transport_parallel.py +285 -0
- pyfunda-3.1.2/tests/test_transport_parallel.py +0 -122
- {pyfunda-3.1.2 → pyfunda-3.1.4}/.dockerignore +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/.github/FUNDING.yml +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/.github/workflows/publish.yml +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/.gitignore +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/Dockerfile +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/LICENSE +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/README.md +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/API.md +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/ARCHITECTURE.md +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/DEVELOPMENT.md +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/EXAMPLES.md +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/README.md +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/almere_age_rank.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/analysis.ipynb +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/batch_details.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/broker_due_diligence.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/enrichment_export.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/export_to_csv.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/full_api_walkthrough.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/neighborhood_market_snapshot.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/new_listings_alert.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/poll_new_listings.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/price_history.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/price_tracker.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/search_sold.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/similar_sales_comp.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/__init__.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_autocomplete.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_autocomplete_parser.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_detail_parser.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_enrichment_parser.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_parallel.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_parse_helpers.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_price_history_parser.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_search_parser.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/constants.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/exceptions.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/headers.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/listing.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/models.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/parsing.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/py.typed +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/search.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/__init__.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_autocomplete.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_client.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_enrichment_parser.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_live.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_models.py +0 -0
- {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_search.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyfunda
|
|
3
|
-
Version: 3.1.
|
|
3
|
+
Version: 3.1.4
|
|
4
4
|
Summary: Python API wrapper for Funda.nl, the Dutch real estate platform. Reverse-engineered mobile API client, no scraping, no Selenium, no CAPTCHA.
|
|
5
5
|
Project-URL: Homepage, https://github.com/0xMH/pyfunda
|
|
6
6
|
Project-URL: Repository, https://github.com/0xMH/pyfunda
|
|
@@ -5,6 +5,7 @@ from collections.abc import Callable
|
|
|
5
5
|
from dataclasses import dataclass, field
|
|
6
6
|
from threading import Lock
|
|
7
7
|
from typing import Any, ClassVar
|
|
8
|
+
from urllib.parse import urlsplit
|
|
8
9
|
|
|
9
10
|
from curl_cffi import requests
|
|
10
11
|
import tls_client
|
|
@@ -20,22 +21,28 @@ from funda.models import Fingerprint, HttpMethod, JsonDict, RequestProfile
|
|
|
20
21
|
class _FundaTransport:
|
|
21
22
|
_cached_fingerprint_index: ClassVar[int | None] = None
|
|
22
23
|
_fingerprint_cache_lock: ClassVar[Any] = Lock()
|
|
24
|
+
_request_pacing_lock: ClassVar[Any] = Lock()
|
|
25
|
+
_last_request_at: ClassVar[dict[str, float]] = {}
|
|
23
26
|
|
|
24
27
|
timeout: int = 30
|
|
25
28
|
max_retries: int = DEFAULT_MAX_RETRIES
|
|
26
|
-
retry_backoff: float = 0.
|
|
29
|
+
retry_backoff: float = 0.5
|
|
30
|
+
min_request_interval: float = 0.5
|
|
27
31
|
retry_statuses: set[int] | None = None
|
|
28
32
|
|
|
29
33
|
_curl_session: requests.Session | None = field(default=None, init=False, repr=False)
|
|
30
34
|
_tls_session: tls_client.Session | None = field(default=None, init=False, repr=False)
|
|
31
35
|
_fingerprint: Fingerprint | None = field(default=None, init=False, repr=False)
|
|
32
36
|
_fingerprint_index: int | None = field(default=None, init=False, repr=False)
|
|
37
|
+
_fingerprint_proven: bool = field(default=False, init=False, repr=False)
|
|
33
38
|
|
|
34
39
|
def __post_init__(self) -> None:
|
|
35
40
|
if self.max_retries < 0:
|
|
36
41
|
raise ValueError("max_retries must be >= 0")
|
|
37
42
|
if self.retry_backoff < 0:
|
|
38
43
|
raise ValueError("retry_backoff must be >= 0")
|
|
44
|
+
if self.min_request_interval < 0:
|
|
45
|
+
raise ValueError("min_request_interval must be >= 0")
|
|
39
46
|
if self.retry_statuses is None:
|
|
40
47
|
self.retry_statuses = set(RETRY_STATUS_CODES)
|
|
41
48
|
else:
|
|
@@ -62,25 +69,34 @@ class _FundaTransport:
|
|
|
62
69
|
def _headers_dict(self, profile: RequestProfile = "listing") -> dict[str, str]:
|
|
63
70
|
return dict(make_headers(profile))
|
|
64
71
|
|
|
65
|
-
def _select_fingerprint(
|
|
66
|
-
|
|
72
|
+
def _select_fingerprint(
|
|
73
|
+
self, start_index: int | None = None
|
|
74
|
+
) -> tuple[int, Fingerprint, bool]:
|
|
75
|
+
if start_index is None:
|
|
67
76
|
with type(self)._fingerprint_cache_lock:
|
|
68
77
|
cached_index = type(self)._cached_fingerprint_index
|
|
69
78
|
if cached_index is not None:
|
|
70
|
-
return cached_index, FINGERPRINT_POOL[cached_index]
|
|
79
|
+
return cached_index, FINGERPRINT_POOL[cached_index], True
|
|
80
|
+
start_index = 0
|
|
71
81
|
|
|
72
82
|
index = start_index % len(FINGERPRINT_POOL)
|
|
73
|
-
return index, FINGERPRINT_POOL[index]
|
|
83
|
+
return index, FINGERPRINT_POOL[index], False
|
|
74
84
|
|
|
75
85
|
def _remember_fingerprint(self) -> None:
|
|
76
86
|
if self._fingerprint_index is None:
|
|
77
87
|
return
|
|
88
|
+
self._fingerprint_proven = True
|
|
78
89
|
with type(self)._fingerprint_cache_lock:
|
|
79
90
|
type(self)._cached_fingerprint_index = self._fingerprint_index
|
|
80
91
|
|
|
81
|
-
def _ensure_session(self, fingerprint_start_index: int =
|
|
92
|
+
def _ensure_session(self, fingerprint_start_index: int | None = None) -> None:
|
|
82
93
|
if self._fingerprint is None:
|
|
83
|
-
|
|
94
|
+
index, fingerprint, selected_proven = self._select_fingerprint(
|
|
95
|
+
fingerprint_start_index
|
|
96
|
+
)
|
|
97
|
+
self._fingerprint_index = index
|
|
98
|
+
self._fingerprint = fingerprint
|
|
99
|
+
self._fingerprint_proven = self._fingerprint_proven or selected_proven
|
|
84
100
|
|
|
85
101
|
fingerprint = self._fingerprint
|
|
86
102
|
if fingerprint.kind == "tls_ja3" and self._tls_session is None:
|
|
@@ -106,17 +122,21 @@ class _FundaTransport:
|
|
|
106
122
|
self._curl_session = None
|
|
107
123
|
self._tls_session = None
|
|
108
124
|
|
|
109
|
-
def _reset_transport(self, *, rotate: bool = False) -> int:
|
|
125
|
+
def _reset_transport(self, *, rotate: bool = False) -> int | None:
|
|
110
126
|
current_index = self._fingerprint_index
|
|
111
127
|
self._close_sessions()
|
|
112
128
|
self._fingerprint = None
|
|
113
129
|
self._fingerprint_index = None
|
|
114
|
-
if
|
|
130
|
+
if current_index is None:
|
|
131
|
+
self._fingerprint_proven = False
|
|
132
|
+
return None
|
|
133
|
+
if rotate:
|
|
134
|
+
self._fingerprint_proven = False
|
|
115
135
|
with type(self)._fingerprint_cache_lock:
|
|
116
136
|
if type(self)._cached_fingerprint_index == current_index:
|
|
117
137
|
type(self)._cached_fingerprint_index = None
|
|
118
138
|
return (current_index + 1) % len(FINGERPRINT_POOL)
|
|
119
|
-
return
|
|
139
|
+
return current_index
|
|
120
140
|
|
|
121
141
|
def get(self, url: str, *, profile: RequestProfile = "listing") -> Any:
|
|
122
142
|
return self.request("GET", url, profile=profile)
|
|
@@ -140,7 +160,7 @@ class _FundaTransport:
|
|
|
140
160
|
data: str | None = None,
|
|
141
161
|
json_data: JsonDict | None = None,
|
|
142
162
|
) -> Any:
|
|
143
|
-
fingerprint_start_index =
|
|
163
|
+
fingerprint_start_index: int | None = None
|
|
144
164
|
|
|
145
165
|
for attempt in range(self.max_retries + 1):
|
|
146
166
|
try:
|
|
@@ -155,7 +175,9 @@ class _FundaTransport:
|
|
|
155
175
|
except (requests.errors.RequestsError, TLSClientExeption):
|
|
156
176
|
if attempt >= self.max_retries:
|
|
157
177
|
raise
|
|
158
|
-
fingerprint_start_index = self._reset_transport(
|
|
178
|
+
fingerprint_start_index = self._reset_transport(
|
|
179
|
+
rotate=not self._fingerprint_proven
|
|
180
|
+
)
|
|
159
181
|
self._sleep_backoff(attempt)
|
|
160
182
|
continue
|
|
161
183
|
|
|
@@ -163,19 +185,52 @@ class _FundaTransport:
|
|
|
163
185
|
self._remember_fingerprint()
|
|
164
186
|
return response
|
|
165
187
|
if attempt >= self.max_retries:
|
|
188
|
+
detail = self._rejection_detail(response)
|
|
166
189
|
raise FundaRequestError(
|
|
167
190
|
f"Funda rejected {method} {url} after {attempt + 1} attempts "
|
|
168
|
-
f"(status {response.status_code})"
|
|
191
|
+
f"(status {response.status_code}){detail}"
|
|
169
192
|
)
|
|
170
193
|
|
|
171
|
-
|
|
172
|
-
self.
|
|
194
|
+
rotate = response.status_code == 403 and not self._fingerprint_proven
|
|
195
|
+
fingerprint_start_index = self._reset_transport(rotate=rotate)
|
|
196
|
+
self._sleep_backoff(attempt, status_code=response.status_code)
|
|
173
197
|
|
|
174
198
|
raise AssertionError("unreachable: retry loop must return or raise")
|
|
175
199
|
|
|
176
|
-
def _sleep_backoff(self, attempt: int) -> None:
|
|
177
|
-
if self.retry_backoff
|
|
178
|
-
|
|
200
|
+
def _sleep_backoff(self, attempt: int, *, status_code: int | None = None) -> None:
|
|
201
|
+
if self.retry_backoff <= 0:
|
|
202
|
+
return
|
|
203
|
+
multiplier = 2**attempt if status_code in (403, 429) else attempt + 1
|
|
204
|
+
time.sleep(self.retry_backoff * multiplier)
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def _rejection_detail(response: Any) -> str:
|
|
208
|
+
headers = getattr(response, "headers", {})
|
|
209
|
+
server = headers.get("server", headers.get("Server", ""))
|
|
210
|
+
text = getattr(response, "text", "")
|
|
211
|
+
if "AkamaiGHost" in server or "Access Denied" in text[:500]:
|
|
212
|
+
return (
|
|
213
|
+
": Akamai denied the request; this is usually an IP-based rate or "
|
|
214
|
+
"reputation block, not a search payload error. Stop requests and retry "
|
|
215
|
+
"after a cooldown or from a different network"
|
|
216
|
+
)
|
|
217
|
+
return ""
|
|
218
|
+
|
|
219
|
+
def _pace_request(self, url: str) -> None:
|
|
220
|
+
if self.min_request_interval <= 0:
|
|
221
|
+
return
|
|
222
|
+
|
|
223
|
+
hostname = urlsplit(url).hostname or ""
|
|
224
|
+
pacing_key = "funda.io" if hostname.endswith(".funda.io") else hostname
|
|
225
|
+
with type(self)._request_pacing_lock:
|
|
226
|
+
now = time.monotonic()
|
|
227
|
+
last_request_at = type(self)._last_request_at.get(pacing_key)
|
|
228
|
+
if last_request_at is not None:
|
|
229
|
+
delay = self.min_request_interval - (now - last_request_at)
|
|
230
|
+
if delay > 0:
|
|
231
|
+
time.sleep(delay)
|
|
232
|
+
now = time.monotonic()
|
|
233
|
+
type(self)._last_request_at[pacing_key] = now
|
|
179
234
|
|
|
180
235
|
def _send_once(
|
|
181
236
|
self,
|
|
@@ -185,8 +240,9 @@ class _FundaTransport:
|
|
|
185
240
|
profile: RequestProfile = "listing",
|
|
186
241
|
data: str | None = None,
|
|
187
242
|
json_data: JsonDict | None = None,
|
|
188
|
-
fingerprint_start_index: int =
|
|
243
|
+
fingerprint_start_index: int | None = None,
|
|
189
244
|
) -> Any:
|
|
245
|
+
self._pace_request(url)
|
|
190
246
|
self._ensure_session(fingerprint_start_index=fingerprint_start_index)
|
|
191
247
|
fingerprint = self._fingerprint
|
|
192
248
|
if fingerprint is None:
|
|
@@ -64,8 +64,7 @@ class Funda:
|
|
|
64
64
|
timeout: int = 30
|
|
65
65
|
max_retries: int = DEFAULT_MAX_RETRIES
|
|
66
66
|
retry_backoff: float = 0.5
|
|
67
|
-
min_request_interval: float = 0.
|
|
68
|
-
adaptive_throttle: bool = True
|
|
67
|
+
min_request_interval: float = 0.5
|
|
69
68
|
|
|
70
69
|
_transport: _FundaTransport = field(init=False, repr=False)
|
|
71
70
|
_parallel_runner: _ParallelRunner["Funda"] | None = field(default=None, init=False, repr=False)
|
|
@@ -76,7 +75,6 @@ class Funda:
|
|
|
76
75
|
max_retries=self.max_retries,
|
|
77
76
|
retry_backoff=self.retry_backoff,
|
|
78
77
|
min_request_interval=self.min_request_interval,
|
|
79
|
-
adaptive_throttle=self.adaptive_throttle,
|
|
80
78
|
)
|
|
81
79
|
|
|
82
80
|
def close(self) -> None:
|
|
@@ -508,7 +506,6 @@ class Funda:
|
|
|
508
506
|
max_retries=self.max_retries,
|
|
509
507
|
retry_backoff=self.retry_backoff,
|
|
510
508
|
min_request_interval=self.min_request_interval,
|
|
511
|
-
adaptive_throttle=self.adaptive_throttle,
|
|
512
509
|
)
|
|
513
510
|
|
|
514
511
|
def _listing_id_from_input(self, listing_id: int | str) -> str:
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pyfunda"
|
|
7
|
-
version = "3.1.
|
|
7
|
+
version = "3.1.4"
|
|
8
8
|
description = "Python API wrapper for Funda.nl, the Dutch real estate platform. Reverse-engineered mobile API client, no scraping, no Selenium, no CAPTCHA."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "AGPL-3.0-or-later"
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import Mock, patch
|
|
3
|
+
|
|
4
|
+
from funda._parallel import _ParallelRunner
|
|
5
|
+
from funda._transport import _FundaTransport
|
|
6
|
+
from funda.constants import API_LISTING_TINY, FINGERPRINT_POOL
|
|
7
|
+
from funda.exceptions import FundaRequestError
|
|
8
|
+
from funda.headers import make_headers
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
TEST_URL = API_LISTING_TINY.format(tiny_id=43117443)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FakeResponse:
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
status_code: int,
|
|
18
|
+
*,
|
|
19
|
+
headers: dict[str, str] | None = None,
|
|
20
|
+
text: str = "",
|
|
21
|
+
) -> None:
|
|
22
|
+
self.status_code = status_code
|
|
23
|
+
self.headers = headers or {}
|
|
24
|
+
self.text = text
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class RetryTransport(_FundaTransport):
|
|
28
|
+
def __init__(self, statuses: list[int], *, start_index: int = 0) -> None:
|
|
29
|
+
super().__init__(max_retries=len(statuses) - 1, retry_backoff=0)
|
|
30
|
+
self.statuses = statuses
|
|
31
|
+
self.starts: list[int] = []
|
|
32
|
+
self._fingerprint = FINGERPRINT_POOL[start_index]
|
|
33
|
+
self._fingerprint_index = start_index
|
|
34
|
+
|
|
35
|
+
def _send_once(self, *args, **kwargs):
|
|
36
|
+
if self._fingerprint is None:
|
|
37
|
+
index, fingerprint, selected_proven = self._select_fingerprint(
|
|
38
|
+
kwargs["fingerprint_start_index"]
|
|
39
|
+
)
|
|
40
|
+
self._fingerprint_index = index
|
|
41
|
+
self._fingerprint = fingerprint
|
|
42
|
+
self._fingerprint_proven = self._fingerprint_proven or selected_proven
|
|
43
|
+
self.starts.append(self._fingerprint_index)
|
|
44
|
+
return FakeResponse(self.statuses.pop(0))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TransportTests(unittest.TestCase):
|
|
48
|
+
def setUp(self) -> None:
|
|
49
|
+
for transport_type in (_FundaTransport, RetryTransport):
|
|
50
|
+
transport_type._cached_fingerprint_index = None
|
|
51
|
+
transport_type._last_request_at.clear()
|
|
52
|
+
|
|
53
|
+
def test_fingerprint_pool_entries_have_required_fields(self) -> None:
|
|
54
|
+
self.assertGreaterEqual(len(FINGERPRINT_POOL), 1)
|
|
55
|
+
|
|
56
|
+
for fingerprint in FINGERPRINT_POOL:
|
|
57
|
+
with self.subTest(fingerprint=fingerprint):
|
|
58
|
+
self.assertIn(
|
|
59
|
+
fingerprint.kind,
|
|
60
|
+
{"tls_ja3", "curl_ja3", "curl_impersonate", "tls_client"},
|
|
61
|
+
)
|
|
62
|
+
if fingerprint.kind in {"tls_ja3", "curl_ja3"}:
|
|
63
|
+
self.assertTrue(fingerprint.ja3)
|
|
64
|
+
elif fingerprint.kind == "curl_impersonate":
|
|
65
|
+
self.assertTrue(fingerprint.target)
|
|
66
|
+
elif fingerprint.kind == "tls_client":
|
|
67
|
+
self.assertTrue(fingerprint.identifier)
|
|
68
|
+
|
|
69
|
+
def test_search_headers_are_convertible_for_tls_client(self) -> None:
|
|
70
|
+
header_list = make_headers("search")
|
|
71
|
+
header_dict = dict(header_list)
|
|
72
|
+
|
|
73
|
+
self.assertEqual(len(header_list), len(header_dict))
|
|
74
|
+
self.assertEqual(header_dict["referer"], "https://www.funda.nl/")
|
|
75
|
+
self.assertEqual(header_dict["accept"], "application/json")
|
|
76
|
+
|
|
77
|
+
def test_transport_validates_retry_settings(self) -> None:
|
|
78
|
+
with self.assertRaises(ValueError):
|
|
79
|
+
_FundaTransport(max_retries=-1)
|
|
80
|
+
with self.assertRaises(ValueError):
|
|
81
|
+
_FundaTransport(retry_backoff=-0.1)
|
|
82
|
+
with self.assertRaises(ValueError):
|
|
83
|
+
_FundaTransport(min_request_interval=-0.1)
|
|
84
|
+
|
|
85
|
+
def test_reset_transport_closes_sessions_and_rotates(self) -> None:
|
|
86
|
+
transport = _FundaTransport(retry_backoff=0)
|
|
87
|
+
curl_session = Mock()
|
|
88
|
+
tls_session = object()
|
|
89
|
+
transport._curl_session = curl_session
|
|
90
|
+
transport._tls_session = tls_session
|
|
91
|
+
transport._fingerprint = FINGERPRINT_POOL[0]
|
|
92
|
+
transport._fingerprint_index = 0
|
|
93
|
+
|
|
94
|
+
next_index = transport._reset_transport(rotate=True)
|
|
95
|
+
|
|
96
|
+
curl_session.close.assert_called_once()
|
|
97
|
+
self.assertIsNone(transport._curl_session)
|
|
98
|
+
self.assertIsNone(transport._tls_session)
|
|
99
|
+
self.assertIsNone(transport._fingerprint)
|
|
100
|
+
self.assertIsNone(transport._fingerprint_index)
|
|
101
|
+
self.assertFalse(transport._fingerprint_proven)
|
|
102
|
+
self.assertEqual(next_index, 1)
|
|
103
|
+
|
|
104
|
+
def test_reset_transport_preserves_proven_fingerprint_index(self) -> None:
|
|
105
|
+
transport = _FundaTransport(retry_backoff=0)
|
|
106
|
+
transport._fingerprint = FINGERPRINT_POOL[3]
|
|
107
|
+
transport._fingerprint_index = 3
|
|
108
|
+
transport._fingerprint_proven = True
|
|
109
|
+
|
|
110
|
+
next_index = transport._reset_transport(rotate=False)
|
|
111
|
+
|
|
112
|
+
self.assertEqual(next_index, 3)
|
|
113
|
+
self.assertTrue(transport._fingerprint_proven)
|
|
114
|
+
|
|
115
|
+
def test_request_rotates_unproven_fingerprint_after_403(self) -> None:
|
|
116
|
+
transport = RetryTransport([403, 200])
|
|
117
|
+
type(transport)._cached_fingerprint_index = None
|
|
118
|
+
|
|
119
|
+
response = transport.request("GET", TEST_URL)
|
|
120
|
+
|
|
121
|
+
self.assertEqual(response.status_code, 200)
|
|
122
|
+
self.assertEqual(transport.starts, [0, 1])
|
|
123
|
+
|
|
124
|
+
def test_request_keeps_proven_fingerprint_after_403(self) -> None:
|
|
125
|
+
transport = RetryTransport([200, 403, 200], start_index=3)
|
|
126
|
+
|
|
127
|
+
first = transport.request("GET", TEST_URL)
|
|
128
|
+
second = transport.request("GET", TEST_URL)
|
|
129
|
+
|
|
130
|
+
self.assertEqual(first.status_code, 200)
|
|
131
|
+
self.assertEqual(second.status_code, 200)
|
|
132
|
+
self.assertEqual(transport.starts, [3, 3, 3])
|
|
133
|
+
self.assertEqual(type(transport)._cached_fingerprint_index, 3)
|
|
134
|
+
|
|
135
|
+
def test_proven_fingerprint_stays_sticky_if_global_preference_changes(self) -> None:
|
|
136
|
+
transport = RetryTransport([200, 403, 200], start_index=3)
|
|
137
|
+
|
|
138
|
+
transport.request("GET", TEST_URL)
|
|
139
|
+
type(transport)._cached_fingerprint_index = 4
|
|
140
|
+
response = transport.request("GET", TEST_URL)
|
|
141
|
+
|
|
142
|
+
self.assertEqual(response.status_code, 200)
|
|
143
|
+
self.assertEqual(transport.starts, [3, 3, 3])
|
|
144
|
+
|
|
145
|
+
def test_proven_fingerprint_stays_sticky_for_retry_statuses(self) -> None:
|
|
146
|
+
for status in (403, 429, 503):
|
|
147
|
+
with self.subTest(status=status):
|
|
148
|
+
transport = RetryTransport([200, status, 200], start_index=3)
|
|
149
|
+
|
|
150
|
+
transport.request("GET", TEST_URL)
|
|
151
|
+
response = transport.request("GET", TEST_URL)
|
|
152
|
+
|
|
153
|
+
self.assertEqual(response.status_code, 200)
|
|
154
|
+
self.assertEqual(transport.starts, [3, 3, 3])
|
|
155
|
+
|
|
156
|
+
def test_rate_limit_does_not_rotate_unproven_fingerprint(self) -> None:
|
|
157
|
+
transport = RetryTransport([429, 200], start_index=3)
|
|
158
|
+
|
|
159
|
+
response = transport.request("GET", TEST_URL)
|
|
160
|
+
|
|
161
|
+
self.assertEqual(response.status_code, 200)
|
|
162
|
+
self.assertEqual(transport.starts, [3, 3])
|
|
163
|
+
|
|
164
|
+
def test_akamai_rejection_uses_exponential_backoff(self) -> None:
|
|
165
|
+
transport = RetryTransport([403, 403, 200])
|
|
166
|
+
transport.retry_backoff = 0.5
|
|
167
|
+
|
|
168
|
+
with patch("funda._transport.time.sleep") as sleep:
|
|
169
|
+
response = transport.request("GET", TEST_URL)
|
|
170
|
+
|
|
171
|
+
self.assertEqual(response.status_code, 200)
|
|
172
|
+
self.assertEqual([call.args[0] for call in sleep.call_args_list], [0.5, 1.0])
|
|
173
|
+
|
|
174
|
+
def test_request_raises_after_retry_statuses_are_exhausted(self) -> None:
|
|
175
|
+
with self.assertRaises(FundaRequestError):
|
|
176
|
+
RetryTransport([403, 403]).request("GET", TEST_URL)
|
|
177
|
+
|
|
178
|
+
def test_akamai_rejection_explains_ip_block(self) -> None:
|
|
179
|
+
transport = RetryTransport([403])
|
|
180
|
+
|
|
181
|
+
with patch.object(
|
|
182
|
+
transport,
|
|
183
|
+
"_send_once",
|
|
184
|
+
return_value=FakeResponse(
|
|
185
|
+
403,
|
|
186
|
+
headers={"Server": "AkamaiGHost"},
|
|
187
|
+
text="<title>Access Denied</title>",
|
|
188
|
+
),
|
|
189
|
+
), self.assertRaises(FundaRequestError) as ctx:
|
|
190
|
+
transport.request("POST", "https://listing-search-wonen.funda.io/_msearch/template")
|
|
191
|
+
|
|
192
|
+
self.assertIn("IP-based", str(ctx.exception))
|
|
193
|
+
self.assertIn("rate or reputation block", str(ctx.exception))
|
|
194
|
+
self.assertIn("retry after a cooldown", str(ctx.exception))
|
|
195
|
+
|
|
196
|
+
def test_rate_limit_statuses_use_exponential_backoff(self) -> None:
|
|
197
|
+
transport = _FundaTransport(retry_backoff=0.5)
|
|
198
|
+
|
|
199
|
+
with patch("funda._transport.time.sleep") as sleep:
|
|
200
|
+
for attempt in range(4):
|
|
201
|
+
transport._sleep_backoff(attempt, status_code=403)
|
|
202
|
+
|
|
203
|
+
self.assertEqual(
|
|
204
|
+
[call.args[0] for call in sleep.call_args_list],
|
|
205
|
+
[0.5, 1.0, 2.0, 4.0],
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
def test_pacing_can_be_disabled(self) -> None:
|
|
209
|
+
transport = _FundaTransport(min_request_interval=0)
|
|
210
|
+
|
|
211
|
+
with patch("funda._transport.time.monotonic") as monotonic, patch(
|
|
212
|
+
"funda._transport.time.sleep"
|
|
213
|
+
) as sleep:
|
|
214
|
+
transport._pace_request("https://listing-search-wonen.funda.io/first")
|
|
215
|
+
|
|
216
|
+
monotonic.assert_not_called()
|
|
217
|
+
sleep.assert_not_called()
|
|
218
|
+
|
|
219
|
+
def test_pacing_is_shared_by_host_across_transports(self) -> None:
|
|
220
|
+
first = _FundaTransport(min_request_interval=0.5)
|
|
221
|
+
second = _FundaTransport(min_request_interval=0.5)
|
|
222
|
+
type(first)._last_request_at.clear()
|
|
223
|
+
|
|
224
|
+
with patch("funda._transport.time.monotonic", side_effect=[10.0, 10.1, 10.5]), patch(
|
|
225
|
+
"funda._transport.time.sleep"
|
|
226
|
+
) as sleep:
|
|
227
|
+
first._pace_request("https://listing-search-wonen.funda.io/first")
|
|
228
|
+
second._pace_request("https://listing-detail-page.funda.io/second")
|
|
229
|
+
|
|
230
|
+
sleep.assert_called_once()
|
|
231
|
+
self.assertAlmostEqual(sleep.call_args.args[0], 0.4)
|
|
232
|
+
self.assertEqual(set(type(first)._last_request_at), {"funda.io"})
|
|
233
|
+
|
|
234
|
+
def test_client_defaults_to_half_second_pacing(self) -> None:
|
|
235
|
+
from funda import Funda
|
|
236
|
+
|
|
237
|
+
client = Funda()
|
|
238
|
+
try:
|
|
239
|
+
self.assertEqual(client.min_request_interval, 0.5)
|
|
240
|
+
self.assertEqual(client.retry_backoff, 0.5)
|
|
241
|
+
finally:
|
|
242
|
+
client.close()
|
|
243
|
+
|
|
244
|
+
def test_parallel_clients_preserve_request_interval(self) -> None:
|
|
245
|
+
from funda import Funda
|
|
246
|
+
|
|
247
|
+
client = Funda(min_request_interval=1.25)
|
|
248
|
+
parallel = client._parallel_client()
|
|
249
|
+
try:
|
|
250
|
+
self.assertEqual(parallel.min_request_interval, 1.25)
|
|
251
|
+
self.assertEqual(parallel._transport.min_request_interval, 1.25)
|
|
252
|
+
finally:
|
|
253
|
+
client.close()
|
|
254
|
+
parallel.close()
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class ParallelRunnerTests(unittest.TestCase):
|
|
258
|
+
def test_runner_reuses_thread_local_clients_and_preserves_order(self) -> None:
|
|
259
|
+
class Client:
|
|
260
|
+
created = 0
|
|
261
|
+
closed = 0
|
|
262
|
+
|
|
263
|
+
def __init__(self) -> None:
|
|
264
|
+
type(self).created += 1
|
|
265
|
+
self.id = type(self).created
|
|
266
|
+
|
|
267
|
+
def close(self) -> None:
|
|
268
|
+
type(self).closed += 1
|
|
269
|
+
|
|
270
|
+
runner = _ParallelRunner(Client, lambda client: client.close())
|
|
271
|
+
try:
|
|
272
|
+
first = runner.map(lambda client, item: (client.id, item * 2), range(8), workers=4)
|
|
273
|
+
second = runner.map(lambda client, item: (client.id, item * 3), range(8), workers=4)
|
|
274
|
+
|
|
275
|
+
self.assertEqual([value for _, value in first], [item * 2 for item in range(8)])
|
|
276
|
+
self.assertEqual([value for _, value in second], [item * 3 for item in range(8)])
|
|
277
|
+
self.assertLessEqual(Client.created, 4)
|
|
278
|
+
finally:
|
|
279
|
+
runner.close()
|
|
280
|
+
|
|
281
|
+
self.assertEqual(Client.closed, Client.created)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
if __name__ == "__main__":
|
|
285
|
+
unittest.main()
|
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
import unittest
|
|
2
|
-
from unittest.mock import Mock
|
|
3
|
-
|
|
4
|
-
from funda._parallel import _ParallelRunner
|
|
5
|
-
from funda._transport import _FundaTransport
|
|
6
|
-
from funda.constants import API_LISTING_TINY, FINGERPRINT_POOL
|
|
7
|
-
from funda.exceptions import FundaRequestError
|
|
8
|
-
from funda.headers import make_headers
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
TEST_URL = API_LISTING_TINY.format(tiny_id=43117443)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class FakeResponse:
|
|
15
|
-
def __init__(self, status_code: int) -> None:
|
|
16
|
-
self.status_code = status_code
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class RetryTransport(_FundaTransport):
|
|
20
|
-
def __init__(self, statuses: list[int]) -> None:
|
|
21
|
-
super().__init__(max_retries=len(statuses) - 1, retry_backoff=0)
|
|
22
|
-
self.statuses = statuses
|
|
23
|
-
self.starts: list[int] = []
|
|
24
|
-
self._fingerprint = FINGERPRINT_POOL[0]
|
|
25
|
-
self._fingerprint_index = 0
|
|
26
|
-
|
|
27
|
-
def _send_once(self, *args, **kwargs):
|
|
28
|
-
self.starts.append(kwargs["fingerprint_start_index"])
|
|
29
|
-
return FakeResponse(self.statuses.pop(0))
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
class TransportTests(unittest.TestCase):
|
|
33
|
-
def test_fingerprint_pool_entries_have_required_fields(self) -> None:
|
|
34
|
-
self.assertGreaterEqual(len(FINGERPRINT_POOL), 1)
|
|
35
|
-
|
|
36
|
-
for fingerprint in FINGERPRINT_POOL:
|
|
37
|
-
with self.subTest(fingerprint=fingerprint):
|
|
38
|
-
self.assertIn(
|
|
39
|
-
fingerprint.kind,
|
|
40
|
-
{"tls_ja3", "curl_ja3", "curl_impersonate", "tls_client"},
|
|
41
|
-
)
|
|
42
|
-
if fingerprint.kind in {"tls_ja3", "curl_ja3"}:
|
|
43
|
-
self.assertTrue(fingerprint.ja3)
|
|
44
|
-
elif fingerprint.kind == "curl_impersonate":
|
|
45
|
-
self.assertTrue(fingerprint.target)
|
|
46
|
-
elif fingerprint.kind == "tls_client":
|
|
47
|
-
self.assertTrue(fingerprint.identifier)
|
|
48
|
-
|
|
49
|
-
def test_search_headers_are_convertible_for_tls_client(self) -> None:
|
|
50
|
-
header_list = make_headers("search")
|
|
51
|
-
header_dict = dict(header_list)
|
|
52
|
-
|
|
53
|
-
self.assertEqual(len(header_list), len(header_dict))
|
|
54
|
-
self.assertEqual(header_dict["referer"], "https://www.funda.nl/")
|
|
55
|
-
self.assertEqual(header_dict["accept"], "application/json")
|
|
56
|
-
|
|
57
|
-
def test_transport_validates_retry_settings(self) -> None:
|
|
58
|
-
with self.assertRaises(ValueError):
|
|
59
|
-
_FundaTransport(max_retries=-1)
|
|
60
|
-
with self.assertRaises(ValueError):
|
|
61
|
-
_FundaTransport(retry_backoff=-0.1)
|
|
62
|
-
|
|
63
|
-
def test_reset_transport_closes_sessions_and_rotates(self) -> None:
|
|
64
|
-
transport = _FundaTransport(retry_backoff=0)
|
|
65
|
-
curl_session = Mock()
|
|
66
|
-
tls_session = object()
|
|
67
|
-
transport._curl_session = curl_session
|
|
68
|
-
transport._tls_session = tls_session
|
|
69
|
-
transport._fingerprint = FINGERPRINT_POOL[0]
|
|
70
|
-
transport._fingerprint_index = 0
|
|
71
|
-
|
|
72
|
-
next_index = transport._reset_transport(rotate=True)
|
|
73
|
-
|
|
74
|
-
curl_session.close.assert_called_once()
|
|
75
|
-
self.assertIsNone(transport._curl_session)
|
|
76
|
-
self.assertIsNone(transport._tls_session)
|
|
77
|
-
self.assertIsNone(transport._fingerprint)
|
|
78
|
-
self.assertIsNone(transport._fingerprint_index)
|
|
79
|
-
self.assertEqual(next_index, 1)
|
|
80
|
-
|
|
81
|
-
def test_request_retries_retry_status_with_next_fingerprint(self) -> None:
|
|
82
|
-
transport = RetryTransport([403, 200])
|
|
83
|
-
|
|
84
|
-
response = transport.request("GET", TEST_URL)
|
|
85
|
-
|
|
86
|
-
self.assertEqual(response.status_code, 200)
|
|
87
|
-
self.assertEqual(transport.starts, [0, 1])
|
|
88
|
-
|
|
89
|
-
def test_request_raises_after_retry_statuses_are_exhausted(self) -> None:
|
|
90
|
-
with self.assertRaises(FundaRequestError):
|
|
91
|
-
RetryTransport([403, 403]).request("GET", TEST_URL)
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
class ParallelRunnerTests(unittest.TestCase):
|
|
95
|
-
def test_runner_reuses_thread_local_clients_and_preserves_order(self) -> None:
|
|
96
|
-
class Client:
|
|
97
|
-
created = 0
|
|
98
|
-
closed = 0
|
|
99
|
-
|
|
100
|
-
def __init__(self) -> None:
|
|
101
|
-
type(self).created += 1
|
|
102
|
-
self.id = type(self).created
|
|
103
|
-
|
|
104
|
-
def close(self) -> None:
|
|
105
|
-
type(self).closed += 1
|
|
106
|
-
|
|
107
|
-
runner = _ParallelRunner(Client, lambda client: client.close())
|
|
108
|
-
try:
|
|
109
|
-
first = runner.map(lambda client, item: (client.id, item * 2), range(8), workers=4)
|
|
110
|
-
second = runner.map(lambda client, item: (client.id, item * 3), range(8), workers=4)
|
|
111
|
-
|
|
112
|
-
self.assertEqual([value for _, value in first], [item * 2 for item in range(8)])
|
|
113
|
-
self.assertEqual([value for _, value in second], [item * 3 for item in range(8)])
|
|
114
|
-
self.assertLessEqual(Client.created, 4)
|
|
115
|
-
finally:
|
|
116
|
-
runner.close()
|
|
117
|
-
|
|
118
|
-
self.assertEqual(Client.closed, Client.created)
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if __name__ == "__main__":
|
|
122
|
-
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|