pyfunda 3.1.2__tar.gz → 3.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {pyfunda-3.1.2 → pyfunda-3.1.4}/PKG-INFO +1 -1
  2. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_transport.py +75 -19
  3. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/funda.py +1 -4
  4. {pyfunda-3.1.2 → pyfunda-3.1.4}/pyproject.toml +1 -1
  5. pyfunda-3.1.4/tests/test_transport_parallel.py +285 -0
  6. pyfunda-3.1.2/tests/test_transport_parallel.py +0 -122
  7. {pyfunda-3.1.2 → pyfunda-3.1.4}/.dockerignore +0 -0
  8. {pyfunda-3.1.2 → pyfunda-3.1.4}/.github/FUNDING.yml +0 -0
  9. {pyfunda-3.1.2 → pyfunda-3.1.4}/.github/workflows/publish.yml +0 -0
  10. {pyfunda-3.1.2 → pyfunda-3.1.4}/.gitignore +0 -0
  11. {pyfunda-3.1.2 → pyfunda-3.1.4}/Dockerfile +0 -0
  12. {pyfunda-3.1.2 → pyfunda-3.1.4}/LICENSE +0 -0
  13. {pyfunda-3.1.2 → pyfunda-3.1.4}/README.md +0 -0
  14. {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/API.md +0 -0
  15. {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/ARCHITECTURE.md +0 -0
  16. {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/DEVELOPMENT.md +0 -0
  17. {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/EXAMPLES.md +0 -0
  18. {pyfunda-3.1.2 → pyfunda-3.1.4}/docs/README.md +0 -0
  19. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/almere_age_rank.py +0 -0
  20. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/analysis.ipynb +0 -0
  21. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/batch_details.py +0 -0
  22. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/broker_due_diligence.py +0 -0
  23. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/enrichment_export.py +0 -0
  24. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/export_to_csv.py +0 -0
  25. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/full_api_walkthrough.py +0 -0
  26. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/neighborhood_market_snapshot.py +0 -0
  27. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/new_listings_alert.py +0 -0
  28. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/poll_new_listings.py +0 -0
  29. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/price_history.py +0 -0
  30. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/price_tracker.py +0 -0
  31. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/search_sold.py +0 -0
  32. {pyfunda-3.1.2 → pyfunda-3.1.4}/examples/similar_sales_comp.py +0 -0
  33. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/__init__.py +0 -0
  34. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_autocomplete.py +0 -0
  35. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_autocomplete_parser.py +0 -0
  36. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_detail_parser.py +0 -0
  37. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_enrichment_parser.py +0 -0
  38. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_parallel.py +0 -0
  39. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_parse_helpers.py +0 -0
  40. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_price_history_parser.py +0 -0
  41. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/_search_parser.py +0 -0
  42. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/constants.py +0 -0
  43. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/exceptions.py +0 -0
  44. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/headers.py +0 -0
  45. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/listing.py +0 -0
  46. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/models.py +0 -0
  47. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/parsing.py +0 -0
  48. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/py.typed +0 -0
  49. {pyfunda-3.1.2 → pyfunda-3.1.4}/funda/search.py +0 -0
  50. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/__init__.py +0 -0
  51. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_autocomplete.py +0 -0
  52. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_client.py +0 -0
  53. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_enrichment_parser.py +0 -0
  54. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_live.py +0 -0
  55. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_models.py +0 -0
  56. {pyfunda-3.1.2 → pyfunda-3.1.4}/tests/test_search.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyfunda
3
- Version: 3.1.2
3
+ Version: 3.1.4
4
4
  Summary: Python API wrapper for Funda.nl, the Dutch real estate platform. Reverse-engineered mobile API client, no scraping, no Selenium, no CAPTCHA.
5
5
  Project-URL: Homepage, https://github.com/0xMH/pyfunda
6
6
  Project-URL: Repository, https://github.com/0xMH/pyfunda
@@ -5,6 +5,7 @@ from collections.abc import Callable
5
5
  from dataclasses import dataclass, field
6
6
  from threading import Lock
7
7
  from typing import Any, ClassVar
8
+ from urllib.parse import urlsplit
8
9
 
9
10
  from curl_cffi import requests
10
11
  import tls_client
@@ -20,22 +21,28 @@ from funda.models import Fingerprint, HttpMethod, JsonDict, RequestProfile
20
21
  class _FundaTransport:
21
22
  _cached_fingerprint_index: ClassVar[int | None] = None
22
23
  _fingerprint_cache_lock: ClassVar[Any] = Lock()
24
+ _request_pacing_lock: ClassVar[Any] = Lock()
25
+ _last_request_at: ClassVar[dict[str, float]] = {}
23
26
 
24
27
  timeout: int = 30
25
28
  max_retries: int = DEFAULT_MAX_RETRIES
26
- retry_backoff: float = 0.1
29
+ retry_backoff: float = 0.5
30
+ min_request_interval: float = 0.5
27
31
  retry_statuses: set[int] | None = None
28
32
 
29
33
  _curl_session: requests.Session | None = field(default=None, init=False, repr=False)
30
34
  _tls_session: tls_client.Session | None = field(default=None, init=False, repr=False)
31
35
  _fingerprint: Fingerprint | None = field(default=None, init=False, repr=False)
32
36
  _fingerprint_index: int | None = field(default=None, init=False, repr=False)
37
+ _fingerprint_proven: bool = field(default=False, init=False, repr=False)
33
38
 
34
39
  def __post_init__(self) -> None:
35
40
  if self.max_retries < 0:
36
41
  raise ValueError("max_retries must be >= 0")
37
42
  if self.retry_backoff < 0:
38
43
  raise ValueError("retry_backoff must be >= 0")
44
+ if self.min_request_interval < 0:
45
+ raise ValueError("min_request_interval must be >= 0")
39
46
  if self.retry_statuses is None:
40
47
  self.retry_statuses = set(RETRY_STATUS_CODES)
41
48
  else:
@@ -62,25 +69,34 @@ class _FundaTransport:
62
69
  def _headers_dict(self, profile: RequestProfile = "listing") -> dict[str, str]:
63
70
  return dict(make_headers(profile))
64
71
 
65
- def _select_fingerprint(self, start_index: int = 0) -> tuple[int, Fingerprint]:
66
- if start_index == 0:
72
+ def _select_fingerprint(
73
+ self, start_index: int | None = None
74
+ ) -> tuple[int, Fingerprint, bool]:
75
+ if start_index is None:
67
76
  with type(self)._fingerprint_cache_lock:
68
77
  cached_index = type(self)._cached_fingerprint_index
69
78
  if cached_index is not None:
70
- return cached_index, FINGERPRINT_POOL[cached_index]
79
+ return cached_index, FINGERPRINT_POOL[cached_index], True
80
+ start_index = 0
71
81
 
72
82
  index = start_index % len(FINGERPRINT_POOL)
73
- return index, FINGERPRINT_POOL[index]
83
+ return index, FINGERPRINT_POOL[index], False
74
84
 
75
85
  def _remember_fingerprint(self) -> None:
76
86
  if self._fingerprint_index is None:
77
87
  return
88
+ self._fingerprint_proven = True
78
89
  with type(self)._fingerprint_cache_lock:
79
90
  type(self)._cached_fingerprint_index = self._fingerprint_index
80
91
 
81
- def _ensure_session(self, fingerprint_start_index: int = 0) -> None:
92
+ def _ensure_session(self, fingerprint_start_index: int | None = None) -> None:
82
93
  if self._fingerprint is None:
83
- self._fingerprint_index, self._fingerprint = self._select_fingerprint(fingerprint_start_index)
94
+ index, fingerprint, selected_proven = self._select_fingerprint(
95
+ fingerprint_start_index
96
+ )
97
+ self._fingerprint_index = index
98
+ self._fingerprint = fingerprint
99
+ self._fingerprint_proven = self._fingerprint_proven or selected_proven
84
100
 
85
101
  fingerprint = self._fingerprint
86
102
  if fingerprint.kind == "tls_ja3" and self._tls_session is None:
@@ -106,17 +122,21 @@ class _FundaTransport:
106
122
  self._curl_session = None
107
123
  self._tls_session = None
108
124
 
109
- def _reset_transport(self, *, rotate: bool = False) -> int:
125
+ def _reset_transport(self, *, rotate: bool = False) -> int | None:
110
126
  current_index = self._fingerprint_index
111
127
  self._close_sessions()
112
128
  self._fingerprint = None
113
129
  self._fingerprint_index = None
114
- if rotate and current_index is not None:
130
+ if current_index is None:
131
+ self._fingerprint_proven = False
132
+ return None
133
+ if rotate:
134
+ self._fingerprint_proven = False
115
135
  with type(self)._fingerprint_cache_lock:
116
136
  if type(self)._cached_fingerprint_index == current_index:
117
137
  type(self)._cached_fingerprint_index = None
118
138
  return (current_index + 1) % len(FINGERPRINT_POOL)
119
- return 0
139
+ return current_index
120
140
 
121
141
  def get(self, url: str, *, profile: RequestProfile = "listing") -> Any:
122
142
  return self.request("GET", url, profile=profile)
@@ -140,7 +160,7 @@ class _FundaTransport:
140
160
  data: str | None = None,
141
161
  json_data: JsonDict | None = None,
142
162
  ) -> Any:
143
- fingerprint_start_index = 0
163
+ fingerprint_start_index: int | None = None
144
164
 
145
165
  for attempt in range(self.max_retries + 1):
146
166
  try:
@@ -155,7 +175,9 @@ class _FundaTransport:
155
175
  except (requests.errors.RequestsError, TLSClientExeption):
156
176
  if attempt >= self.max_retries:
157
177
  raise
158
- fingerprint_start_index = self._reset_transport(rotate=True)
178
+ fingerprint_start_index = self._reset_transport(
179
+ rotate=not self._fingerprint_proven
180
+ )
159
181
  self._sleep_backoff(attempt)
160
182
  continue
161
183
 
@@ -163,19 +185,52 @@ class _FundaTransport:
163
185
  self._remember_fingerprint()
164
186
  return response
165
187
  if attempt >= self.max_retries:
188
+ detail = self._rejection_detail(response)
166
189
  raise FundaRequestError(
167
190
  f"Funda rejected {method} {url} after {attempt + 1} attempts "
168
- f"(status {response.status_code})"
191
+ f"(status {response.status_code}){detail}"
169
192
  )
170
193
 
171
- fingerprint_start_index = self._reset_transport(rotate=True)
172
- self._sleep_backoff(attempt)
194
+ rotate = response.status_code == 403 and not self._fingerprint_proven
195
+ fingerprint_start_index = self._reset_transport(rotate=rotate)
196
+ self._sleep_backoff(attempt, status_code=response.status_code)
173
197
 
174
198
  raise AssertionError("unreachable: retry loop must return or raise")
175
199
 
176
- def _sleep_backoff(self, attempt: int) -> None:
177
- if self.retry_backoff > 0:
178
- time.sleep(self.retry_backoff * (attempt + 1))
200
+ def _sleep_backoff(self, attempt: int, *, status_code: int | None = None) -> None:
201
+ if self.retry_backoff <= 0:
202
+ return
203
+ multiplier = 2**attempt if status_code in (403, 429) else attempt + 1
204
+ time.sleep(self.retry_backoff * multiplier)
205
+
206
+ @staticmethod
207
+ def _rejection_detail(response: Any) -> str:
208
+ headers = getattr(response, "headers", {})
209
+ server = headers.get("server", headers.get("Server", ""))
210
+ text = getattr(response, "text", "")
211
+ if "AkamaiGHost" in server or "Access Denied" in text[:500]:
212
+ return (
213
+ ": Akamai denied the request; this is usually an IP-based rate or "
214
+ "reputation block, not a search payload error. Stop requests and retry "
215
+ "after a cooldown or from a different network"
216
+ )
217
+ return ""
218
+
219
+ def _pace_request(self, url: str) -> None:
220
+ if self.min_request_interval <= 0:
221
+ return
222
+
223
+ hostname = urlsplit(url).hostname or ""
224
+ pacing_key = "funda.io" if hostname.endswith(".funda.io") else hostname
225
+ with type(self)._request_pacing_lock:
226
+ now = time.monotonic()
227
+ last_request_at = type(self)._last_request_at.get(pacing_key)
228
+ if last_request_at is not None:
229
+ delay = self.min_request_interval - (now - last_request_at)
230
+ if delay > 0:
231
+ time.sleep(delay)
232
+ now = time.monotonic()
233
+ type(self)._last_request_at[pacing_key] = now
179
234
 
180
235
  def _send_once(
181
236
  self,
@@ -185,8 +240,9 @@ class _FundaTransport:
185
240
  profile: RequestProfile = "listing",
186
241
  data: str | None = None,
187
242
  json_data: JsonDict | None = None,
188
- fingerprint_start_index: int = 0,
243
+ fingerprint_start_index: int | None = None,
189
244
  ) -> Any:
245
+ self._pace_request(url)
190
246
  self._ensure_session(fingerprint_start_index=fingerprint_start_index)
191
247
  fingerprint = self._fingerprint
192
248
  if fingerprint is None:
@@ -64,8 +64,7 @@ class Funda:
64
64
  timeout: int = 30
65
65
  max_retries: int = DEFAULT_MAX_RETRIES
66
66
  retry_backoff: float = 0.5
67
- min_request_interval: float = 0.0
68
- adaptive_throttle: bool = True
67
+ min_request_interval: float = 0.5
69
68
 
70
69
  _transport: _FundaTransport = field(init=False, repr=False)
71
70
  _parallel_runner: _ParallelRunner["Funda"] | None = field(default=None, init=False, repr=False)
@@ -76,7 +75,6 @@ class Funda:
76
75
  max_retries=self.max_retries,
77
76
  retry_backoff=self.retry_backoff,
78
77
  min_request_interval=self.min_request_interval,
79
- adaptive_throttle=self.adaptive_throttle,
80
78
  )
81
79
 
82
80
  def close(self) -> None:
@@ -508,7 +506,6 @@ class Funda:
508
506
  max_retries=self.max_retries,
509
507
  retry_backoff=self.retry_backoff,
510
508
  min_request_interval=self.min_request_interval,
511
- adaptive_throttle=self.adaptive_throttle,
512
509
  )
513
510
 
514
511
  def _listing_id_from_input(self, listing_id: int | str) -> str:
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "pyfunda"
7
- version = "3.1.2"
7
+ version = "3.1.4"
8
8
  description = "Python API wrapper for Funda.nl, the Dutch real estate platform. Reverse-engineered mobile API client, no scraping, no Selenium, no CAPTCHA."
9
9
  readme = "README.md"
10
10
  license = "AGPL-3.0-or-later"
@@ -0,0 +1,285 @@
1
+ import unittest
2
+ from unittest.mock import Mock, patch
3
+
4
+ from funda._parallel import _ParallelRunner
5
+ from funda._transport import _FundaTransport
6
+ from funda.constants import API_LISTING_TINY, FINGERPRINT_POOL
7
+ from funda.exceptions import FundaRequestError
8
+ from funda.headers import make_headers
9
+
10
+
11
+ TEST_URL = API_LISTING_TINY.format(tiny_id=43117443)
12
+
13
+
14
+ class FakeResponse:
15
+ def __init__(
16
+ self,
17
+ status_code: int,
18
+ *,
19
+ headers: dict[str, str] | None = None,
20
+ text: str = "",
21
+ ) -> None:
22
+ self.status_code = status_code
23
+ self.headers = headers or {}
24
+ self.text = text
25
+
26
+
27
+ class RetryTransport(_FundaTransport):
28
+ def __init__(self, statuses: list[int], *, start_index: int = 0) -> None:
29
+ super().__init__(max_retries=len(statuses) - 1, retry_backoff=0)
30
+ self.statuses = statuses
31
+ self.starts: list[int] = []
32
+ self._fingerprint = FINGERPRINT_POOL[start_index]
33
+ self._fingerprint_index = start_index
34
+
35
+ def _send_once(self, *args, **kwargs):
36
+ if self._fingerprint is None:
37
+ index, fingerprint, selected_proven = self._select_fingerprint(
38
+ kwargs["fingerprint_start_index"]
39
+ )
40
+ self._fingerprint_index = index
41
+ self._fingerprint = fingerprint
42
+ self._fingerprint_proven = self._fingerprint_proven or selected_proven
43
+ self.starts.append(self._fingerprint_index)
44
+ return FakeResponse(self.statuses.pop(0))
45
+
46
+
47
+ class TransportTests(unittest.TestCase):
48
+ def setUp(self) -> None:
49
+ for transport_type in (_FundaTransport, RetryTransport):
50
+ transport_type._cached_fingerprint_index = None
51
+ transport_type._last_request_at.clear()
52
+
53
+ def test_fingerprint_pool_entries_have_required_fields(self) -> None:
54
+ self.assertGreaterEqual(len(FINGERPRINT_POOL), 1)
55
+
56
+ for fingerprint in FINGERPRINT_POOL:
57
+ with self.subTest(fingerprint=fingerprint):
58
+ self.assertIn(
59
+ fingerprint.kind,
60
+ {"tls_ja3", "curl_ja3", "curl_impersonate", "tls_client"},
61
+ )
62
+ if fingerprint.kind in {"tls_ja3", "curl_ja3"}:
63
+ self.assertTrue(fingerprint.ja3)
64
+ elif fingerprint.kind == "curl_impersonate":
65
+ self.assertTrue(fingerprint.target)
66
+ elif fingerprint.kind == "tls_client":
67
+ self.assertTrue(fingerprint.identifier)
68
+
69
+ def test_search_headers_are_convertible_for_tls_client(self) -> None:
70
+ header_list = make_headers("search")
71
+ header_dict = dict(header_list)
72
+
73
+ self.assertEqual(len(header_list), len(header_dict))
74
+ self.assertEqual(header_dict["referer"], "https://www.funda.nl/")
75
+ self.assertEqual(header_dict["accept"], "application/json")
76
+
77
+ def test_transport_validates_retry_settings(self) -> None:
78
+ with self.assertRaises(ValueError):
79
+ _FundaTransport(max_retries=-1)
80
+ with self.assertRaises(ValueError):
81
+ _FundaTransport(retry_backoff=-0.1)
82
+ with self.assertRaises(ValueError):
83
+ _FundaTransport(min_request_interval=-0.1)
84
+
85
+ def test_reset_transport_closes_sessions_and_rotates(self) -> None:
86
+ transport = _FundaTransport(retry_backoff=0)
87
+ curl_session = Mock()
88
+ tls_session = object()
89
+ transport._curl_session = curl_session
90
+ transport._tls_session = tls_session
91
+ transport._fingerprint = FINGERPRINT_POOL[0]
92
+ transport._fingerprint_index = 0
93
+
94
+ next_index = transport._reset_transport(rotate=True)
95
+
96
+ curl_session.close.assert_called_once()
97
+ self.assertIsNone(transport._curl_session)
98
+ self.assertIsNone(transport._tls_session)
99
+ self.assertIsNone(transport._fingerprint)
100
+ self.assertIsNone(transport._fingerprint_index)
101
+ self.assertFalse(transport._fingerprint_proven)
102
+ self.assertEqual(next_index, 1)
103
+
104
+ def test_reset_transport_preserves_proven_fingerprint_index(self) -> None:
105
+ transport = _FundaTransport(retry_backoff=0)
106
+ transport._fingerprint = FINGERPRINT_POOL[3]
107
+ transport._fingerprint_index = 3
108
+ transport._fingerprint_proven = True
109
+
110
+ next_index = transport._reset_transport(rotate=False)
111
+
112
+ self.assertEqual(next_index, 3)
113
+ self.assertTrue(transport._fingerprint_proven)
114
+
115
+ def test_request_rotates_unproven_fingerprint_after_403(self) -> None:
116
+ transport = RetryTransport([403, 200])
117
+ type(transport)._cached_fingerprint_index = None
118
+
119
+ response = transport.request("GET", TEST_URL)
120
+
121
+ self.assertEqual(response.status_code, 200)
122
+ self.assertEqual(transport.starts, [0, 1])
123
+
124
+ def test_request_keeps_proven_fingerprint_after_403(self) -> None:
125
+ transport = RetryTransport([200, 403, 200], start_index=3)
126
+
127
+ first = transport.request("GET", TEST_URL)
128
+ second = transport.request("GET", TEST_URL)
129
+
130
+ self.assertEqual(first.status_code, 200)
131
+ self.assertEqual(second.status_code, 200)
132
+ self.assertEqual(transport.starts, [3, 3, 3])
133
+ self.assertEqual(type(transport)._cached_fingerprint_index, 3)
134
+
135
+ def test_proven_fingerprint_stays_sticky_if_global_preference_changes(self) -> None:
136
+ transport = RetryTransport([200, 403, 200], start_index=3)
137
+
138
+ transport.request("GET", TEST_URL)
139
+ type(transport)._cached_fingerprint_index = 4
140
+ response = transport.request("GET", TEST_URL)
141
+
142
+ self.assertEqual(response.status_code, 200)
143
+ self.assertEqual(transport.starts, [3, 3, 3])
144
+
145
+ def test_proven_fingerprint_stays_sticky_for_retry_statuses(self) -> None:
146
+ for status in (403, 429, 503):
147
+ with self.subTest(status=status):
148
+ transport = RetryTransport([200, status, 200], start_index=3)
149
+
150
+ transport.request("GET", TEST_URL)
151
+ response = transport.request("GET", TEST_URL)
152
+
153
+ self.assertEqual(response.status_code, 200)
154
+ self.assertEqual(transport.starts, [3, 3, 3])
155
+
156
+ def test_rate_limit_does_not_rotate_unproven_fingerprint(self) -> None:
157
+ transport = RetryTransport([429, 200], start_index=3)
158
+
159
+ response = transport.request("GET", TEST_URL)
160
+
161
+ self.assertEqual(response.status_code, 200)
162
+ self.assertEqual(transport.starts, [3, 3])
163
+
164
+ def test_akamai_rejection_uses_exponential_backoff(self) -> None:
165
+ transport = RetryTransport([403, 403, 200])
166
+ transport.retry_backoff = 0.5
167
+
168
+ with patch("funda._transport.time.sleep") as sleep:
169
+ response = transport.request("GET", TEST_URL)
170
+
171
+ self.assertEqual(response.status_code, 200)
172
+ self.assertEqual([call.args[0] for call in sleep.call_args_list], [0.5, 1.0])
173
+
174
+ def test_request_raises_after_retry_statuses_are_exhausted(self) -> None:
175
+ with self.assertRaises(FundaRequestError):
176
+ RetryTransport([403, 403]).request("GET", TEST_URL)
177
+
178
+ def test_akamai_rejection_explains_ip_block(self) -> None:
179
+ transport = RetryTransport([403])
180
+
181
+ with patch.object(
182
+ transport,
183
+ "_send_once",
184
+ return_value=FakeResponse(
185
+ 403,
186
+ headers={"Server": "AkamaiGHost"},
187
+ text="<title>Access Denied</title>",
188
+ ),
189
+ ), self.assertRaises(FundaRequestError) as ctx:
190
+ transport.request("POST", "https://listing-search-wonen.funda.io/_msearch/template")
191
+
192
+ self.assertIn("IP-based", str(ctx.exception))
193
+ self.assertIn("rate or reputation block", str(ctx.exception))
194
+ self.assertIn("retry after a cooldown", str(ctx.exception))
195
+
196
+ def test_rate_limit_statuses_use_exponential_backoff(self) -> None:
197
+ transport = _FundaTransport(retry_backoff=0.5)
198
+
199
+ with patch("funda._transport.time.sleep") as sleep:
200
+ for attempt in range(4):
201
+ transport._sleep_backoff(attempt, status_code=403)
202
+
203
+ self.assertEqual(
204
+ [call.args[0] for call in sleep.call_args_list],
205
+ [0.5, 1.0, 2.0, 4.0],
206
+ )
207
+
208
+ def test_pacing_can_be_disabled(self) -> None:
209
+ transport = _FundaTransport(min_request_interval=0)
210
+
211
+ with patch("funda._transport.time.monotonic") as monotonic, patch(
212
+ "funda._transport.time.sleep"
213
+ ) as sleep:
214
+ transport._pace_request("https://listing-search-wonen.funda.io/first")
215
+
216
+ monotonic.assert_not_called()
217
+ sleep.assert_not_called()
218
+
219
+ def test_pacing_is_shared_by_host_across_transports(self) -> None:
220
+ first = _FundaTransport(min_request_interval=0.5)
221
+ second = _FundaTransport(min_request_interval=0.5)
222
+ type(first)._last_request_at.clear()
223
+
224
+ with patch("funda._transport.time.monotonic", side_effect=[10.0, 10.1, 10.5]), patch(
225
+ "funda._transport.time.sleep"
226
+ ) as sleep:
227
+ first._pace_request("https://listing-search-wonen.funda.io/first")
228
+ second._pace_request("https://listing-detail-page.funda.io/second")
229
+
230
+ sleep.assert_called_once()
231
+ self.assertAlmostEqual(sleep.call_args.args[0], 0.4)
232
+ self.assertEqual(set(type(first)._last_request_at), {"funda.io"})
233
+
234
+ def test_client_defaults_to_half_second_pacing(self) -> None:
235
+ from funda import Funda
236
+
237
+ client = Funda()
238
+ try:
239
+ self.assertEqual(client.min_request_interval, 0.5)
240
+ self.assertEqual(client.retry_backoff, 0.5)
241
+ finally:
242
+ client.close()
243
+
244
+ def test_parallel_clients_preserve_request_interval(self) -> None:
245
+ from funda import Funda
246
+
247
+ client = Funda(min_request_interval=1.25)
248
+ parallel = client._parallel_client()
249
+ try:
250
+ self.assertEqual(parallel.min_request_interval, 1.25)
251
+ self.assertEqual(parallel._transport.min_request_interval, 1.25)
252
+ finally:
253
+ client.close()
254
+ parallel.close()
255
+
256
+
257
+ class ParallelRunnerTests(unittest.TestCase):
258
+ def test_runner_reuses_thread_local_clients_and_preserves_order(self) -> None:
259
+ class Client:
260
+ created = 0
261
+ closed = 0
262
+
263
+ def __init__(self) -> None:
264
+ type(self).created += 1
265
+ self.id = type(self).created
266
+
267
+ def close(self) -> None:
268
+ type(self).closed += 1
269
+
270
+ runner = _ParallelRunner(Client, lambda client: client.close())
271
+ try:
272
+ first = runner.map(lambda client, item: (client.id, item * 2), range(8), workers=4)
273
+ second = runner.map(lambda client, item: (client.id, item * 3), range(8), workers=4)
274
+
275
+ self.assertEqual([value for _, value in first], [item * 2 for item in range(8)])
276
+ self.assertEqual([value for _, value in second], [item * 3 for item in range(8)])
277
+ self.assertLessEqual(Client.created, 4)
278
+ finally:
279
+ runner.close()
280
+
281
+ self.assertEqual(Client.closed, Client.created)
282
+
283
+
284
+ if __name__ == "__main__":
285
+ unittest.main()
@@ -1,122 +0,0 @@
1
- import unittest
2
- from unittest.mock import Mock
3
-
4
- from funda._parallel import _ParallelRunner
5
- from funda._transport import _FundaTransport
6
- from funda.constants import API_LISTING_TINY, FINGERPRINT_POOL
7
- from funda.exceptions import FundaRequestError
8
- from funda.headers import make_headers
9
-
10
-
11
- TEST_URL = API_LISTING_TINY.format(tiny_id=43117443)
12
-
13
-
14
- class FakeResponse:
15
- def __init__(self, status_code: int) -> None:
16
- self.status_code = status_code
17
-
18
-
19
- class RetryTransport(_FundaTransport):
20
- def __init__(self, statuses: list[int]) -> None:
21
- super().__init__(max_retries=len(statuses) - 1, retry_backoff=0)
22
- self.statuses = statuses
23
- self.starts: list[int] = []
24
- self._fingerprint = FINGERPRINT_POOL[0]
25
- self._fingerprint_index = 0
26
-
27
- def _send_once(self, *args, **kwargs):
28
- self.starts.append(kwargs["fingerprint_start_index"])
29
- return FakeResponse(self.statuses.pop(0))
30
-
31
-
32
- class TransportTests(unittest.TestCase):
33
- def test_fingerprint_pool_entries_have_required_fields(self) -> None:
34
- self.assertGreaterEqual(len(FINGERPRINT_POOL), 1)
35
-
36
- for fingerprint in FINGERPRINT_POOL:
37
- with self.subTest(fingerprint=fingerprint):
38
- self.assertIn(
39
- fingerprint.kind,
40
- {"tls_ja3", "curl_ja3", "curl_impersonate", "tls_client"},
41
- )
42
- if fingerprint.kind in {"tls_ja3", "curl_ja3"}:
43
- self.assertTrue(fingerprint.ja3)
44
- elif fingerprint.kind == "curl_impersonate":
45
- self.assertTrue(fingerprint.target)
46
- elif fingerprint.kind == "tls_client":
47
- self.assertTrue(fingerprint.identifier)
48
-
49
- def test_search_headers_are_convertible_for_tls_client(self) -> None:
50
- header_list = make_headers("search")
51
- header_dict = dict(header_list)
52
-
53
- self.assertEqual(len(header_list), len(header_dict))
54
- self.assertEqual(header_dict["referer"], "https://www.funda.nl/")
55
- self.assertEqual(header_dict["accept"], "application/json")
56
-
57
- def test_transport_validates_retry_settings(self) -> None:
58
- with self.assertRaises(ValueError):
59
- _FundaTransport(max_retries=-1)
60
- with self.assertRaises(ValueError):
61
- _FundaTransport(retry_backoff=-0.1)
62
-
63
- def test_reset_transport_closes_sessions_and_rotates(self) -> None:
64
- transport = _FundaTransport(retry_backoff=0)
65
- curl_session = Mock()
66
- tls_session = object()
67
- transport._curl_session = curl_session
68
- transport._tls_session = tls_session
69
- transport._fingerprint = FINGERPRINT_POOL[0]
70
- transport._fingerprint_index = 0
71
-
72
- next_index = transport._reset_transport(rotate=True)
73
-
74
- curl_session.close.assert_called_once()
75
- self.assertIsNone(transport._curl_session)
76
- self.assertIsNone(transport._tls_session)
77
- self.assertIsNone(transport._fingerprint)
78
- self.assertIsNone(transport._fingerprint_index)
79
- self.assertEqual(next_index, 1)
80
-
81
- def test_request_retries_retry_status_with_next_fingerprint(self) -> None:
82
- transport = RetryTransport([403, 200])
83
-
84
- response = transport.request("GET", TEST_URL)
85
-
86
- self.assertEqual(response.status_code, 200)
87
- self.assertEqual(transport.starts, [0, 1])
88
-
89
- def test_request_raises_after_retry_statuses_are_exhausted(self) -> None:
90
- with self.assertRaises(FundaRequestError):
91
- RetryTransport([403, 403]).request("GET", TEST_URL)
92
-
93
-
94
- class ParallelRunnerTests(unittest.TestCase):
95
- def test_runner_reuses_thread_local_clients_and_preserves_order(self) -> None:
96
- class Client:
97
- created = 0
98
- closed = 0
99
-
100
- def __init__(self) -> None:
101
- type(self).created += 1
102
- self.id = type(self).created
103
-
104
- def close(self) -> None:
105
- type(self).closed += 1
106
-
107
- runner = _ParallelRunner(Client, lambda client: client.close())
108
- try:
109
- first = runner.map(lambda client, item: (client.id, item * 2), range(8), workers=4)
110
- second = runner.map(lambda client, item: (client.id, item * 3), range(8), workers=4)
111
-
112
- self.assertEqual([value for _, value in first], [item * 2 for item in range(8)])
113
- self.assertEqual([value for _, value in second], [item * 3 for item in range(8)])
114
- self.assertLessEqual(Client.created, 4)
115
- finally:
116
- runner.close()
117
-
118
- self.assertEqual(Client.closed, Client.created)
119
-
120
-
121
- if __name__ == "__main__":
122
- unittest.main()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes