crawlee 1.0.1b9__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of crawlee might be problematic. Click here for more details.

Files changed (93) hide show
  1. crawlee/__init__.py +2 -1
  2. crawlee/_browserforge_workaround.py +7 -3
  3. crawlee/_request.py +62 -32
  4. crawlee/_service_locator.py +4 -4
  5. crawlee/_types.py +52 -19
  6. crawlee/_utils/context.py +3 -3
  7. crawlee/_utils/file.py +8 -1
  8. crawlee/_utils/globs.py +4 -4
  9. crawlee/_utils/recoverable_state.py +32 -8
  10. crawlee/_utils/recurring_task.py +27 -3
  11. crawlee/_utils/robots.py +17 -5
  12. crawlee/_utils/sitemap.py +13 -6
  13. crawlee/_utils/system.py +27 -11
  14. crawlee/_utils/time.py +41 -1
  15. crawlee/_utils/urls.py +9 -2
  16. crawlee/browsers/_browser_pool.py +5 -2
  17. crawlee/browsers/_playwright_browser.py +2 -1
  18. crawlee/browsers/_playwright_browser_controller.py +1 -1
  19. crawlee/browsers/_playwright_browser_plugin.py +17 -3
  20. crawlee/browsers/_types.py +1 -1
  21. crawlee/configuration.py +3 -1
  22. crawlee/crawlers/__init__.py +5 -1
  23. crawlee/crawlers/_abstract_http/__init__.py +2 -1
  24. crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +54 -16
  25. crawlee/crawlers/_adaptive_playwright/__init__.py +5 -2
  26. crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +21 -30
  27. crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawling_context.py +6 -2
  28. crawlee/crawlers/_basic/_basic_crawler.py +160 -134
  29. crawlee/crawlers/_basic/_context_utils.py +24 -0
  30. crawlee/crawlers/_basic/_logging_utils.py +23 -4
  31. crawlee/crawlers/_beautifulsoup/_beautifulsoup_crawler.py +2 -2
  32. crawlee/crawlers/_parsel/_parsel_crawler.py +2 -2
  33. crawlee/crawlers/_parsel/_parsel_parser.py +1 -1
  34. crawlee/crawlers/_playwright/_playwright_crawler.py +68 -23
  35. crawlee/crawlers/_playwright/_playwright_http_client.py +7 -1
  36. crawlee/crawlers/_playwright/_playwright_pre_nav_crawling_context.py +4 -1
  37. crawlee/crawlers/_playwright/_types.py +12 -2
  38. crawlee/errors.py +4 -0
  39. crawlee/events/_event_manager.py +12 -6
  40. crawlee/fingerprint_suite/_browserforge_adapter.py +1 -1
  41. crawlee/fingerprint_suite/_header_generator.py +2 -2
  42. crawlee/http_clients/_base.py +4 -0
  43. crawlee/http_clients/_curl_impersonate.py +68 -14
  44. crawlee/http_clients/_httpx.py +16 -6
  45. crawlee/http_clients/_impit.py +25 -10
  46. crawlee/otel/crawler_instrumentor.py +4 -6
  47. crawlee/request_loaders/_sitemap_request_loader.py +23 -5
  48. crawlee/router.py +13 -3
  49. crawlee/sessions/_cookies.py +13 -8
  50. crawlee/sessions/_models.py +3 -3
  51. crawlee/sessions/_session_pool.py +1 -1
  52. crawlee/statistics/_error_snapshotter.py +1 -1
  53. crawlee/statistics/_models.py +51 -9
  54. crawlee/statistics/_statistics.py +24 -33
  55. crawlee/storage_clients/__init__.py +4 -0
  56. crawlee/storage_clients/_base/_dataset_client.py +2 -2
  57. crawlee/storage_clients/_base/_key_value_store_client.py +2 -2
  58. crawlee/storage_clients/_file_system/_dataset_client.py +8 -7
  59. crawlee/storage_clients/_file_system/_key_value_store_client.py +9 -6
  60. crawlee/storage_clients/_file_system/_request_queue_client.py +31 -12
  61. crawlee/storage_clients/_memory/_dataset_client.py +2 -2
  62. crawlee/storage_clients/_memory/_key_value_store_client.py +2 -2
  63. crawlee/storage_clients/_memory/_request_queue_client.py +2 -2
  64. crawlee/storage_clients/_redis/__init__.py +6 -0
  65. crawlee/storage_clients/_redis/_client_mixin.py +292 -0
  66. crawlee/storage_clients/_redis/_dataset_client.py +329 -0
  67. crawlee/storage_clients/_redis/_key_value_store_client.py +262 -0
  68. crawlee/storage_clients/_redis/_request_queue_client.py +583 -0
  69. crawlee/storage_clients/_redis/_storage_client.py +149 -0
  70. crawlee/storage_clients/_redis/_utils.py +23 -0
  71. crawlee/storage_clients/_redis/lua_scripts/atomic_bloom_add_requests.lua +36 -0
  72. crawlee/storage_clients/_redis/lua_scripts/atomic_fetch_request.lua +49 -0
  73. crawlee/storage_clients/_redis/lua_scripts/atomic_set_add_requests.lua +37 -0
  74. crawlee/storage_clients/_redis/lua_scripts/reclaim_stale_requests.lua +34 -0
  75. crawlee/storage_clients/_redis/py.typed +0 -0
  76. crawlee/storage_clients/_sql/_client_mixin.py +1 -1
  77. crawlee/storage_clients/_sql/_dataset_client.py +2 -2
  78. crawlee/storage_clients/_sql/_db_models.py +1 -2
  79. crawlee/storage_clients/_sql/_key_value_store_client.py +5 -4
  80. crawlee/storage_clients/_sql/_request_queue_client.py +20 -6
  81. crawlee/storage_clients/_sql/_storage_client.py +1 -1
  82. crawlee/storage_clients/models.py +8 -3
  83. crawlee/storages/_base.py +3 -1
  84. crawlee/storages/_dataset.py +3 -0
  85. crawlee/storages/_key_value_store.py +8 -2
  86. crawlee/storages/_request_queue.py +3 -0
  87. crawlee/storages/_storage_instance_manager.py +109 -42
  88. crawlee/storages/_utils.py +11 -0
  89. {crawlee-1.0.1b9.dist-info → crawlee-1.3.1b3.dist-info}/METADATA +14 -16
  90. {crawlee-1.0.1b9.dist-info → crawlee-1.3.1b3.dist-info}/RECORD +93 -79
  91. {crawlee-1.0.1b9.dist-info → crawlee-1.3.1b3.dist-info}/WHEEL +1 -1
  92. {crawlee-1.0.1b9.dist-info → crawlee-1.3.1b3.dist-info}/entry_points.txt +0 -0
  93. {crawlee-1.0.1b9.dist-info → crawlee-1.3.1b3.dist-info}/licenses/LICENSE +0 -0
@@ -1,7 +1,9 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import asyncio
3
4
  from contextlib import asynccontextmanager
4
- from typing import TYPE_CHECKING, Any
5
+ from http.cookiejar import Cookie
6
+ from typing import TYPE_CHECKING, Any, cast
5
7
 
6
8
  from curl_cffi import CurlInfo
7
9
  from curl_cffi.const import CurlHttpVersion
@@ -10,10 +12,11 @@ from curl_cffi.requests.cookies import Cookies as CurlCookies
10
12
  from curl_cffi.requests.cookies import CurlMorsel
11
13
  from curl_cffi.requests.exceptions import ProxyError as CurlProxyError
12
14
  from curl_cffi.requests.exceptions import RequestException as CurlRequestError
15
+ from curl_cffi.requests.exceptions import Timeout
13
16
  from curl_cffi.requests.impersonate import DEFAULT_CHROME as CURL_DEFAULT_CHROME
14
17
  from typing_extensions import override
15
18
 
16
- from crawlee._types import HttpHeaders, HttpPayload
19
+ from crawlee._types import HttpHeaders, HttpMethod, HttpPayload
17
20
  from crawlee._utils.blocked import ROTATE_PROXY_ERRORS
18
21
  from crawlee._utils.docs import docs_group
19
22
  from crawlee.errors import ProxyError
@@ -22,11 +25,11 @@ from crawlee.http_clients import HttpClient, HttpCrawlingResult, HttpResponse
22
25
  if TYPE_CHECKING:
23
26
  from collections.abc import AsyncGenerator
24
27
  from datetime import timedelta
25
- from http.cookiejar import Cookie
26
28
 
27
29
  from curl_cffi import Curl
28
30
  from curl_cffi.requests import Request as CurlRequest
29
31
  from curl_cffi.requests import Response
32
+ from curl_cffi.requests.session import HttpMethod as CurlHttpMethod
30
33
 
31
34
  from crawlee import Request
32
35
  from crawlee._types import HttpMethod
@@ -88,15 +91,17 @@ class _CurlImpersonateResponse:
88
91
  async def read(self) -> bytes:
89
92
  if self._response.astream_task:
90
93
  raise RuntimeError('Use `read_stream` to read the body of the Response received from the `stream` method')
94
+
91
95
  return self._response.content
92
96
 
93
97
  async def read_stream(self) -> AsyncGenerator[bytes, None]:
94
- if not self._response.astream_task or self._response.astream_task.done(): # type: ignore[attr-defined]
95
- raise RuntimeError(
96
- 'Cannot read stream: either already consumed or Response not obtained from `stream` method'
97
- )
98
+ if not self._response.astream_task:
99
+ raise RuntimeError('Cannot read stream, Response not obtained from `stream` method.')
98
100
 
99
- async for chunk in self._response.aiter_content(): # type: ignore[no-untyped-call]
101
+ if isinstance(self._response.astream_task, asyncio.Future) and self._response.astream_task.done():
102
+ raise RuntimeError('Cannot read stream, it was already consumed.')
103
+
104
+ async for chunk in self._response.aiter_content():
100
105
  yield chunk
101
106
 
102
107
 
@@ -147,17 +152,21 @@ class CurlImpersonateHttpClient(HttpClient):
147
152
  session: Session | None = None,
148
153
  proxy_info: ProxyInfo | None = None,
149
154
  statistics: Statistics | None = None,
155
+ timeout: timedelta | None = None,
150
156
  ) -> HttpCrawlingResult:
151
157
  client = self._get_client(proxy_info.url if proxy_info else None)
152
158
 
153
159
  try:
154
160
  response = await client.request(
155
161
  url=request.url,
156
- method=request.method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method
162
+ method=self._convert_method(request.method),
157
163
  headers=request.headers,
158
164
  data=request.payload,
159
165
  cookies=session.cookies.jar if session else None,
166
+ timeout=timeout.total_seconds() if timeout else None,
160
167
  )
168
+ except Timeout as exc:
169
+ raise asyncio.TimeoutError from exc
161
170
  except CurlRequestError as exc:
162
171
  if self._is_proxy_error(exc):
163
172
  raise ProxyError from exc
@@ -186,6 +195,7 @@ class CurlImpersonateHttpClient(HttpClient):
186
195
  payload: HttpPayload | None = None,
187
196
  session: Session | None = None,
188
197
  proxy_info: ProxyInfo | None = None,
198
+ timeout: timedelta | None = None,
189
199
  ) -> HttpResponse:
190
200
  if isinstance(headers, dict) or headers is None:
191
201
  headers = HttpHeaders(headers or {})
@@ -196,11 +206,14 @@ class CurlImpersonateHttpClient(HttpClient):
196
206
  try:
197
207
  response = await client.request(
198
208
  url=url,
199
- method=method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method
209
+ method=self._convert_method(method),
200
210
  headers=dict(headers) if headers else None,
201
211
  data=payload,
202
212
  cookies=session.cookies.jar if session else None,
213
+ timeout=timeout.total_seconds() if timeout else None,
203
214
  )
215
+ except Timeout as exc:
216
+ raise asyncio.TimeoutError from exc
204
217
  except CurlRequestError as exc:
205
218
  if self._is_proxy_error(exc):
206
219
  raise ProxyError from exc
@@ -234,13 +247,15 @@ class CurlImpersonateHttpClient(HttpClient):
234
247
  try:
235
248
  response = await client.request(
236
249
  url=url,
237
- method=method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method
250
+ method=self._convert_method(method),
238
251
  headers=dict(headers) if headers else None,
239
252
  data=payload,
240
253
  cookies=session.cookies.jar if session else None,
241
254
  stream=True,
242
255
  timeout=timeout.total_seconds() if timeout else None,
243
256
  )
257
+ except Timeout as exc:
258
+ raise asyncio.TimeoutError from exc
244
259
  except CurlRequestError as exc:
245
260
  if self._is_proxy_error(exc):
246
261
  raise ProxyError from exc
@@ -279,6 +294,40 @@ class CurlImpersonateHttpClient(HttpClient):
279
294
 
280
295
  return self._client_by_proxy_url[proxy_url]
281
296
 
297
+ def _convert_method(self, method: HttpMethod) -> CurlHttpMethod:
298
+ """Convert from Crawlee HTTP method to curl-cffi HTTP method.
299
+
300
+ Args:
301
+ method: Crawlee HTTP method.
302
+
303
+ Returns:
304
+ Corresponding curl-cffi HTTP method.
305
+
306
+ Raises:
307
+ ValueError: If the provided HTTP method is not supported.
308
+ """
309
+ method_upper = method.upper() # curl-cffi requires uppercase methods
310
+
311
+ match method_upper:
312
+ case 'GET':
313
+ return 'GET'
314
+ case 'POST':
315
+ return 'POST'
316
+ case 'PUT':
317
+ return 'PUT'
318
+ case 'DELETE':
319
+ return 'DELETE'
320
+ case 'OPTIONS':
321
+ return 'OPTIONS'
322
+ case 'HEAD':
323
+ return 'HEAD'
324
+ case 'TRACE':
325
+ return 'TRACE'
326
+ case 'PATCH':
327
+ return 'PATCH'
328
+ case _:
329
+ raise ValueError(f'HTTP method {method} is not supported in {self.__class__.__name__}.')
330
+
282
331
  @staticmethod
283
332
  def _is_proxy_error(error: CurlRequestError) -> bool:
284
333
  """Determine whether the given error is related to a proxy issue.
@@ -296,11 +345,16 @@ class CurlImpersonateHttpClient(HttpClient):
296
345
 
297
346
  @staticmethod
298
347
  def _get_cookies(curl: Curl) -> list[Cookie]:
299
- cookies: list[Cookie] = []
300
- for curl_cookie in curl.getinfo(CurlInfo.COOKIELIST): # type: ignore[union-attr]
301
- curl_morsel = CurlMorsel.from_curl_format(curl_cookie) # type: ignore[arg-type]
348
+ cookies = list[Cookie]()
349
+
350
+ # Implementation of getinfo always returns list[bytes] for CurlInfo.COOKIELIST.
351
+ cookie_list = cast('list[bytes]', curl.getinfo(CurlInfo.COOKIELIST))
352
+
353
+ for curl_cookie in cookie_list:
354
+ curl_morsel = CurlMorsel.from_curl_format(curl_cookie)
302
355
  cookie = curl_morsel.to_cookiejar_cookie()
303
356
  cookies.append(cookie)
357
+
304
358
  return cookies
305
359
 
306
360
  async def cleanup(self) -> None:
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import asyncio
3
4
  from contextlib import asynccontextmanager
4
5
  from logging import getLogger
5
6
  from typing import TYPE_CHECKING, Any, cast
@@ -146,6 +147,7 @@ class HttpxHttpClient(HttpClient):
146
147
  session: Session | None = None,
147
148
  proxy_info: ProxyInfo | None = None,
148
149
  statistics: Statistics | None = None,
150
+ timeout: timedelta | None = None,
149
151
  ) -> HttpCrawlingResult:
150
152
  client = self._get_client(proxy_info.url if proxy_info else None)
151
153
  headers = self._combine_headers(request.headers)
@@ -157,10 +159,13 @@ class HttpxHttpClient(HttpClient):
157
159
  content=request.payload,
158
160
  cookies=session.cookies.jar if session else None,
159
161
  extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
162
+ timeout=timeout.total_seconds() if timeout is not None else httpx.USE_CLIENT_DEFAULT,
160
163
  )
161
164
 
162
165
  try:
163
166
  response = await client.send(http_request)
167
+ except httpx.TimeoutException as exc:
168
+ raise asyncio.TimeoutError from exc
164
169
  except httpx.TransportError as exc:
165
170
  if self._is_proxy_error(exc):
166
171
  raise ProxyError from exc
@@ -185,6 +190,7 @@ class HttpxHttpClient(HttpClient):
185
190
  payload: HttpPayload | None = None,
186
191
  session: Session | None = None,
187
192
  proxy_info: ProxyInfo | None = None,
193
+ timeout: timedelta | None = None,
188
194
  ) -> HttpResponse:
189
195
  client = self._get_client(proxy_info.url if proxy_info else None)
190
196
 
@@ -195,10 +201,13 @@ class HttpxHttpClient(HttpClient):
195
201
  headers=headers,
196
202
  payload=payload,
197
203
  session=session,
204
+ timeout=httpx.Timeout(timeout.total_seconds()) if timeout is not None else None,
198
205
  )
199
206
 
200
207
  try:
201
208
  response = await client.send(http_request)
209
+ except httpx.TimeoutException as exc:
210
+ raise asyncio.TimeoutError from exc
202
211
  except httpx.TransportError as exc:
203
212
  if self._is_proxy_error(exc):
204
213
  raise ProxyError from exc
@@ -228,10 +237,13 @@ class HttpxHttpClient(HttpClient):
228
237
  headers=headers,
229
238
  payload=payload,
230
239
  session=session,
231
- timeout=timeout,
240
+ timeout=httpx.Timeout(None, connect=timeout.total_seconds()) if timeout else None,
232
241
  )
233
242
 
234
- response = await client.send(http_request, stream=True)
243
+ try:
244
+ response = await client.send(http_request, stream=True)
245
+ except httpx.TimeoutException as exc:
246
+ raise asyncio.TimeoutError from exc
235
247
 
236
248
  try:
237
249
  yield _HttpxResponse(response)
@@ -246,7 +258,7 @@ class HttpxHttpClient(HttpClient):
246
258
  headers: HttpHeaders | dict[str, str] | None,
247
259
  payload: HttpPayload | None,
248
260
  session: Session | None = None,
249
- timeout: timedelta | None = None,
261
+ timeout: httpx.Timeout | None = None,
250
262
  ) -> httpx.Request:
251
263
  """Build an `httpx.Request` using the provided parameters."""
252
264
  if isinstance(headers, dict) or headers is None:
@@ -254,15 +266,13 @@ class HttpxHttpClient(HttpClient):
254
266
 
255
267
  headers = self._combine_headers(headers)
256
268
 
257
- httpx_timeout = httpx.Timeout(None, connect=timeout.total_seconds()) if timeout else None
258
-
259
269
  return client.build_request(
260
270
  url=url,
261
271
  method=method,
262
272
  headers=dict(headers) if headers else None,
263
273
  content=payload,
264
274
  extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
265
- timeout=httpx_timeout,
275
+ timeout=timeout if timeout else httpx.USE_CLIENT_DEFAULT,
266
276
  )
267
277
 
268
278
  def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
@@ -6,7 +6,7 @@ from logging import getLogger
6
6
  from typing import TYPE_CHECKING, Any, TypedDict
7
7
 
8
8
  from cachetools import LRUCache
9
- from impit import AsyncClient, Browser, HTTPError, Response, TransportError
9
+ from impit import AsyncClient, Browser, HTTPError, Response, TimeoutException, TransportError
10
10
  from impit import ProxyError as ImpitProxyError
11
11
  from typing_extensions import override
12
12
 
@@ -125,6 +125,7 @@ class ImpitHttpClient(HttpClient):
125
125
  session: Session | None = None,
126
126
  proxy_info: ProxyInfo | None = None,
127
127
  statistics: Statistics | None = None,
128
+ timeout: timedelta | None = None,
128
129
  ) -> HttpCrawlingResult:
129
130
  client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None)
130
131
 
@@ -134,7 +135,10 @@ class ImpitHttpClient(HttpClient):
134
135
  method=request.method,
135
136
  content=request.payload,
136
137
  headers=dict(request.headers) if request.headers else None,
138
+ timeout=timeout.total_seconds() if timeout else None,
137
139
  )
140
+ except TimeoutException as exc:
141
+ raise asyncio.TimeoutError from exc
138
142
  except (TransportError, HTTPError) as exc:
139
143
  if self._is_proxy_error(exc):
140
144
  raise ProxyError from exc
@@ -157,6 +161,7 @@ class ImpitHttpClient(HttpClient):
157
161
  payload: HttpPayload | None = None,
158
162
  session: Session | None = None,
159
163
  proxy_info: ProxyInfo | None = None,
164
+ timeout: timedelta | None = None,
160
165
  ) -> HttpResponse:
161
166
  if isinstance(headers, dict) or headers is None:
162
167
  headers = HttpHeaders(headers or {})
@@ -165,8 +170,14 @@ class ImpitHttpClient(HttpClient):
165
170
 
166
171
  try:
167
172
  response = await client.request(
168
- method=method, url=url, content=payload, headers=dict(headers) if headers else None
173
+ method=method,
174
+ url=url,
175
+ content=payload,
176
+ headers=dict(headers) if headers else None,
177
+ timeout=timeout.total_seconds() if timeout else None,
169
178
  )
179
+ except TimeoutException as exc:
180
+ raise asyncio.TimeoutError from exc
170
181
  except (TransportError, HTTPError) as exc:
171
182
  if self._is_proxy_error(exc):
172
183
  raise ProxyError from exc
@@ -189,14 +200,18 @@ class ImpitHttpClient(HttpClient):
189
200
  ) -> AsyncGenerator[HttpResponse]:
190
201
  client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None)
191
202
 
192
- response = await client.request(
193
- method=method,
194
- url=url,
195
- content=payload,
196
- headers=dict(headers) if headers else None,
197
- timeout=timeout.total_seconds() if timeout else None,
198
- stream=True,
199
- )
203
+ try:
204
+ response = await client.request(
205
+ method=method,
206
+ url=url,
207
+ content=payload,
208
+ headers=dict(headers) if headers else None,
209
+ timeout=timeout.total_seconds() if timeout else None,
210
+ stream=True,
211
+ )
212
+ except TimeoutException as exc:
213
+ raise asyncio.TimeoutError from exc
214
+
200
215
  try:
201
216
  yield _ImpitResponse(response)
202
217
  finally:
@@ -3,9 +3,7 @@ from __future__ import annotations
3
3
  import inspect
4
4
  from typing import TYPE_CHECKING, Any
5
5
 
6
- from opentelemetry.instrumentation.instrumentor import ( # type:ignore[attr-defined] # Mypy has troubles with OTEL
7
- BaseInstrumentor,
8
- )
6
+ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
9
7
  from opentelemetry.instrumentation.utils import unwrap
10
8
  from opentelemetry.semconv.attributes.code_attributes import CODE_FUNCTION_NAME
11
9
  from opentelemetry.semconv.attributes.http_attributes import HTTP_REQUEST_METHOD
@@ -69,7 +67,7 @@ class CrawlerInstrumentor(BaseInstrumentor):
69
67
 
70
68
  if request_handling_instrumentation:
71
69
 
72
- async def middlware_wrapper(wrapped: Any, instance: _Middleware, args: Any, kwargs: Any) -> Any:
70
+ async def middleware_wrapper(wrapped: Any, instance: _Middleware, args: Any, kwargs: Any) -> Any:
73
71
  with self._tracer.start_as_current_span(
74
72
  name=f'{instance.generator.__name__}, {wrapped.__name__}', # type:ignore[attr-defined] # valid in our context
75
73
  attributes={
@@ -111,8 +109,8 @@ class CrawlerInstrumentor(BaseInstrumentor):
111
109
  # Handpicked interesting methods to instrument
112
110
  self._instrumented.extend(
113
111
  [
114
- (_Middleware, 'action', middlware_wrapper),
115
- (_Middleware, 'cleanup', middlware_wrapper),
112
+ (_Middleware, 'action', middleware_wrapper),
113
+ (_Middleware, 'cleanup', middleware_wrapper),
116
114
  (ContextPipeline, '__call__', context_pipeline_wrapper),
117
115
  (BasicCrawler, '_BasicCrawler__run_task_function', self._simple_async_wrapper),
118
116
  (BasicCrawler, '_commit_request_handler_result', _commit_request_handler_result_wrapper),
@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Annotated, Any
9
9
  from pydantic import BaseModel, ConfigDict, Field
10
10
  from typing_extensions import override
11
11
 
12
- from crawlee import Request
12
+ from crawlee import Request, RequestOptions
13
13
  from crawlee._utils.docs import docs_group
14
14
  from crawlee._utils.globs import Glob
15
15
  from crawlee._utils.recoverable_state import RecoverableState
@@ -18,9 +18,10 @@ from crawlee.request_loaders._request_loader import RequestLoader
18
18
 
19
19
  if TYPE_CHECKING:
20
20
  import re
21
- from collections.abc import Sequence
21
+ from collections.abc import Callable, Sequence
22
22
  from types import TracebackType
23
23
 
24
+ from crawlee import RequestTransformAction
24
25
  from crawlee.http_clients import HttpClient
25
26
  from crawlee.proxy_configuration import ProxyInfo
26
27
  from crawlee.storage_clients.models import ProcessedRequest
@@ -90,6 +91,11 @@ class SitemapRequestLoaderState(BaseModel):
90
91
  class SitemapRequestLoader(RequestLoader):
91
92
  """A request loader that reads URLs from sitemap(s).
92
93
 
94
+ The loader is designed to handle sitemaps that follow the format described in the Sitemaps protocol
95
+ (https://www.sitemaps.org/protocol.html). It supports both XML and plain text sitemap formats.
96
+ Note that HTML pages containing links are not supported - those should be handled by regular crawlers
97
+ and the `enqueue_links` functionality.
98
+
93
99
  The loader fetches and parses sitemaps in the background, allowing crawling to start
94
100
  before all URLs are loaded. It supports filtering URLs using glob and regex patterns.
95
101
 
@@ -107,6 +113,7 @@ class SitemapRequestLoader(RequestLoader):
107
113
  exclude: list[re.Pattern[Any] | Glob] | None = None,
108
114
  max_buffer_size: int = 200,
109
115
  persist_state_key: str | None = None,
116
+ transform_request_function: Callable[[RequestOptions], RequestOptions | RequestTransformAction] | None = None,
110
117
  ) -> None:
111
118
  """Initialize the sitemap request loader.
112
119
 
@@ -120,6 +127,9 @@ class SitemapRequestLoader(RequestLoader):
120
127
  persist_state_key: A key for persisting the loader's state in the KeyValueStore.
121
128
  When provided, allows resuming from where it left off after interruption.
122
129
  If None, no state persistence occurs.
130
+ transform_request_function: An optional function to transform requests
131
+ generated by the loader. It receives `RequestOptions` with `url` and should return either
132
+ modified `RequestOptions` or a `RequestTransformAction`.
123
133
  """
124
134
  self._http_client = http_client
125
135
  self._sitemap_urls = sitemap_urls
@@ -127,6 +137,7 @@ class SitemapRequestLoader(RequestLoader):
127
137
  self._exclude = exclude
128
138
  self._proxy_info = proxy_info
129
139
  self._max_buffer_size = max_buffer_size
140
+ self._transform_request_function = transform_request_function
130
141
 
131
142
  # Synchronization for queue operations
132
143
  self._queue_has_capacity = asyncio.Event()
@@ -219,7 +230,7 @@ class SitemapRequestLoader(RequestLoader):
219
230
  continue
220
231
  state.in_progress_sitemap_url = sitemap_url
221
232
 
222
- parse_options = ParseSitemapOptions(max_depth=0, emit_nested_sitemaps=True)
233
+ parse_options = ParseSitemapOptions(max_depth=0, emit_nested_sitemaps=True, sitemap_retries=3)
223
234
 
224
235
  async for item in parse_sitemap(
225
236
  [SitemapSource(type='url', url=sitemap_url)],
@@ -308,8 +319,15 @@ class SitemapRequestLoader(RequestLoader):
308
319
 
309
320
  async with self._queue_lock:
310
321
  url = state.url_queue.popleft()
311
-
312
- request = Request.from_url(url)
322
+ request_option = RequestOptions(url=url)
323
+ if self._transform_request_function:
324
+ transform_request_option = self._transform_request_function(request_option)
325
+ if transform_request_option == 'skip':
326
+ state.total_count -= 1
327
+ continue
328
+ if transform_request_option != 'unchanged':
329
+ request_option = transform_request_option
330
+ request = Request.from_url(**request_option)
313
331
  state.in_progress.add(request.url)
314
332
  if len(state.url_queue) < self._max_buffer_size:
315
333
  self._queue_has_capacity.set()
crawlee/router.py CHANGED
@@ -1,13 +1,17 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import asyncio
3
4
  from collections.abc import Awaitable, Callable
4
5
  from typing import Generic, TypeVar
5
6
 
7
+ from crawlee._request import RequestState
6
8
  from crawlee._types import BasicCrawlingContext
7
9
  from crawlee._utils.docs import docs_group
8
10
 
9
11
  __all__ = ['Router']
10
12
 
13
+ from crawlee.errors import UserHandlerTimeoutError
14
+
11
15
  TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext)
12
16
  RequestHandler = Callable[[TCrawlingContext], Awaitable[None]]
13
17
 
@@ -89,13 +93,19 @@ class Router(Generic[TCrawlingContext]):
89
93
 
90
94
  async def __call__(self, context: TCrawlingContext) -> None:
91
95
  """Invoke a request handler that matches the request label (or the default)."""
96
+ context.request.state = RequestState.REQUEST_HANDLER
92
97
  if context.request.label is None or context.request.label not in self._handlers_by_label:
93
98
  if self._default_handler is None:
94
99
  raise RuntimeError(
95
100
  f'No handler matches label `{context.request.label}` and no default handler is configured'
96
101
  )
97
102
 
98
- return await self._default_handler(context)
103
+ user_defined_handler = self._default_handler
104
+ else:
105
+ user_defined_handler = self._handlers_by_label[context.request.label]
99
106
 
100
- handler = self._handlers_by_label[context.request.label]
101
- return await handler(context)
107
+ try:
108
+ return await user_defined_handler(context)
109
+ except asyncio.TimeoutError as e:
110
+ # Timeout in handler, but not timeout of handler.
111
+ raise UserHandlerTimeoutError('Timeout raised by user defined handler') from e
@@ -10,6 +10,7 @@ from crawlee._utils.docs import docs_group
10
10
 
11
11
  if TYPE_CHECKING:
12
12
  from collections.abc import Iterator
13
+ from typing import TypeGuard
13
14
 
14
15
 
15
16
  @docs_group('Session management')
@@ -66,17 +67,18 @@ class SessionCookies:
66
67
 
67
68
  self._jar = CookieJar()
68
69
 
69
- if isinstance(cookies, dict):
70
- for key, value in cookies.items():
71
- self.set(key, value)
72
-
73
- elif isinstance(cookies, list):
70
+ if isinstance(cookies, list):
74
71
  for item in cookies:
75
72
  self.set(**item)
76
73
 
77
74
  elif isinstance(cookies, SessionCookies):
78
75
  for cookie in cookies.jar:
79
- self.jar.set_cookie(cookie)
76
+ self._jar.set_cookie(cookie)
77
+
78
+ elif isinstance(cookies, dict):
79
+ cookies_dict: dict[str, str] = cookies
80
+ for key, value in cookies_dict.items():
81
+ self.set(key, value)
80
82
 
81
83
  @property
82
84
  def jar(self) -> CookieJar:
@@ -151,8 +153,8 @@ class SessionCookies:
151
153
  if cookie.expires:
152
154
  cookie_dict['expires'] = cookie.expires
153
155
 
154
- if (same_site := cookie.get_nonstandard_attr('SameSite')) and same_site in {'Lax', 'None', 'Strict'}:
155
- cookie_dict['same_site'] = same_site # type: ignore[typeddict-item]
156
+ if (same_site := cookie.get_nonstandard_attr('SameSite')) and self._is_valid_same_site(same_site):
157
+ cookie_dict['same_site'] = same_site
156
158
 
157
159
  return cookie_dict
158
160
 
@@ -273,3 +275,6 @@ class SessionCookies:
273
275
  """Return hash based on the cookies key attributes."""
274
276
  cookie_tuples = frozenset((cookie.name, cookie.value, cookie.domain, cookie.path) for cookie in self._jar)
275
277
  return hash(cookie_tuples)
278
+
279
+ def _is_valid_same_site(self, value: str | None) -> TypeGuard[Literal['Lax', 'None', 'Strict']]:
280
+ return value in {'Lax', 'None', 'Strict'}
@@ -63,19 +63,19 @@ class SessionPoolModel(BaseModel):
63
63
  ),
64
64
  ]
65
65
 
66
- @computed_field(alias='sessionCount') # type: ignore[prop-decorator]
66
+ @computed_field(alias='sessionCount')
67
67
  @property
68
68
  def session_count(self) -> int:
69
69
  """Get the total number of sessions currently maintained in the pool."""
70
70
  return len(self.sessions)
71
71
 
72
- @computed_field(alias='usableSessionCount') # type: ignore[prop-decorator]
72
+ @computed_field(alias='usableSessionCount')
73
73
  @property
74
74
  def usable_session_count(self) -> int:
75
75
  """Get the number of sessions that are currently usable."""
76
76
  return len([session for _, session in self.sessions.items() if session.is_usable])
77
77
 
78
- @computed_field(alias='retiredSessionCount') # type: ignore[prop-decorator]
78
+ @computed_field(alias='retiredSessionCount')
79
79
  @property
80
80
  def retired_session_count(self) -> int:
81
81
  """Get the number of sessions that are no longer usable."""
@@ -163,7 +163,7 @@ class SessionPool:
163
163
  def add_session(self, session: Session) -> None:
164
164
  """Add an externally created session to the pool.
165
165
 
166
- This is intened only for the cases when you want to add a session that was created outside of the pool.
166
+ This is intended only for the cases when you want to add a session that was created outside of the pool.
167
167
  Otherwise, the pool will create new sessions automatically.
168
168
 
169
169
  Args:
@@ -32,7 +32,7 @@ class ErrorSnapshotter:
32
32
  """Capture error snapshot and save it to key value store.
33
33
 
34
34
  It saves the error snapshot directly to a key value store. It can't use `context.get_key_value_store` because
35
- it returns `KeyValueStoreChangeRecords` which is commited to the key value store only if the `RequestHandler`
35
+ it returns `KeyValueStoreChangeRecords` which is committed to the key value store only if the `RequestHandler`
36
36
  returned without an exception. ErrorSnapshotter is on the contrary active only when `RequestHandler` fails with
37
37
  an exception.
38
38