scrapy-curl-cffi 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ #
13
+ .python-version
14
+ uv.lock
@@ -0,0 +1,96 @@
1
+ Metadata-Version: 2.4
2
+ Name: scrapy-curl-cffi
3
+ Version: 0.1.0
4
+ Summary: Scrapy integration with curl_cffi (curl-impersonate).
5
+ Project-URL: source, https://github.com/divtiply/scrapy-curl-cffi
6
+ Author-email: Michel Ace <divtiply@gmail.com>
7
+ Requires-Python: >=3.10
8
+ Requires-Dist: curl-cffi>=0.11.1
9
+ Requires-Dist: scrapy>2.10
10
+ Provides-Extra: compression
11
+ Requires-Dist: brotli; extra == 'compression'
12
+ Requires-Dist: zstandard; extra == 'compression'
13
+ Description-Content-Type: text/markdown
14
+
15
+ # scrapy-curl-cffi
16
+
17
+ [Scrapy][1] integration with [curl_cffi][2] ([curl-impersonate][3]).
18
+
19
+ ## Installation
20
+
21
+ ```sh
22
+ pip install scrapy-curl-cffi
23
+ ```
24
+
25
+ Another option, to enable Scrapy's support for modern HTTP compression
26
+ protocols:
27
+
28
+ ```sh
29
+ pip install scrapy-curl-cffi[compression]
30
+ ```
31
+
32
+ ## Configuration
33
+
34
+ Update your Scrapy project settings as follows:
35
+
36
+ ```python
37
+ "DOWNLOAD_HANDLERS": {
38
+ "http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
39
+ "https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
40
+ }
41
+
42
+ "DOWNLOADER_MIDDLEWARES": {
43
+ "scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
44
+ "scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
45
+ "scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
46
+ "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None,
47
+ "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
48
+ }
49
+
50
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
51
+ ```
52
+
53
+ ## Usage
54
+
55
+ To download a [`scrapy.Request`][4] with `curl_cffi`, add the
56
+ `curl_cffi_options` special key to the [`Request.meta`][5] attribute. The value
57
+ should be a dict with any of the following options:
58
+
59
+ - `impersonate` (`str`) - which browser version to impersonate
60
+ - `default_headers` (`bool`) - whether to set default browser headers when
61
+ impersonating (default: `True`)
62
+ - `ja3` (`str`) - ja3 string to impersonate
63
+ - `akamai` (`str`) - akamai string to impersonate
64
+ - `extra_fp` (`str`) - extra fingerprints options, in complement to ja3 and
65
+ akamai strings
66
+ - `verify` (`bool`) - whether to verify https certs (default: `False`)
67
+
68
+ See the [curl_cffi documentation][6] for more info on these options.
69
+
70
+ Alternatively, you can use the `curl_cffi_options` spider attribute or the
71
+ `CURL_CFFI_OPTIONS` setting to automatically assign the `curl_cffi_options` meta
72
+ for all requests.
73
+
74
+ ### Example spider
75
+
76
+ ```python
77
+ class FingerprintsSpider(scrapy.Spider):
78
+ name = "fingerprints"
79
+ start_urls = ["https://tls.browserleaks.com/json"]
80
+ curl_cffi_options = {"impersonate": "chrome"}
81
+
82
+ def parse(self, response):
83
+ yield response.json()
84
+ ```
85
+
86
+ ## Similar projects
87
+
88
+ - [scrapy-impersonate](https://github.com/jxlil/scrapy-impersonate)
89
+ - [scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint)
90
+
91
+ [1]: https://github.com/scrapy/scrapy
92
+ [2]: https://github.com/lexiforest/curl_cffi
93
+ [3]: https://github.com/lexiforest/curl-impersonate
94
+ [4]: https://docs.scrapy.org/en/latest/topics/request-response.html#request-objects
95
+ [5]: https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.Request.meta
96
+ [6]: https://curl-cffi.readthedocs.io
@@ -0,0 +1,82 @@
1
+ # scrapy-curl-cffi
2
+
3
+ [Scrapy][1] integration with [curl_cffi][2] ([curl-impersonate][3]).
4
+
5
+ ## Installation
6
+
7
+ ```sh
8
+ pip install scrapy-curl-cffi
9
+ ```
10
+
11
+ Another option, to enable Scrapy's support for modern HTTP compression
12
+ protocols:
13
+
14
+ ```sh
15
+ pip install scrapy-curl-cffi[compression]
16
+ ```
17
+
18
+ ## Configuration
19
+
20
+ Update your Scrapy project settings as follows:
21
+
22
+ ```python
23
+ "DOWNLOAD_HANDLERS": {
24
+ "http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
25
+ "https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
26
+ }
27
+
28
+ "DOWNLOADER_MIDDLEWARES": {
29
+ "scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
30
+ "scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
31
+ "scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
32
+ "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None,
33
+ "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
34
+ }
35
+
36
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
37
+ ```
38
+
39
+ ## Usage
40
+
41
+ To download a [`scrapy.Request`][4] with `curl_cffi`, add the
42
+ `curl_cffi_options` special key to the [`Request.meta`][5] attribute. The value
43
+ should be a dict with any of the following options:
44
+
45
+ - `impersonate` (`str`) - which browser version to impersonate
46
+ - `default_headers` (`bool`) - whether to set default browser headers when
47
+ impersonating (default: `True`)
48
+ - `ja3` (`str`) - ja3 string to impersonate
49
+ - `akamai` (`str`) - akamai string to impersonate
50
+ - `extra_fp` (`str`) - extra fingerprints options, in complement to ja3 and
51
+ akamai strings
52
+ - `verify` (`bool`) - whether to verify https certs (default: `False`)
53
+
54
+ See the [curl_cffi documentation][6] for more info on these options.
55
+
56
+ Alternatively, you can use the `curl_cffi_options` spider attribute or the
57
+ `CURL_CFFI_OPTIONS` setting to automatically assign the `curl_cffi_options` meta
58
+ for all requests.
59
+
60
+ ### Example spider
61
+
62
+ ```python
63
+ class FingerprintsSpider(scrapy.Spider):
64
+ name = "fingerprints"
65
+ start_urls = ["https://tls.browserleaks.com/json"]
66
+ curl_cffi_options = {"impersonate": "chrome"}
67
+
68
+ def parse(self, response):
69
+ yield response.json()
70
+ ```
71
+
72
+ ## Similar projects
73
+
74
+ - [scrapy-impersonate](https://github.com/jxlil/scrapy-impersonate)
75
+ - [scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint)
76
+
77
+ [1]: https://github.com/scrapy/scrapy
78
+ [2]: https://github.com/lexiforest/curl_cffi
79
+ [3]: https://github.com/lexiforest/curl-impersonate
80
+ [4]: https://docs.scrapy.org/en/latest/topics/request-response.html#request-objects
81
+ [5]: https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.Request.meta
82
+ [6]: https://curl-cffi.readthedocs.io
@@ -0,0 +1,28 @@
1
+ # usage: scrapy runspider examples/basic_options_spider.py
2
+
3
+
4
+ from scrapy import Spider
5
+
6
+
7
+ class BasicOptionsSpider(Spider):
8
+ name = "basic-options"
9
+ start_urls = ["https://browserleaks.com/ip"]
10
+ curl_cffi_options = {"impersonate": "chrome"}
11
+
12
+ custom_settings = {
13
+ "DOWNLOAD_HANDLERS": {
14
+ "http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
15
+ "https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
16
+ },
17
+ "DOWNLOADER_MIDDLEWARES": {
18
+ "scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
19
+ "scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
20
+ "scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
21
+ "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None, # noqa: E501
22
+ "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
23
+ },
24
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
25
+ }
26
+
27
+ def parse(self, response):
28
+ yield response.json()
@@ -0,0 +1,49 @@
1
+ # usage: scrapy runspider examples/custom_headers_spider.py
2
+
3
+
4
+ from scrapy import Spider
5
+
6
+
7
+ class CustomHeadersSpider(Spider):
8
+ name = "custom-headers"
9
+ start_urls = ["https://browserleaks.com/ip"]
10
+ curl_cffi_options = {
11
+ "impersonate": "chrome136",
12
+ "default_headers": False, # disable curl-side default headers
13
+ }
14
+ default_request_headers = { # enable scrapy-side default headers
15
+ "Sec-Ch-Ua": '"Chromium";v="136", "Microsoft Edge";v="136", "Not.A/Brand";v="99"', # noqa: E501
16
+ "Sec-Ch-Ua-Mobile": "?0",
17
+ "Sec-Ch-Ua-Platform": '"Windows"',
18
+ "Upgrade-Insecure-Requests": "1",
19
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0", # noqa: E501
20
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", # noqa: E501
21
+ "Sec-Fetch-Site": "none",
22
+ "Sec-Fetch-Mode": "navigate",
23
+ "Sec-Fetch-User": "?1",
24
+ "Sec-Fetch-Dest": "document",
25
+ "Accept-Language": "en-US,en;q=0.5",
26
+ "Accept-Encoding": "gzip, deflate, br, zstd",
27
+ "Priority": "u=0, i",
28
+ }
29
+
30
+ custom_settings = {
31
+ "DOWNLOAD_HANDLERS": {
32
+ "http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
33
+ "https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
34
+ },
35
+ "DOWNLOADER_MIDDLEWARES": {
36
+ "scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
37
+ "scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
38
+ "scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
39
+ "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None, # noqa: E501
40
+ "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
41
+ },
42
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
43
+ }
44
+
45
+ def parse(self, response):
46
+ def extract_table(selector):
47
+ return [sel.css("td::text").getall() for sel in selector.css("tr")]
48
+
49
+ yield {"headers": extract_table(response.css("#headers"))}
@@ -0,0 +1,46 @@
1
+ [project]
2
+ name = "scrapy-curl-cffi"
3
+ version = "0.1.0"
4
+ description = "Scrapy integration with curl_cffi (curl-impersonate)."
5
+ readme = "README.md"
6
+ authors = [
7
+ { name = "Michel Ace", email = "divtiply@gmail.com" }
8
+ ]
9
+ requires-python = ">=3.10"
10
+ dependencies = [
11
+ "curl-cffi>=0.11.1",
12
+ "scrapy>2.10",
13
+ ]
14
+
15
+ [project.urls]
16
+ source = "https://github.com/divtiply/scrapy-curl-cffi"
17
+
18
+ [project.optional-dependencies]
19
+ compression = [
20
+ "brotli",
21
+ "zstandard",
22
+ ]
23
+
24
+ [build-system]
25
+ requires = ["hatchling"]
26
+ build-backend = "hatchling.build"
27
+
28
+ [tool.ruff.lint]
29
+ select = ["ALL"]
30
+ ignore = [
31
+ "COM",
32
+ "D1",
33
+ "FIX",
34
+ "PGH003",
35
+ "PLC0414",
36
+ "TD001",
37
+ "TD002",
38
+ "TD003",
39
+ ]
40
+
41
+ [tool.ruff.lint.per-file-ignores]
42
+ "examples/**.py" = [
43
+ "ANN",
44
+ "INP",
45
+ "RUF012",
46
+ ]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,83 @@
1
+ from ipaddress import ip_address
2
+ from typing import Any
3
+
4
+ import curl_cffi
5
+ import scrapy.http
6
+ from scrapy.responsetypes import responsetypes
7
+
8
+ from .utils import parse_basic_auth_header
9
+
10
+
11
+ def to_curl_cffi_request_kwargs(scrapy_request: scrapy.http.Request) -> dict[str, Any]:
12
+ curl_cffi_headers = to_curl_cffi_headers(scrapy_request.headers)
13
+ proxy_auth_header = curl_cffi_headers.pop("Proxy-Authorization", None)
14
+
15
+ request_kwargs = {
16
+ "method": scrapy_request.method,
17
+ "url": scrapy_request.url,
18
+ "data": scrapy_request.body,
19
+ "headers": curl_cffi_headers,
20
+ "allow_redirects": False, # disable curl-side redirection
21
+ "accept_encoding": None, # disable curl-side decompression
22
+ }
23
+
24
+ proxy = scrapy_request.meta.get("proxy")
25
+ if proxy is not None:
26
+ request_kwargs["proxy"] = proxy
27
+ if proxy_auth_header is not None:
28
+ request_kwargs["proxy_auth"] = parse_basic_auth_header(proxy_auth_header)
29
+
30
+ timeout = scrapy_request.meta.get("download_timeout")
31
+ if timeout is not None:
32
+ request_kwargs["timeout"] = timeout
33
+
34
+ options = scrapy_request.meta.get("curl_cffi_options", {})
35
+ supported_keys = {
36
+ "impersonate",
37
+ "default_headers",
38
+ "ja3",
39
+ "akamai",
40
+ "extra_fp",
41
+ "verify",
42
+ }
43
+ unsupported_keys = options.keys() - supported_keys
44
+ if unsupported_keys:
45
+ msg = f"Unsupported curl_cffi_options keys: {', '.join(unsupported_keys)}"
46
+ raise ValueError(msg)
47
+ request_kwargs.update(
48
+ {key: options[key] for key in supported_keys & options.keys()}
49
+ )
50
+
51
+ return request_kwargs
52
+
53
+
54
+ def to_curl_cffi_headers(scrapy_headers: scrapy.http.Headers) -> curl_cffi.Headers:
55
+ return curl_cffi.Headers([(k, v) for k, vs in scrapy_headers.items() for v in vs])
56
+
57
+
58
+ def to_scrapy_response(
59
+ curl_cffi_response: curl_cffi.Response, scrapy_request: scrapy.http.Request
60
+ ) -> scrapy.http.Response:
61
+ scrapy_headers = to_scrapy_headers(curl_cffi_response.headers)
62
+ response_class = responsetypes.from_args(
63
+ headers=scrapy_headers,
64
+ url=curl_cffi_response.url,
65
+ body=curl_cffi_response.content,
66
+ )
67
+ response = response_class(
68
+ url=curl_cffi_response.url,
69
+ status=curl_cffi_response.status_code,
70
+ headers=scrapy_headers,
71
+ body=curl_cffi_response.content,
72
+ flags=["curl_cffi"],
73
+ request=scrapy_request,
74
+ # certificate=
75
+ ip_address=ip_address(curl_cffi_response.primary_ip),
76
+ # protocol=
77
+ )
78
+ response.meta["download_latency"] = curl_cffi_response.elapsed
79
+ return response
80
+
81
+
82
+ def to_scrapy_headers(curl_cffi_headers: curl_cffi.Headers) -> scrapy.http.Headers:
83
+ return scrapy.http.Headers(curl_cffi_headers.multi_items())
@@ -0,0 +1,69 @@
1
+ import logging
2
+ from http.cookiejar import Cookie, CookieJar
3
+
4
+ import curl_cffi
5
+ from scrapy import Spider
6
+ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
7
+ from scrapy.crawler import Crawler
8
+ from scrapy.http import Request, Response
9
+ from scrapy.settings import BaseSettings
10
+ from scrapy.utils.defer import deferred_from_coro
11
+ from scrapy.utils.reactor import is_asyncio_reactor_installed
12
+ from twisted.internet.defer import Deferred
13
+
14
+ from .adapters import to_curl_cffi_request_kwargs, to_scrapy_response
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ class CurlCffiDownloadHandler(HTTPDownloadHandler):
20
+ def __init__(self, settings: BaseSettings, crawler: Crawler) -> None:
21
+ super().__init__(settings, crawler)
22
+ if not is_asyncio_reactor_installed():
23
+ msg = (
24
+ f"{self.__class__.__qualname__} requires the asyncio Twisted "
25
+ f"reactor. Make sure you have it configured in the "
26
+ f"TWISTED_REACTOR setting. See the asyncio documentation "
27
+ f"of Scrapy for more information."
28
+ )
29
+ raise ValueError(msg)
30
+ self._session: curl_cffi.AsyncSession | None = None
31
+
32
+ def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
33
+ if "curl_cffi_options" in request.meta:
34
+ return deferred_from_coro(self._download_request(request, spider))
35
+ return super().download_request(request, spider)
36
+
37
+ async def _download_request(self, request: Request, spider: Spider) -> Response: # noqa: ARG002
38
+ curl_cffi_request_kwargs = to_curl_cffi_request_kwargs(request)
39
+ if self._session:
40
+ curl_cffi_response = await self._session.request(**curl_cffi_request_kwargs)
41
+ else:
42
+ async with self._create_session() as session:
43
+ curl_cffi_response = await session.request(**curl_cffi_request_kwargs)
44
+ return to_scrapy_response(curl_cffi_response, request)
45
+
46
+ def _create_session(
47
+ self,
48
+ max_clients: int = 1,
49
+ verify: bool = False, # noqa: FBT001, FBT002
50
+ ) -> curl_cffi.AsyncSession:
51
+ return curl_cffi.AsyncSession(
52
+ max_clients=max_clients,
53
+ cookies=_DummyCookieJar(),
54
+ trust_env=False,
55
+ allow_redirects=False,
56
+ verify=verify,
57
+ )
58
+
59
+ def close(self) -> Deferred[None]:
60
+ d = super().close()
61
+ if self._session:
62
+ close_session_dfd = deferred_from_coro(self._session.close())
63
+ d.addBoth(lambda _: close_session_dfd)
64
+ return d
65
+
66
+
67
+ class _DummyCookieJar(CookieJar):
68
+ def set_cookie(self, cookie: Cookie) -> None:
69
+ pass
@@ -0,0 +1,68 @@
1
+ from collections.abc import Mapping
2
+
3
+ from scrapy import Request, Spider, signals
4
+ from scrapy.crawler import Crawler
5
+ from scrapy.downloadermiddlewares.defaultheaders import (
6
+ DefaultHeadersMiddleware as _DefaultHeadersMiddleware,
7
+ )
8
+ from scrapy.downloadermiddlewares.useragent import (
9
+ UserAgentMiddleware as _UserAgentMiddleware,
10
+ )
11
+ from scrapy.http import Response
12
+ from typing_extensions import Self
13
+
14
+
15
+ class CurlCffiMiddleware:
16
+ def __init__(self, options: Mapping | None) -> None:
17
+ self._options = options
18
+
19
+ @classmethod
20
+ def from_crawler(cls, crawler: Crawler) -> Self:
21
+ settings = crawler.settings
22
+ if "CURL_CFFI_OPTIONS" in settings:
23
+ options = settings.getdict("CURL_CFFI_OPTIONS")
24
+ else:
25
+ options = None
26
+ mw = cls(options)
27
+ crawler.signals.connect(mw.spider_opened, signal=signals.spider_opened)
28
+ return mw
29
+
30
+ def spider_opened(self, spider: Spider) -> None:
31
+ self._options = getattr(spider, "curl_cffi_options", self._options)
32
+
33
+ def process_request(
34
+ self,
35
+ request: Request,
36
+ spider: Spider, # noqa: ARG002
37
+ ) -> Request | Response | None:
38
+ if self._options is not None:
39
+ request.meta.setdefault("curl_cffi_options", self._options)
40
+ return None
41
+
42
+
43
+ class _CurlCffiMixin:
44
+ def process_request(
45
+ self, request: Request, spider: Spider
46
+ ) -> Request | Response | None:
47
+ if (
48
+ "curl_cffi_options" in request.meta # _
49
+ and request.meta["curl_cffi_options"].get("default_headers", True)
50
+ ):
51
+ return None
52
+ return super().process_request(request, spider) # type: ignore
53
+
54
+
55
+ class DefaultHeadersMiddleware(_CurlCffiMixin, _DefaultHeadersMiddleware):
56
+ @classmethod
57
+ def from_crawler(cls, crawler: Crawler) -> Self:
58
+ mw = super().from_crawler(crawler)
59
+ crawler.signals.connect(mw.spider_opened, signal=signals.spider_opened)
60
+ return mw
61
+
62
+ def spider_opened(self, spider: Spider) -> None:
63
+ headers = getattr(spider, "default_request_headers", None) or {}
64
+ self._headers = headers.items() if isinstance(headers, Mapping) else headers
65
+
66
+
67
+ class UserAgentMiddleware(_CurlCffiMixin, _UserAgentMiddleware):
68
+ pass
@@ -0,0 +1,13 @@
1
+ from base64 import b64decode
2
+ from urllib.parse import unquote
3
+
4
+ from scrapy.utils.python import to_bytes
5
+
6
+
7
+ def parse_basic_auth_header(header_value: str | bytes) -> tuple[str, str]:
8
+ parts = to_bytes(header_value).split(b" ")
9
+ if parts[0] != b"Basic" or len(parts) != 2: # noqa: PLR2004
10
+ msg = "Invalid Basic Auth header"
11
+ raise ValueError(msg)
12
+ tokens = b64decode(parts[1]).split(b":", 1)
13
+ return unquote(tokens[0]), unquote(tokens[1])