scrapy-curl-cffi 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapy_curl_cffi-0.1.0/.gitignore +14 -0
- scrapy_curl_cffi-0.1.0/PKG-INFO +96 -0
- scrapy_curl_cffi-0.1.0/README.md +82 -0
- scrapy_curl_cffi-0.1.0/examples/basic_options_spider.py +28 -0
- scrapy_curl_cffi-0.1.0/examples/custom_headers_spider.py +49 -0
- scrapy_curl_cffi-0.1.0/pyproject.toml +46 -0
- scrapy_curl_cffi-0.1.0/src/scrapy_curl_cffi/__init__.py +1 -0
- scrapy_curl_cffi-0.1.0/src/scrapy_curl_cffi/adapters.py +83 -0
- scrapy_curl_cffi-0.1.0/src/scrapy_curl_cffi/handler.py +69 -0
- scrapy_curl_cffi-0.1.0/src/scrapy_curl_cffi/middlewares.py +68 -0
- scrapy_curl_cffi-0.1.0/src/scrapy_curl_cffi/utils.py +13 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scrapy-curl-cffi
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scrapy integration with curl_cffi (curl-impersonate).
|
|
5
|
+
Project-URL: source, https://github.com/divtiply/scrapy-curl-cffi
|
|
6
|
+
Author-email: Michel Ace <divtiply@gmail.com>
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Requires-Dist: curl-cffi>=0.11.1
|
|
9
|
+
Requires-Dist: scrapy>2.10
|
|
10
|
+
Provides-Extra: compression
|
|
11
|
+
Requires-Dist: brotli; extra == 'compression'
|
|
12
|
+
Requires-Dist: zstandard; extra == 'compression'
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# scrapy-curl-cffi
|
|
16
|
+
|
|
17
|
+
[Scrapy][1] integration with [curl_cffi][2] ([curl-impersonate][3]).
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
pip install scrapy-curl-cffi
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Another option, to enable Scrapy's support for modern HTTP compression
|
|
26
|
+
protocols:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
pip install scrapy-curl-cffi[compression]
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Configuration
|
|
33
|
+
|
|
34
|
+
Update your Scrapy project settings as follows:
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
"DOWNLOAD_HANDLERS": {
|
|
38
|
+
"http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
39
|
+
"https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
"DOWNLOADER_MIDDLEWARES": {
|
|
43
|
+
"scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
|
|
44
|
+
"scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
|
|
45
|
+
"scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
|
|
46
|
+
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None,
|
|
47
|
+
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Usage
|
|
54
|
+
|
|
55
|
+
To download a [`scrapy.Request`][4] with `curl_cffi`, add the
|
|
56
|
+
`curl_cffi_options` special key to the [`Request.meta`][5] attribute. The value
|
|
57
|
+
should be a dict with any of the following options:
|
|
58
|
+
|
|
59
|
+
- `impersonate` (`str`) - which browser version to impersonate
|
|
60
|
+
- `default_headers` (`bool`) - whether to set default browser headers when
|
|
61
|
+
impersonating (default: `True`)
|
|
62
|
+
- `ja3` (`str`) - ja3 string to impersonate
|
|
63
|
+
- `akamai` (`str`) - akamai string to impersonate
|
|
64
|
+
- `extra_fp` (`str`) - extra fingerprints options, in complement to ja3 and
|
|
65
|
+
akamai strings
|
|
66
|
+
- `verify` (`bool`) - whether to verify https certs (default: `False`)
|
|
67
|
+
|
|
68
|
+
See the [curl_cffi documentation][6] for more info on these options.
|
|
69
|
+
|
|
70
|
+
Alternatively, you can use the `curl_cffi_options` spider attribute or the
|
|
71
|
+
`CURL_CFFI_OPTIONS` setting to automatically assign the `curl_cffi_options` meta
|
|
72
|
+
for all requests.
|
|
73
|
+
|
|
74
|
+
### Example spider
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
class FingerprintsSpider(scrapy.Spider):
|
|
78
|
+
name = "fingerprints"
|
|
79
|
+
start_urls = ["https://tls.browserleaks.com/json"]
|
|
80
|
+
curl_cffi_options = {"impersonate": "chrome"}
|
|
81
|
+
|
|
82
|
+
def parse(self, response):
|
|
83
|
+
yield response.json()
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Similar projects
|
|
87
|
+
|
|
88
|
+
- [scrapy-impersonate](https://github.com/jxlil/scrapy-impersonate)
|
|
89
|
+
- [scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint)
|
|
90
|
+
|
|
91
|
+
[1]: https://github.com/scrapy/scrapy
|
|
92
|
+
[2]: https://github.com/lexiforest/curl_cffi
|
|
93
|
+
[3]: https://github.com/lexiforest/curl-impersonate
|
|
94
|
+
[4]: https://docs.scrapy.org/en/latest/topics/request-response.html#request-objects
|
|
95
|
+
[5]: https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.Request.meta
|
|
96
|
+
[6]: https://curl-cffi.readthedocs.io
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# scrapy-curl-cffi
|
|
2
|
+
|
|
3
|
+
[Scrapy][1] integration with [curl_cffi][2] ([curl-impersonate][3]).
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
pip install scrapy-curl-cffi
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Another option, to enable Scrapy's support for modern HTTP compression
|
|
12
|
+
protocols:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
pip install scrapy-curl-cffi[compression]
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Configuration
|
|
19
|
+
|
|
20
|
+
Update your Scrapy project settings as follows:
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
"DOWNLOAD_HANDLERS": {
|
|
24
|
+
"http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
25
|
+
"https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
"DOWNLOADER_MIDDLEWARES": {
|
|
29
|
+
"scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
|
|
30
|
+
"scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
|
|
31
|
+
"scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
|
|
32
|
+
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None,
|
|
33
|
+
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Usage
|
|
40
|
+
|
|
41
|
+
To download a [`scrapy.Request`][4] with `curl_cffi`, add the
|
|
42
|
+
`curl_cffi_options` special key to the [`Request.meta`][5] attribute. The value
|
|
43
|
+
should be a dict with any of the following options:
|
|
44
|
+
|
|
45
|
+
- `impersonate` (`str`) - which browser version to impersonate
|
|
46
|
+
- `default_headers` (`bool`) - whether to set default browser headers when
|
|
47
|
+
impersonating (default: `True`)
|
|
48
|
+
- `ja3` (`str`) - ja3 string to impersonate
|
|
49
|
+
- `akamai` (`str`) - akamai string to impersonate
|
|
50
|
+
- `extra_fp` (`str`) - extra fingerprints options, in complement to ja3 and
|
|
51
|
+
akamai strings
|
|
52
|
+
- `verify` (`bool`) - whether to verify https certs (default: `False`)
|
|
53
|
+
|
|
54
|
+
See the [curl_cffi documentation][6] for more info on these options.
|
|
55
|
+
|
|
56
|
+
Alternatively, you can use the `curl_cffi_options` spider attribute or the
|
|
57
|
+
`CURL_CFFI_OPTIONS` setting to automatically assign the `curl_cffi_options` meta
|
|
58
|
+
for all requests.
|
|
59
|
+
|
|
60
|
+
### Example spider
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
class FingerprintsSpider(scrapy.Spider):
|
|
64
|
+
name = "fingerprints"
|
|
65
|
+
start_urls = ["https://tls.browserleaks.com/json"]
|
|
66
|
+
curl_cffi_options = {"impersonate": "chrome"}
|
|
67
|
+
|
|
68
|
+
def parse(self, response):
|
|
69
|
+
yield response.json()
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Similar projects
|
|
73
|
+
|
|
74
|
+
- [scrapy-impersonate](https://github.com/jxlil/scrapy-impersonate)
|
|
75
|
+
- [scrapy-fingerprint](https://github.com/tieyongjie/scrapy-fingerprint)
|
|
76
|
+
|
|
77
|
+
[1]: https://github.com/scrapy/scrapy
|
|
78
|
+
[2]: https://github.com/lexiforest/curl_cffi
|
|
79
|
+
[3]: https://github.com/lexiforest/curl-impersonate
|
|
80
|
+
[4]: https://docs.scrapy.org/en/latest/topics/request-response.html#request-objects
|
|
81
|
+
[5]: https://docs.scrapy.org/en/latest/topics/request-response.html#scrapy.Request.meta
|
|
82
|
+
[6]: https://curl-cffi.readthedocs.io
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# usage: scrapy runspider examples/basic_options_spider.py
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
from scrapy import Spider
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class BasicOptionsSpider(Spider):
|
|
8
|
+
name = "basic-options"
|
|
9
|
+
start_urls = ["https://browserleaks.com/ip"]
|
|
10
|
+
curl_cffi_options = {"impersonate": "chrome"}
|
|
11
|
+
|
|
12
|
+
custom_settings = {
|
|
13
|
+
"DOWNLOAD_HANDLERS": {
|
|
14
|
+
"http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
15
|
+
"https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
16
|
+
},
|
|
17
|
+
"DOWNLOADER_MIDDLEWARES": {
|
|
18
|
+
"scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
|
|
19
|
+
"scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
|
|
20
|
+
"scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
|
|
21
|
+
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None, # noqa: E501
|
|
22
|
+
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
|
|
23
|
+
},
|
|
24
|
+
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
def parse(self, response):
|
|
28
|
+
yield response.json()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# usage: scrapy runspider examples/custom_headers_spider.py
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
from scrapy import Spider
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CustomHeadersSpider(Spider):
|
|
8
|
+
name = "custom-headers"
|
|
9
|
+
start_urls = ["https://browserleaks.com/ip"]
|
|
10
|
+
curl_cffi_options = {
|
|
11
|
+
"impersonate": "chrome136",
|
|
12
|
+
"default_headers": False, # disable curl-side default headers
|
|
13
|
+
}
|
|
14
|
+
default_request_headers = { # enable scrapy-side default headers
|
|
15
|
+
"Sec-Ch-Ua": '"Chromium";v="136", "Microsoft Edge";v="136", "Not.A/Brand";v="99"', # noqa: E501
|
|
16
|
+
"Sec-Ch-Ua-Mobile": "?0",
|
|
17
|
+
"Sec-Ch-Ua-Platform": '"Windows"',
|
|
18
|
+
"Upgrade-Insecure-Requests": "1",
|
|
19
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36 Edg/136.0.0.0", # noqa: E501
|
|
20
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", # noqa: E501
|
|
21
|
+
"Sec-Fetch-Site": "none",
|
|
22
|
+
"Sec-Fetch-Mode": "navigate",
|
|
23
|
+
"Sec-Fetch-User": "?1",
|
|
24
|
+
"Sec-Fetch-Dest": "document",
|
|
25
|
+
"Accept-Language": "en-US,en;q=0.5",
|
|
26
|
+
"Accept-Encoding": "gzip, deflate, br, zstd",
|
|
27
|
+
"Priority": "u=0, i",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
custom_settings = {
|
|
31
|
+
"DOWNLOAD_HANDLERS": {
|
|
32
|
+
"http": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
33
|
+
"https": "scrapy_curl_cffi.handler.CurlCffiDownloadHandler",
|
|
34
|
+
},
|
|
35
|
+
"DOWNLOADER_MIDDLEWARES": {
|
|
36
|
+
"scrapy_curl_cffi.middlewares.CurlCffiMiddleware": 200,
|
|
37
|
+
"scrapy_curl_cffi.middlewares.DefaultHeadersMiddleware": 400,
|
|
38
|
+
"scrapy_curl_cffi.middlewares.UserAgentMiddleware": 500,
|
|
39
|
+
"scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": None, # noqa: E501
|
|
40
|
+
"scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
|
|
41
|
+
},
|
|
42
|
+
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
def parse(self, response):
|
|
46
|
+
def extract_table(selector):
|
|
47
|
+
return [sel.css("td::text").getall() for sel in selector.css("tr")]
|
|
48
|
+
|
|
49
|
+
yield {"headers": extract_table(response.css("#headers"))}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "scrapy-curl-cffi"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Scrapy integration with curl_cffi (curl-impersonate)."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Michel Ace", email = "divtiply@gmail.com" }
|
|
8
|
+
]
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"curl-cffi>=0.11.1",
|
|
12
|
+
"scrapy>2.10",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.urls]
|
|
16
|
+
source = "https://github.com/divtiply/scrapy-curl-cffi"
|
|
17
|
+
|
|
18
|
+
[project.optional-dependencies]
|
|
19
|
+
compression = [
|
|
20
|
+
"brotli",
|
|
21
|
+
"zstandard",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[build-system]
|
|
25
|
+
requires = ["hatchling"]
|
|
26
|
+
build-backend = "hatchling.build"
|
|
27
|
+
|
|
28
|
+
[tool.ruff.lint]
|
|
29
|
+
select = ["ALL"]
|
|
30
|
+
ignore = [
|
|
31
|
+
"COM",
|
|
32
|
+
"D1",
|
|
33
|
+
"FIX",
|
|
34
|
+
"PGH003",
|
|
35
|
+
"PLC0414",
|
|
36
|
+
"TD001",
|
|
37
|
+
"TD002",
|
|
38
|
+
"TD003",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[tool.ruff.lint.per-file-ignores]
|
|
42
|
+
"examples/**.py" = [
|
|
43
|
+
"ANN",
|
|
44
|
+
"INP",
|
|
45
|
+
"RUF012",
|
|
46
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
from ipaddress import ip_address
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
import curl_cffi
|
|
5
|
+
import scrapy.http
|
|
6
|
+
from scrapy.responsetypes import responsetypes
|
|
7
|
+
|
|
8
|
+
from .utils import parse_basic_auth_header
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def to_curl_cffi_request_kwargs(scrapy_request: scrapy.http.Request) -> dict[str, Any]:
|
|
12
|
+
curl_cffi_headers = to_curl_cffi_headers(scrapy_request.headers)
|
|
13
|
+
proxy_auth_header = curl_cffi_headers.pop("Proxy-Authorization", None)
|
|
14
|
+
|
|
15
|
+
request_kwargs = {
|
|
16
|
+
"method": scrapy_request.method,
|
|
17
|
+
"url": scrapy_request.url,
|
|
18
|
+
"data": scrapy_request.body,
|
|
19
|
+
"headers": curl_cffi_headers,
|
|
20
|
+
"allow_redirects": False, # disable curl-side redirection
|
|
21
|
+
"accept_encoding": None, # disable curl-side decompression
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
proxy = scrapy_request.meta.get("proxy")
|
|
25
|
+
if proxy is not None:
|
|
26
|
+
request_kwargs["proxy"] = proxy
|
|
27
|
+
if proxy_auth_header is not None:
|
|
28
|
+
request_kwargs["proxy_auth"] = parse_basic_auth_header(proxy_auth_header)
|
|
29
|
+
|
|
30
|
+
timeout = scrapy_request.meta.get("download_timeout")
|
|
31
|
+
if timeout is not None:
|
|
32
|
+
request_kwargs["timeout"] = timeout
|
|
33
|
+
|
|
34
|
+
options = scrapy_request.meta.get("curl_cffi_options", {})
|
|
35
|
+
supported_keys = {
|
|
36
|
+
"impersonate",
|
|
37
|
+
"default_headers",
|
|
38
|
+
"ja3",
|
|
39
|
+
"akamai",
|
|
40
|
+
"extra_fp",
|
|
41
|
+
"verify",
|
|
42
|
+
}
|
|
43
|
+
unsupported_keys = options.keys() - supported_keys
|
|
44
|
+
if unsupported_keys:
|
|
45
|
+
msg = f"Unsupported curl_cffi_options keys: {', '.join(unsupported_keys)}"
|
|
46
|
+
raise ValueError(msg)
|
|
47
|
+
request_kwargs.update(
|
|
48
|
+
{key: options[key] for key in supported_keys & options.keys()}
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
return request_kwargs
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def to_curl_cffi_headers(scrapy_headers: scrapy.http.Headers) -> curl_cffi.Headers:
|
|
55
|
+
return curl_cffi.Headers([(k, v) for k, vs in scrapy_headers.items() for v in vs])
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def to_scrapy_response(
|
|
59
|
+
curl_cffi_response: curl_cffi.Response, scrapy_request: scrapy.http.Request
|
|
60
|
+
) -> scrapy.http.Response:
|
|
61
|
+
scrapy_headers = to_scrapy_headers(curl_cffi_response.headers)
|
|
62
|
+
response_class = responsetypes.from_args(
|
|
63
|
+
headers=scrapy_headers,
|
|
64
|
+
url=curl_cffi_response.url,
|
|
65
|
+
body=curl_cffi_response.content,
|
|
66
|
+
)
|
|
67
|
+
response = response_class(
|
|
68
|
+
url=curl_cffi_response.url,
|
|
69
|
+
status=curl_cffi_response.status_code,
|
|
70
|
+
headers=scrapy_headers,
|
|
71
|
+
body=curl_cffi_response.content,
|
|
72
|
+
flags=["curl_cffi"],
|
|
73
|
+
request=scrapy_request,
|
|
74
|
+
# certificate=
|
|
75
|
+
ip_address=ip_address(curl_cffi_response.primary_ip),
|
|
76
|
+
# protocol=
|
|
77
|
+
)
|
|
78
|
+
response.meta["download_latency"] = curl_cffi_response.elapsed
|
|
79
|
+
return response
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def to_scrapy_headers(curl_cffi_headers: curl_cffi.Headers) -> scrapy.http.Headers:
|
|
83
|
+
return scrapy.http.Headers(curl_cffi_headers.multi_items())
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from http.cookiejar import Cookie, CookieJar
|
|
3
|
+
|
|
4
|
+
import curl_cffi
|
|
5
|
+
from scrapy import Spider
|
|
6
|
+
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
|
7
|
+
from scrapy.crawler import Crawler
|
|
8
|
+
from scrapy.http import Request, Response
|
|
9
|
+
from scrapy.settings import BaseSettings
|
|
10
|
+
from scrapy.utils.defer import deferred_from_coro
|
|
11
|
+
from scrapy.utils.reactor import is_asyncio_reactor_installed
|
|
12
|
+
from twisted.internet.defer import Deferred
|
|
13
|
+
|
|
14
|
+
from .adapters import to_curl_cffi_request_kwargs, to_scrapy_response
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class CurlCffiDownloadHandler(HTTPDownloadHandler):
|
|
20
|
+
def __init__(self, settings: BaseSettings, crawler: Crawler) -> None:
|
|
21
|
+
super().__init__(settings, crawler)
|
|
22
|
+
if not is_asyncio_reactor_installed():
|
|
23
|
+
msg = (
|
|
24
|
+
f"{self.__class__.__qualname__} requires the asyncio Twisted "
|
|
25
|
+
f"reactor. Make sure you have it configured in the "
|
|
26
|
+
f"TWISTED_REACTOR setting. See the asyncio documentation "
|
|
27
|
+
f"of Scrapy for more information."
|
|
28
|
+
)
|
|
29
|
+
raise ValueError(msg)
|
|
30
|
+
self._session: curl_cffi.AsyncSession | None = None
|
|
31
|
+
|
|
32
|
+
def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
|
|
33
|
+
if "curl_cffi_options" in request.meta:
|
|
34
|
+
return deferred_from_coro(self._download_request(request, spider))
|
|
35
|
+
return super().download_request(request, spider)
|
|
36
|
+
|
|
37
|
+
async def _download_request(self, request: Request, spider: Spider) -> Response: # noqa: ARG002
|
|
38
|
+
curl_cffi_request_kwargs = to_curl_cffi_request_kwargs(request)
|
|
39
|
+
if self._session:
|
|
40
|
+
curl_cffi_response = await self._session.request(**curl_cffi_request_kwargs)
|
|
41
|
+
else:
|
|
42
|
+
async with self._create_session() as session:
|
|
43
|
+
curl_cffi_response = await session.request(**curl_cffi_request_kwargs)
|
|
44
|
+
return to_scrapy_response(curl_cffi_response, request)
|
|
45
|
+
|
|
46
|
+
def _create_session(
|
|
47
|
+
self,
|
|
48
|
+
max_clients: int = 1,
|
|
49
|
+
verify: bool = False, # noqa: FBT001, FBT002
|
|
50
|
+
) -> curl_cffi.AsyncSession:
|
|
51
|
+
return curl_cffi.AsyncSession(
|
|
52
|
+
max_clients=max_clients,
|
|
53
|
+
cookies=_DummyCookieJar(),
|
|
54
|
+
trust_env=False,
|
|
55
|
+
allow_redirects=False,
|
|
56
|
+
verify=verify,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
def close(self) -> Deferred[None]:
|
|
60
|
+
d = super().close()
|
|
61
|
+
if self._session:
|
|
62
|
+
close_session_dfd = deferred_from_coro(self._session.close())
|
|
63
|
+
d.addBoth(lambda _: close_session_dfd)
|
|
64
|
+
return d
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _DummyCookieJar(CookieJar):
|
|
68
|
+
def set_cookie(self, cookie: Cookie) -> None:
|
|
69
|
+
pass
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
|
|
3
|
+
from scrapy import Request, Spider, signals
|
|
4
|
+
from scrapy.crawler import Crawler
|
|
5
|
+
from scrapy.downloadermiddlewares.defaultheaders import (
|
|
6
|
+
DefaultHeadersMiddleware as _DefaultHeadersMiddleware,
|
|
7
|
+
)
|
|
8
|
+
from scrapy.downloadermiddlewares.useragent import (
|
|
9
|
+
UserAgentMiddleware as _UserAgentMiddleware,
|
|
10
|
+
)
|
|
11
|
+
from scrapy.http import Response
|
|
12
|
+
from typing_extensions import Self
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CurlCffiMiddleware:
|
|
16
|
+
def __init__(self, options: Mapping | None) -> None:
|
|
17
|
+
self._options = options
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def from_crawler(cls, crawler: Crawler) -> Self:
|
|
21
|
+
settings = crawler.settings
|
|
22
|
+
if "CURL_CFFI_OPTIONS" in settings:
|
|
23
|
+
options = settings.getdict("CURL_CFFI_OPTIONS")
|
|
24
|
+
else:
|
|
25
|
+
options = None
|
|
26
|
+
mw = cls(options)
|
|
27
|
+
crawler.signals.connect(mw.spider_opened, signal=signals.spider_opened)
|
|
28
|
+
return mw
|
|
29
|
+
|
|
30
|
+
def spider_opened(self, spider: Spider) -> None:
|
|
31
|
+
self._options = getattr(spider, "curl_cffi_options", self._options)
|
|
32
|
+
|
|
33
|
+
def process_request(
|
|
34
|
+
self,
|
|
35
|
+
request: Request,
|
|
36
|
+
spider: Spider, # noqa: ARG002
|
|
37
|
+
) -> Request | Response | None:
|
|
38
|
+
if self._options is not None:
|
|
39
|
+
request.meta.setdefault("curl_cffi_options", self._options)
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _CurlCffiMixin:
|
|
44
|
+
def process_request(
|
|
45
|
+
self, request: Request, spider: Spider
|
|
46
|
+
) -> Request | Response | None:
|
|
47
|
+
if (
|
|
48
|
+
"curl_cffi_options" in request.meta # _
|
|
49
|
+
and request.meta["curl_cffi_options"].get("default_headers", True)
|
|
50
|
+
):
|
|
51
|
+
return None
|
|
52
|
+
return super().process_request(request, spider) # type: ignore
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class DefaultHeadersMiddleware(_CurlCffiMixin, _DefaultHeadersMiddleware):
|
|
56
|
+
@classmethod
|
|
57
|
+
def from_crawler(cls, crawler: Crawler) -> Self:
|
|
58
|
+
mw = super().from_crawler(crawler)
|
|
59
|
+
crawler.signals.connect(mw.spider_opened, signal=signals.spider_opened)
|
|
60
|
+
return mw
|
|
61
|
+
|
|
62
|
+
def spider_opened(self, spider: Spider) -> None:
|
|
63
|
+
headers = getattr(spider, "default_request_headers", None) or {}
|
|
64
|
+
self._headers = headers.items() if isinstance(headers, Mapping) else headers
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class UserAgentMiddleware(_CurlCffiMixin, _UserAgentMiddleware):
|
|
68
|
+
pass
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from base64 import b64decode
|
|
2
|
+
from urllib.parse import unquote
|
|
3
|
+
|
|
4
|
+
from scrapy.utils.python import to_bytes
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def parse_basic_auth_header(header_value: str | bytes) -> tuple[str, str]:
|
|
8
|
+
parts = to_bytes(header_value).split(b" ")
|
|
9
|
+
if parts[0] != b"Basic" or len(parts) != 2: # noqa: PLR2004
|
|
10
|
+
msg = "Invalid Basic Auth header"
|
|
11
|
+
raise ValueError(msg)
|
|
12
|
+
tokens = b64decode(parts[1]).split(b":", 1)
|
|
13
|
+
return unquote(tokens[0]), unquote(tokens[1])
|