scrapy-lightpanda-middleware 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapy_lightpanda_middleware-0.1.0/LICENSE +29 -0
- scrapy_lightpanda_middleware-0.1.0/MANIFEST.in +1 -0
- scrapy_lightpanda_middleware-0.1.0/PKG-INFO +116 -0
- scrapy_lightpanda_middleware-0.1.0/README.md +100 -0
- scrapy_lightpanda_middleware-0.1.0/examples/quotes_spider.py +41 -0
- scrapy_lightpanda_middleware-0.1.0/examples/spider.py +35 -0
- scrapy_lightpanda_middleware-0.1.0/pyproject.toml +19 -0
- scrapy_lightpanda_middleware-0.1.0/setup.cfg +4 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda/__init__.py +3 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda/middleware.py +215 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda_middleware.egg-info/PKG-INFO +116 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda_middleware.egg-info/SOURCES.txt +17 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda_middleware.egg-info/dependency_links.txt +1 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda_middleware.egg-info/requires.txt +7 -0
- scrapy_lightpanda_middleware-0.1.0/src/scrapy_lightpanda_middleware.egg-info/top_level.txt +1 -0
- scrapy_lightpanda_middleware-0.1.0/tests/test_example.py +20 -0
- scrapy_lightpanda_middleware-0.1.0/tests/test_middleware.py +470 -0
- scrapy_lightpanda_middleware-0.1.0/tests/test_package.py +17 -0
- scrapy_lightpanda_middleware-0.1.0/tests/test_quotes_spider.py +88 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Kristian Marlowe Ole
|
|
4
|
+
All rights reserved.
|
|
5
|
+
|
|
6
|
+
Redistribution and use in source and binary forms, with or without
|
|
7
|
+
modification, are permitted provided that the following conditions are met:
|
|
8
|
+
|
|
9
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
10
|
+
list of conditions and the following disclaimer.
|
|
11
|
+
|
|
12
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
13
|
+
this list of conditions and the following disclaimer in the documentation
|
|
14
|
+
and/or other materials provided with the distribution.
|
|
15
|
+
|
|
16
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
17
|
+
contributors may be used to endorse or promote products derived from
|
|
18
|
+
this software without specific prior written permission.
|
|
19
|
+
|
|
20
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
|
|
21
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
22
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
23
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
24
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
25
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
26
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
27
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
28
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
29
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
recursive-include examples *.py
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scrapy-lightpanda-middleware
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Opt-in Lightpanda downloader middleware for Scrapy
|
|
5
|
+
License-Expression: BSD-3-Clause
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: Scrapy<3,>=2.11
|
|
10
|
+
Requires-Dist: lightpanda-py
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
13
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
14
|
+
Requires-Dist: twine>=6.0; extra == "dev"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# scrapy-lightpanda
|
|
18
|
+
|
|
19
|
+
[](https://github.com/bisdak/scrapy-lightpanda/actions/workflows/ci.yml)
|
|
20
|
+
|
|
21
|
+
An opt-in Scrapy downloader middleware that renders selected GET requests with [Lightpanda](https://github.com/lightpanda-io/browser) via [`lightpanda-py`](https://github.com/tclesius/lightpanda-py). Unmarked requests continue through Scrapy's normal downloader.
|
|
22
|
+
|
|
23
|
+
## Install
|
|
24
|
+
|
|
25
|
+
After publication, install the package with:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
python -m pip install scrapy-lightpanda-middleware
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
For development from this repository:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
python -m venv .venv
|
|
35
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
36
|
+
python -m pip install -e '.[dev]'
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`lightpanda-py` bundles the Lightpanda executable for supported platforms.
|
|
40
|
+
|
|
41
|
+
## Enable the middleware
|
|
42
|
+
|
|
43
|
+
Enable it in a spider's `custom_settings` or your Scrapy project's settings:
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
DOWNLOADER_MIDDLEWARES = {
|
|
47
|
+
"scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
|
|
48
|
+
}
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Keep this middleware after Scrapy's request-preparation middleware, but before HTTP cache middleware. Priority `800` places it after Scrapy's default headers, cookies, and proxy middleware and before HTTP cache. Opted-in requests are marked uncacheable so rendered and ordinary responses cannot share cache entries.
|
|
52
|
+
|
|
53
|
+
Opt in individual requests with `meta={"lightpanda": True}`:
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
yield scrapy.Request(
|
|
57
|
+
"https://example.com/javascript-page",
|
|
58
|
+
callback=self.parse,
|
|
59
|
+
meta={
|
|
60
|
+
"lightpanda": True,
|
|
61
|
+
"lightpanda_wait_until": "networkidle",
|
|
62
|
+
"lightpanda_wait_selector": "main article",
|
|
63
|
+
},
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The middleware accepts these optional per-request settings:
|
|
68
|
+
|
|
69
|
+
- `lightpanda_wait_until`: Lightpanda navigation readiness event, such as `load`, `domcontentloaded`, `networkalmostidle`, `networkidle`, or `done`.
|
|
70
|
+
- `lightpanda_wait_selector`: CSS selector to wait for before returning the rendered page.
|
|
71
|
+
|
|
72
|
+
Only `Accept`, `Accept-Language`, and `User-Agent` are forwarded. Repeated `Accept` and `Accept-Language` values are combined, and `User-Agent` uses Lightpanda's dedicated option. `Referer`, cookies, authorization, and custom headers are not forwarded, even with opt-in: Lightpanda applies CLI headers to every outgoing request, including cross-origin subrequests. Sensitive origin headers cause a clear error when the opt-in is set rather than risking disclosure.
|
|
73
|
+
|
|
74
|
+
An unauthenticated Scrapy proxy (`request.meta["proxy"]`) is forwarded to Lightpanda. Scrapy's Basic proxy credentials are reconstructed from `Proxy-Authorization` into the proxy URL only when `lightpanda_allow_sensitive_headers=True`; other proxy authorization schemes are not supported. This opt-in is only for proxy credentials. Credentialed proxy values become subprocess arguments and may be visible to same-user process inspection; Lightpanda errors are redacted when such credentials are in use. Inherited proxy environment variables can override Scrapy's decision, so the middleware fails clearly if they conflict rather than silently using or bypassing a proxy.
|
|
75
|
+
|
|
76
|
+
Scrapy's per-request `download_timeout` (seconds), or the configured `DOWNLOAD_TIMEOUT` when no request override is set, becomes Lightpanda's `http_timeout` in milliseconds. Positive sub-millisecond values round up to one millisecond. Results without an HTTP status are treated as retryable download timeouts. The timeout limits Lightpanda's network request but is not a hard deadline for its subprocess; cancelling the Scrapy worker thread does not guarantee that the subprocess is stopped. Lightpanda also rejects browser-impersonating User-Agent strings containing `Mozilla`; use a compatible custom value if setting `USER_AGENT`.
|
|
77
|
+
|
|
78
|
+
The returned object is a Scrapy `HtmlResponse`, so callbacks can use the usual `.css()`, `.xpath()`, `.text`, and `.url` APIs. The synchronous Lightpanda fetch runs in a worker thread rather than blocking Scrapy's reactor. Lightpanda errors are passed back through Scrapy's normal request failure handling.
|
|
79
|
+
|
|
80
|
+
Only opted-in GET requests are supported. An opted-in request using another HTTP method raises Scrapy's `NotSupported` exception; it is not silently sent through a different request path.
|
|
81
|
+
|
|
82
|
+
## Example
|
|
83
|
+
|
|
84
|
+
[`examples/spider.py`](examples/spider.py) issues one normal Scrapy request and one Lightpanda-rendered request to the same URL:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
scrapy runspider examples/spider.py -a url=https://example.com
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Lightpanda is a lightweight browser engine, not a full desktop browser. JavaScript and browser feature coverage can differ from Chromium or Firefox, so verify the rendered output for your target site.
|
|
91
|
+
|
|
92
|
+
## JavaScript-rendered quotes example
|
|
93
|
+
|
|
94
|
+
[`examples/quotes_spider.py`](examples/quotes_spider.py) visits [Quotes to Scrape's JavaScript demo](https://quotes.toscrape.com/js/), emits each quote's text, author, and tags, and follows the next-page link using Lightpanda for every page:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
scrapy runspider examples/quotes_spider.py
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Run it from this project directory after installation. It requires network access to the demo site.
|
|
101
|
+
|
|
102
|
+
## Releases
|
|
103
|
+
|
|
104
|
+
The PyPI distribution is named `scrapy-lightpanda-middleware` (the Python import remains `scrapy_lightpanda`). Releases use PyPI Trusted Publishing; no PyPI API token is stored in GitHub.
|
|
105
|
+
|
|
106
|
+
Before the first release, configure a PyPI pending Trusted Publisher for:
|
|
107
|
+
|
|
108
|
+
- PyPI project: `scrapy-lightpanda-middleware`
|
|
109
|
+
- GitHub owner: `bisdak`
|
|
110
|
+
- Repository: `scrapy-lightpanda`
|
|
111
|
+
- Workflow filename: `publish.yml` (the file is `.github/workflows/publish.yml`)
|
|
112
|
+
- GitHub environment: `pypi`
|
|
113
|
+
|
|
114
|
+
Create the `pypi` environment in GitHub repository settings with a required reviewer and deployment tags restricted to `v*`; enable prevention of self-review and disallow administrator bypass where the repository plan permits. Protect changes to the release workflow with required PR review/branch protection. These controls matter because PyPI trusts the configured workflow identity and environment, not the workflow's event trigger.
|
|
115
|
+
|
|
116
|
+
Once the pending publisher and environment protections are configured and CI is green, publish a GitHub Release tagged `v0.1.0` to upload the first release. The release workflow verifies the tag matches `pyproject.toml` and publishes only after the full test matrix, wheel/sdist checks, and tests against the extracted source distribution pass. The shorter PyPI name `scrapy-lightpanda` belongs to a separate, unrelated project and must not be used.
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# scrapy-lightpanda
|
|
2
|
+
|
|
3
|
+
[](https://github.com/bisdak/scrapy-lightpanda/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
An opt-in Scrapy downloader middleware that renders selected GET requests with [Lightpanda](https://github.com/lightpanda-io/browser) via [`lightpanda-py`](https://github.com/tclesius/lightpanda-py). Unmarked requests continue through Scrapy's normal downloader.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
After publication, install the package with:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
python -m pip install scrapy-lightpanda-middleware
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
For development from this repository:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
python -m venv .venv
|
|
19
|
+
source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
20
|
+
python -m pip install -e '.[dev]'
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
`lightpanda-py` bundles the Lightpanda executable for supported platforms.
|
|
24
|
+
|
|
25
|
+
## Enable the middleware
|
|
26
|
+
|
|
27
|
+
Enable it in a spider's `custom_settings` or your Scrapy project's settings:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
DOWNLOADER_MIDDLEWARES = {
|
|
31
|
+
"scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Keep this middleware after Scrapy's request-preparation middleware, but before HTTP cache middleware. Priority `800` places it after Scrapy's default headers, cookies, and proxy middleware and before HTTP cache. Opted-in requests are marked uncacheable so rendered and ordinary responses cannot share cache entries.
|
|
36
|
+
|
|
37
|
+
Opt in individual requests with `meta={"lightpanda": True}`:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
yield scrapy.Request(
|
|
41
|
+
"https://example.com/javascript-page",
|
|
42
|
+
callback=self.parse,
|
|
43
|
+
meta={
|
|
44
|
+
"lightpanda": True,
|
|
45
|
+
"lightpanda_wait_until": "networkidle",
|
|
46
|
+
"lightpanda_wait_selector": "main article",
|
|
47
|
+
},
|
|
48
|
+
)
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The middleware accepts these optional per-request settings:
|
|
52
|
+
|
|
53
|
+
- `lightpanda_wait_until`: Lightpanda navigation readiness event, such as `load`, `domcontentloaded`, `networkalmostidle`, `networkidle`, or `done`.
|
|
54
|
+
- `lightpanda_wait_selector`: CSS selector to wait for before returning the rendered page.
|
|
55
|
+
|
|
56
|
+
Only `Accept`, `Accept-Language`, and `User-Agent` are forwarded. Repeated `Accept` and `Accept-Language` values are combined, and `User-Agent` uses Lightpanda's dedicated option. `Referer`, cookies, authorization, and custom headers are not forwarded, even with opt-in: Lightpanda applies CLI headers to every outgoing request, including cross-origin subrequests. Sensitive origin headers cause a clear error when the opt-in is set rather than risking disclosure.
|
|
57
|
+
|
|
58
|
+
An unauthenticated Scrapy proxy (`request.meta["proxy"]`) is forwarded to Lightpanda. Scrapy's Basic proxy credentials are reconstructed from `Proxy-Authorization` into the proxy URL only when `lightpanda_allow_sensitive_headers=True`; other proxy authorization schemes are not supported. This opt-in is only for proxy credentials. Credentialed proxy values become subprocess arguments and may be visible to same-user process inspection; Lightpanda errors are redacted when such credentials are in use. Inherited proxy environment variables can override Scrapy's decision, so the middleware fails clearly if they conflict rather than silently using or bypassing a proxy.
|
|
59
|
+
|
|
60
|
+
Scrapy's per-request `download_timeout` (seconds), or the configured `DOWNLOAD_TIMEOUT` when no request override is set, becomes Lightpanda's `http_timeout` in milliseconds. Positive sub-millisecond values round up to one millisecond. Results without an HTTP status are treated as retryable download timeouts. The timeout limits Lightpanda's network request but is not a hard deadline for its subprocess; cancelling the Scrapy worker thread does not guarantee that the subprocess is stopped. Lightpanda also rejects browser-impersonating User-Agent strings containing `Mozilla`; use a compatible custom value if setting `USER_AGENT`.
|
|
61
|
+
|
|
62
|
+
The returned object is a Scrapy `HtmlResponse`, so callbacks can use the usual `.css()`, `.xpath()`, `.text`, and `.url` APIs. The synchronous Lightpanda fetch runs in a worker thread rather than blocking Scrapy's reactor. Lightpanda errors are passed back through Scrapy's normal request failure handling.
|
|
63
|
+
|
|
64
|
+
Only opted-in GET requests are supported. An opted-in request using another HTTP method raises Scrapy's `NotSupported` exception; it is not silently sent through a different request path.
|
|
65
|
+
|
|
66
|
+
## Example
|
|
67
|
+
|
|
68
|
+
[`examples/spider.py`](examples/spider.py) issues one normal Scrapy request and one Lightpanda-rendered request to the same URL:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
scrapy runspider examples/spider.py -a url=https://example.com
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Lightpanda is a lightweight browser engine, not a full desktop browser. JavaScript and browser feature coverage can differ from Chromium or Firefox, so verify the rendered output for your target site.
|
|
75
|
+
|
|
76
|
+
## JavaScript-rendered quotes example
|
|
77
|
+
|
|
78
|
+
[`examples/quotes_spider.py`](examples/quotes_spider.py) visits [Quotes to Scrape's JavaScript demo](https://quotes.toscrape.com/js/), emits each quote's text, author, and tags, and follows the next-page link using Lightpanda for every page:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
scrapy runspider examples/quotes_spider.py
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Run it from this project directory after installation. It requires network access to the demo site.
|
|
85
|
+
|
|
86
|
+
## Releases
|
|
87
|
+
|
|
88
|
+
The PyPI distribution is named `scrapy-lightpanda-middleware` (the Python import remains `scrapy_lightpanda`). Releases use PyPI Trusted Publishing; no PyPI API token is stored in GitHub.
|
|
89
|
+
|
|
90
|
+
Before the first release, configure a PyPI pending Trusted Publisher for:
|
|
91
|
+
|
|
92
|
+
- PyPI project: `scrapy-lightpanda-middleware`
|
|
93
|
+
- GitHub owner: `bisdak`
|
|
94
|
+
- Repository: `scrapy-lightpanda`
|
|
95
|
+
- Workflow filename: `publish.yml` (the file is `.github/workflows/publish.yml`)
|
|
96
|
+
- GitHub environment: `pypi`
|
|
97
|
+
|
|
98
|
+
Create the `pypi` environment in GitHub repository settings with a required reviewer and deployment tags restricted to `v*`; enable prevention of self-review and disallow administrator bypass where the repository plan permits. Protect changes to the release workflow with required PR review/branch protection. These controls matter because PyPI trusts the configured workflow identity and environment, not the workflow's event trigger.
|
|
99
|
+
|
|
100
|
+
Once the pending publisher and environment protections are configured and CI is green, publish a GitHub Release tagged `v0.1.0` to upload the first release. The release workflow verifies the tag matches `pyproject.toml` and publishes only after the full test matrix, wheel/sdist checks, and tests against the extracted source distribution pass. The shorter PyPI name `scrapy-lightpanda` belongs to a separate, unrelated project and must not be used.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Scrape quotes from the JavaScript-rendered Quotes to Scrape demo."""
|
|
2
|
+
|
|
3
|
+
import scrapy
|
|
4
|
+
from scrapy import Request
|
|
5
|
+
from scrapy.http import Response
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _clean_text(value: str) -> str:
|
|
9
|
+
return " ".join(value.split())
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QuotesSpider(scrapy.Spider):
|
|
13
|
+
name = "quotes_js"
|
|
14
|
+
start_url = "https://quotes.toscrape.com/js/"
|
|
15
|
+
|
|
16
|
+
custom_settings = {
|
|
17
|
+
"DOWNLOADER_MIDDLEWARES": {
|
|
18
|
+
"scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
|
|
19
|
+
},
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async def start(self):
|
|
23
|
+
yield self._start_request()
|
|
24
|
+
|
|
25
|
+
def start_requests(self):
|
|
26
|
+
yield self._start_request()
|
|
27
|
+
|
|
28
|
+
def _start_request(self) -> Request:
|
|
29
|
+
return Request(self.start_url, callback=self.parse, meta={"lightpanda": True})
|
|
30
|
+
|
|
31
|
+
def parse(self, response: Response):
|
|
32
|
+
for quote in response.css(".quote"):
|
|
33
|
+
yield {
|
|
34
|
+
"text": _clean_text(" ".join(quote.css(".text::text").getall())),
|
|
35
|
+
"author": _clean_text(" ".join(quote.css(".author::text").getall())),
|
|
36
|
+
"tags": [_clean_text(tag) for tag in quote.css(".tag::text").getall()],
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
next_page = response.css("li.next > a::attr(href)").get()
|
|
40
|
+
if next_page:
|
|
41
|
+
yield response.follow(next_page, callback=self.parse, meta={"lightpanda": True})
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Run with: scrapy runspider examples/spider.py -a url=https://example.com."""
|
|
2
|
+
|
|
3
|
+
import scrapy
|
|
4
|
+
from scrapy import Request
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LightpandaExampleSpider(scrapy.Spider):
|
|
8
|
+
name = "lightpanda_example"
|
|
9
|
+
|
|
10
|
+
custom_settings = {
|
|
11
|
+
"DOWNLOADER_MIDDLEWARES": {
|
|
12
|
+
"scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
|
|
13
|
+
},
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
def __init__(self, url: str = "https://example.com", **kwargs):
|
|
17
|
+
super().__init__(**kwargs)
|
|
18
|
+
self.url = url
|
|
19
|
+
|
|
20
|
+
def start_requests(self):
|
|
21
|
+
yield Request(self.url, callback=self.parse, cb_kwargs={"rendered": False})
|
|
22
|
+
yield Request(
|
|
23
|
+
self.url,
|
|
24
|
+
callback=self.parse,
|
|
25
|
+
cb_kwargs={"rendered": True},
|
|
26
|
+
dont_filter=True,
|
|
27
|
+
meta={"lightpanda": True, "lightpanda_wait_until": "networkidle"},
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
def parse(self, response, rendered: bool):
|
|
31
|
+
yield {
|
|
32
|
+
"url": response.url,
|
|
33
|
+
"rendered_with_lightpanda": rendered,
|
|
34
|
+
"title": response.css("title::text").get(),
|
|
35
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scrapy-lightpanda-middleware"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Opt-in Lightpanda downloader middleware for Scrapy"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "BSD-3-Clause"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
dependencies = ["Scrapy>=2.11,<3", "lightpanda-py"]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
dev = ["pytest>=8.0", "build>=1.2", "twine>=6.0"]
|
|
17
|
+
|
|
18
|
+
[tool.setuptools.packages.find]
|
|
19
|
+
where = ["src"]
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Scrapy downloader middleware backed by Lightpanda's ephemeral browser."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import base64
|
|
7
|
+
import binascii
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
from urllib.parse import quote_from_bytes, urlsplit, urlunsplit
|
|
11
|
+
from urllib.request import proxy_bypass
|
|
12
|
+
|
|
13
|
+
import lightpanda
|
|
14
|
+
from scrapy import Request, Spider
|
|
15
|
+
from scrapy.exceptions import NotSupported
|
|
16
|
+
from scrapy.http import Headers, HtmlResponse, Response
|
|
17
|
+
from twisted.internet.error import TimeoutError as TwistedTimeoutError
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
from scrapy.exceptions import DownloadTimeoutError
|
|
21
|
+
except ImportError: # Scrapy < 2.20 uses Twisted timeout exceptions for retries.
|
|
22
|
+
DownloadTimeoutError = TwistedTimeoutError
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class LightpandaMiddleware:
|
|
26
|
+
"""Render requests marked with ``meta={"lightpanda": True}``."""
|
|
27
|
+
|
|
28
|
+
_safe_headers = {"accept", "accept-language", "user-agent"}
|
|
29
|
+
_sensitive_headers = {
|
|
30
|
+
"authorization",
|
|
31
|
+
"cookie",
|
|
32
|
+
"proxy-authorization",
|
|
33
|
+
"x-api-key",
|
|
34
|
+
"api-key",
|
|
35
|
+
"x-auth-token",
|
|
36
|
+
"x-access-token",
|
|
37
|
+
}
|
|
38
|
+
_proxy_environment = (
|
|
39
|
+
"http_proxy",
|
|
40
|
+
"HTTP_PROXY",
|
|
41
|
+
"https_proxy",
|
|
42
|
+
"HTTPS_PROXY",
|
|
43
|
+
"all_proxy",
|
|
44
|
+
"ALL_PROXY",
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def __init__(self, default_download_timeout: float | None = None):
|
|
48
|
+
self.default_download_timeout = default_download_timeout
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def from_crawler(cls, crawler) -> LightpandaMiddleware:
|
|
52
|
+
"""Construct the middleware using Scrapy's downloader middleware API."""
|
|
53
|
+
return cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
|
|
54
|
+
|
|
55
|
+
async def process_request(
|
|
56
|
+
self, request: Request, spider: Spider | None = None
|
|
57
|
+
) -> Response | None:
|
|
58
|
+
"""Fetch opted-in GET requests asynchronously from Scrapy's perspective."""
|
|
59
|
+
if not request.meta.get("lightpanda"):
|
|
60
|
+
return None
|
|
61
|
+
if request.method != "GET":
|
|
62
|
+
raise NotSupported("LightpandaMiddleware only supports opted-in GET requests")
|
|
63
|
+
proxy = request.meta.get("proxy")
|
|
64
|
+
allow_sensitive = request.meta.get("lightpanda_allow_sensitive_headers") is True
|
|
65
|
+
proxy_authorization = request.headers.get(b"Proxy-Authorization")
|
|
66
|
+
sensitive_origin_headers = self._sensitive_headers - {"proxy-authorization"}
|
|
67
|
+
for name in request.headers:
|
|
68
|
+
normalized_name = name.decode("latin-1").lower()
|
|
69
|
+
if allow_sensitive and normalized_name in sensitive_origin_headers:
|
|
70
|
+
raise NotSupported(
|
|
71
|
+
"Sensitive origin headers cannot be forwarded to Lightpanda; "
|
|
72
|
+
"Lightpanda applies CLI headers to cross-origin subrequests"
|
|
73
|
+
)
|
|
74
|
+
if proxy_authorization is not None and not allow_sensitive:
|
|
75
|
+
raise NotSupported(
|
|
76
|
+
"Credentialed Lightpanda proxies require "
|
|
77
|
+
"meta['lightpanda_allow_sensitive_headers']=True"
|
|
78
|
+
)
|
|
79
|
+
if proxy and self._proxy_has_credentials(proxy) and not allow_sensitive:
|
|
80
|
+
raise NotSupported(
|
|
81
|
+
"Credentialed Lightpanda proxies require "
|
|
82
|
+
"meta['lightpanda_allow_sensitive_headers']=True"
|
|
83
|
+
)
|
|
84
|
+
if proxy and proxy_authorization is not None:
|
|
85
|
+
proxy = self._proxy_with_authorization(proxy, proxy_authorization)
|
|
86
|
+
self._validate_proxy_environment(request.url, proxy)
|
|
87
|
+
request.meta["dont_cache"] = True
|
|
88
|
+
return await asyncio.to_thread(self._fetch_response, request, proxy)
|
|
89
|
+
|
|
90
|
+
@classmethod
|
|
91
|
+
def _validate_proxy_environment(cls, url: str, proxy: str | None) -> None:
|
|
92
|
+
if proxy is None and any(os.environ.get(name) for name in cls._proxy_environment):
|
|
93
|
+
raise NotSupported(
|
|
94
|
+
"Lightpanda inherits proxy environment variables; clear them or "
|
|
95
|
+
"set an explicit Scrapy proxy to avoid proxy environment conflicts"
|
|
96
|
+
)
|
|
97
|
+
if proxy is not None and proxy_bypass(urlsplit(url).hostname or ""):
|
|
98
|
+
raise NotSupported(
|
|
99
|
+
"The no_proxy environment bypasses this request's explicit Scrapy proxy"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def _proxy_has_credentials(proxy: str) -> bool:
|
|
104
|
+
proxy_parts = urlsplit(proxy)
|
|
105
|
+
return proxy_parts.username is not None or proxy_parts.password is not None
|
|
106
|
+
|
|
107
|
+
@staticmethod
|
|
108
|
+
def _proxy_with_authorization(proxy: str, proxy_authorization: bytes) -> str:
|
|
109
|
+
try:
|
|
110
|
+
scheme, encoded_credentials = proxy_authorization.split(None, 1)
|
|
111
|
+
if scheme.lower() != b"basic":
|
|
112
|
+
raise NotSupported("Only Basic proxy authorization is supported by Lightpanda")
|
|
113
|
+
credentials = base64.b64decode(encoded_credentials, validate=True)
|
|
114
|
+
username, password = credentials.split(b":", 1)
|
|
115
|
+
except (ValueError, binascii.Error) as exc:
|
|
116
|
+
raise NotSupported("Invalid Basic Proxy-Authorization header") from exc
|
|
117
|
+
|
|
118
|
+
parts = urlsplit(proxy)
|
|
119
|
+
user_info = f"{quote_from_bytes(username, safe='')}:{quote_from_bytes(password, safe='')}@"
|
|
120
|
+
host_port = parts.netloc.rsplit("@", 1)[-1]
|
|
121
|
+
return urlunsplit(
|
|
122
|
+
(parts.scheme, user_info + host_port, parts.path, parts.query, parts.fragment)
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
def _fetch_response(self, request: Request, proxy: str | None) -> HtmlResponse:
|
|
126
|
+
options: dict[str, str | int | list[str]] = {}
|
|
127
|
+
wait_until = request.meta.get("lightpanda_wait_until")
|
|
128
|
+
wait_selector = request.meta.get("lightpanda_wait_selector")
|
|
129
|
+
if wait_until is not None:
|
|
130
|
+
options["wait_until"] = wait_until
|
|
131
|
+
if wait_selector is not None:
|
|
132
|
+
options["wait_selector"] = wait_selector
|
|
133
|
+
|
|
134
|
+
forwarded_headers: list[str] = []
|
|
135
|
+
for name in request.headers:
|
|
136
|
+
normalized_name = name.decode("latin-1").lower()
|
|
137
|
+
if normalized_name in {"proxy-authorization", "referer"}:
|
|
138
|
+
continue
|
|
139
|
+
if normalized_name not in self._safe_headers:
|
|
140
|
+
continue
|
|
141
|
+
values = request.headers.getlist(name)
|
|
142
|
+
if normalized_name == "user-agent":
|
|
143
|
+
if values:
|
|
144
|
+
options["user_agent"] = self._decode_header_value(values[0])
|
|
145
|
+
continue
|
|
146
|
+
if values:
|
|
147
|
+
header_values = [self._decode_header_value(value) for value in values]
|
|
148
|
+
forwarded_headers.append(
|
|
149
|
+
f"{name.decode('ascii')}: {', '.join(header_values)}"
|
|
150
|
+
)
|
|
151
|
+
if forwarded_headers:
|
|
152
|
+
options["http_header"] = forwarded_headers
|
|
153
|
+
|
|
154
|
+
if proxy:
|
|
155
|
+
options["http_proxy"] = proxy
|
|
156
|
+
|
|
157
|
+
timeout = request.meta.get("download_timeout", self.default_download_timeout)
|
|
158
|
+
if timeout is not None:
|
|
159
|
+
timeout_ms = float(timeout) * 1000
|
|
160
|
+
options["http_timeout"] = max(1, math.ceil(timeout_ms)) if timeout_ms > 0 else 0
|
|
161
|
+
|
|
162
|
+
has_proxy_credentials = bool(proxy and self._proxy_has_credentials(proxy))
|
|
163
|
+
try:
|
|
164
|
+
result = lightpanda.fetch(request.url, json=True, **options).json()
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
if has_proxy_credentials:
|
|
167
|
+
raise RuntimeError(
|
|
168
|
+
"Lightpanda fetch failed while using a credentialed proxy; "
|
|
169
|
+
"error details redacted"
|
|
170
|
+
) from None
|
|
171
|
+
if isinstance(exc, lightpanda.Error) and "OperationTimedout" in str(exc):
|
|
172
|
+
raise DownloadTimeoutError("Lightpanda request timed out") from None
|
|
173
|
+
raise
|
|
174
|
+
|
|
175
|
+
content = result.get("content", "")
|
|
176
|
+
body = content if isinstance(content, bytes) else str(content).encode("utf-8")
|
|
177
|
+
status_value = result.get("http_status")
|
|
178
|
+
if not status_value:
|
|
179
|
+
raise DownloadTimeoutError("Lightpanda did not receive an HTTP response")
|
|
180
|
+
status = int(status_value)
|
|
181
|
+
headers = self._response_headers(result.get("headers"))
|
|
182
|
+
|
|
183
|
+
return HtmlResponse(
|
|
184
|
+
url=result.get("url") or request.url,
|
|
185
|
+
status=status,
|
|
186
|
+
headers=headers,
|
|
187
|
+
body=body,
|
|
188
|
+
encoding="utf-8",
|
|
189
|
+
request=request,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
@staticmethod
|
|
193
|
+
def _decode_header_value(value: bytes) -> str:
|
|
194
|
+
try:
|
|
195
|
+
return value.decode("ascii")
|
|
196
|
+
except UnicodeDecodeError as exc:
|
|
197
|
+
raise NotSupported(
|
|
198
|
+
"Lightpanda CLI header values must be ASCII; non-ASCII bytes "
|
|
199
|
+
"cannot be preserved through subprocess arguments"
|
|
200
|
+
) from exc
|
|
201
|
+
|
|
202
|
+
@staticmethod
|
|
203
|
+
def _response_headers(raw_headers) -> Headers:
|
|
204
|
+
headers = Headers()
|
|
205
|
+
if isinstance(raw_headers, dict):
|
|
206
|
+
for name, value in raw_headers.items():
|
|
207
|
+
values = value if isinstance(value, list) else [value]
|
|
208
|
+
for item in values:
|
|
209
|
+
headers.appendlist(name, item)
|
|
210
|
+
else:
|
|
211
|
+
for header in raw_headers or []:
|
|
212
|
+
headers.appendlist(header["name"], header["value"])
|
|
213
|
+
for name in (b"Content-Encoding", b"Content-Length", b"Transfer-Encoding"):
|
|
214
|
+
headers.pop(name, None)
|
|
215
|
+
return headers
|