scrapy-lightpanda-middleware 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,29 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, Kristian Marlowe Ole
4
+ All rights reserved.
5
+
6
+ Redistribution and use in source and binary forms, with or without
7
+ modification, are permitted provided that the following conditions are met:
8
+
9
+ 1. Redistributions of source code must retain the above copyright notice, this
10
+ list of conditions and the following disclaimer.
11
+
12
+ 2. Redistributions in binary form must reproduce the above copyright notice,
13
+ this list of conditions and the following disclaimer in the documentation
14
+ and/or other materials provided with the distribution.
15
+
16
+ 3. Neither the name of the copyright holder nor the names of its
17
+ contributors may be used to endorse or promote products derived from
18
+ this software without specific prior written permission.
19
+
20
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
21
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1 @@
1
+ recursive-include examples *.py
@@ -0,0 +1,116 @@
1
+ Metadata-Version: 2.4
2
+ Name: scrapy-lightpanda-middleware
3
+ Version: 0.1.0
4
+ Summary: Opt-in Lightpanda downloader middleware for Scrapy
5
+ License-Expression: BSD-3-Clause
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ License-File: LICENSE
9
+ Requires-Dist: Scrapy<3,>=2.11
10
+ Requires-Dist: lightpanda-py
11
+ Provides-Extra: dev
12
+ Requires-Dist: pytest>=8.0; extra == "dev"
13
+ Requires-Dist: build>=1.2; extra == "dev"
14
+ Requires-Dist: twine>=6.0; extra == "dev"
15
+ Dynamic: license-file
16
+
17
+ # scrapy-lightpanda
18
+
19
+ [![CI](https://github.com/bisdak/scrapy-lightpanda/actions/workflows/ci.yml/badge.svg)](https://github.com/bisdak/scrapy-lightpanda/actions/workflows/ci.yml)
20
+
21
+ An opt-in Scrapy downloader middleware that renders selected GET requests with [Lightpanda](https://github.com/lightpanda-io/browser) via [`lightpanda-py`](https://github.com/tclesius/lightpanda-py). Unmarked requests continue through Scrapy's normal downloader.
22
+
23
+ ## Install
24
+
25
+ After publication, install the package with:
26
+
27
+ ```bash
28
+ python -m pip install scrapy-lightpanda-middleware
29
+ ```
30
+
31
+ For development from this repository:
32
+
33
+ ```bash
34
+ python -m venv .venv
35
+ source .venv/bin/activate # Windows: .venv\Scripts\activate
36
+ python -m pip install -e '.[dev]'
37
+ ```
38
+
39
+ `lightpanda-py` bundles the Lightpanda executable for supported platforms.
40
+
41
+ ## Enable the middleware
42
+
43
+ Enable it in a spider's `custom_settings` or your Scrapy project's settings:
44
+
45
+ ```python
46
+ DOWNLOADER_MIDDLEWARES = {
47
+ "scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
48
+ }
49
+ ```
50
+
51
+ Keep this middleware after Scrapy's request-preparation middleware, but before HTTP cache middleware. Priority `800` places it after Scrapy's default headers, cookies, and proxy middleware and before HTTP cache. Opted-in requests are marked uncacheable so rendered and ordinary responses cannot share cache entries.
52
+
53
+ Opt in individual requests with `meta={"lightpanda": True}`:
54
+
55
+ ```python
56
+ yield scrapy.Request(
57
+ "https://example.com/javascript-page",
58
+ callback=self.parse,
59
+ meta={
60
+ "lightpanda": True,
61
+ "lightpanda_wait_until": "networkidle",
62
+ "lightpanda_wait_selector": "main article",
63
+ },
64
+ )
65
+ ```
66
+
67
+ The middleware accepts these optional per-request settings:
68
+
69
+ - `lightpanda_wait_until`: Lightpanda navigation readiness event, such as `load`, `domcontentloaded`, `networkalmostidle`, `networkidle`, or `done`.
70
+ - `lightpanda_wait_selector`: CSS selector to wait for before returning the rendered page.
71
+
72
+ Only `Accept`, `Accept-Language`, and `User-Agent` are forwarded. Repeated `Accept` and `Accept-Language` values are combined, and `User-Agent` uses Lightpanda's dedicated option. `Referer`, cookies, authorization, and custom headers are not forwarded, even with opt-in: Lightpanda applies CLI headers to every outgoing request, including cross-origin subrequests. Sensitive origin headers cause a clear error when the opt-in is set rather than risking disclosure.
73
+
74
+ An unauthenticated Scrapy proxy (`request.meta["proxy"]`) is forwarded to Lightpanda. Scrapy's Basic proxy credentials are reconstructed from `Proxy-Authorization` into the proxy URL only when `lightpanda_allow_sensitive_headers=True`; other proxy authorization schemes are not supported. This opt-in is only for proxy credentials. Credentialed proxy values become subprocess arguments and may be visible to same-user process inspection; Lightpanda errors are redacted when such credentials are in use. Inherited proxy environment variables can override Scrapy's decision, so the middleware fails clearly if they conflict rather than silently using or bypassing a proxy.
75
+
76
+ Scrapy's per-request `download_timeout` (seconds), or the configured `DOWNLOAD_TIMEOUT` when no request override is set, becomes Lightpanda's `http_timeout` in milliseconds. Positive sub-millisecond values round up to one millisecond. Results without an HTTP status are treated as retryable download timeouts. The timeout limits Lightpanda's network request but is not a hard deadline for its subprocess; cancelling the Scrapy worker thread does not guarantee that the subprocess is stopped. Lightpanda also rejects browser-impersonating User-Agent strings containing `Mozilla`; use a compatible custom value if setting `USER_AGENT`.
77
+
78
+ The returned object is a Scrapy `HtmlResponse`, so callbacks can use the usual `.css()`, `.xpath()`, `.text`, and `.url` APIs. The synchronous Lightpanda fetch runs in a worker thread rather than blocking Scrapy's reactor. Lightpanda errors are passed back through Scrapy's normal request failure handling.
79
+
80
+ Only opted-in GET requests are supported. An opted-in request using another HTTP method raises Scrapy's `NotSupported` exception; it is not silently sent through a different request path.
81
+
82
+ ## Example
83
+
84
+ [`examples/spider.py`](examples/spider.py) issues one normal Scrapy request and one Lightpanda-rendered request to the same URL:
85
+
86
+ ```bash
87
+ scrapy runspider examples/spider.py -a url=https://example.com
88
+ ```
89
+
90
+ Lightpanda is a lightweight browser engine, not a full desktop browser. JavaScript and browser feature coverage can differ from Chromium or Firefox, so verify the rendered output for your target site.
91
+
92
+ ## JavaScript-rendered quotes example
93
+
94
+ [`examples/quotes_spider.py`](examples/quotes_spider.py) visits [Quotes to Scrape's JavaScript demo](https://quotes.toscrape.com/js/), emits each quote's text, author, and tags, and follows the next-page link using Lightpanda for every page:
95
+
96
+ ```bash
97
+ scrapy runspider examples/quotes_spider.py
98
+ ```
99
+
100
+ Run it from this project directory after installation. It requires network access to the demo site.
101
+
102
+ ## Releases
103
+
104
+ The PyPI distribution is named `scrapy-lightpanda-middleware` (the Python import remains `scrapy_lightpanda`). Releases use PyPI Trusted Publishing; no PyPI API token is stored in GitHub.
105
+
106
+ Before the first release, configure a PyPI pending Trusted Publisher for:
107
+
108
+ - PyPI project: `scrapy-lightpanda-middleware`
109
+ - GitHub owner: `bisdak`
110
+ - Repository: `scrapy-lightpanda`
111
+ - Workflow filename: `publish.yml` (the file is `.github/workflows/publish.yml`)
112
+ - GitHub environment: `pypi`
113
+
114
+ Create the `pypi` environment in GitHub repository settings with a required reviewer and deployment tags restricted to `v*`; enable prevention of self-review and disallow administrator bypass where the repository plan permits. Protect changes to the release workflow with required PR review/branch protection. These controls matter because PyPI trusts the configured workflow identity and environment, not the workflow's event trigger.
115
+
116
+ Once the pending publisher and environment protections are configured and CI is green, publish a GitHub Release tagged `v0.1.0` to upload the first release. The release workflow verifies the tag matches `pyproject.toml` and publishes only after the full test matrix, wheel/sdist checks, and tests against the extracted source distribution pass. The shorter PyPI name `scrapy-lightpanda` belongs to a separate, unrelated project and must not be used.
@@ -0,0 +1,100 @@
1
+ # scrapy-lightpanda
2
+
3
+ [![CI](https://github.com/bisdak/scrapy-lightpanda/actions/workflows/ci.yml/badge.svg)](https://github.com/bisdak/scrapy-lightpanda/actions/workflows/ci.yml)
4
+
5
+ An opt-in Scrapy downloader middleware that renders selected GET requests with [Lightpanda](https://github.com/lightpanda-io/browser) via [`lightpanda-py`](https://github.com/tclesius/lightpanda-py). Unmarked requests continue through Scrapy's normal downloader.
6
+
7
+ ## Install
8
+
9
+ After publication, install the package with:
10
+
11
+ ```bash
12
+ python -m pip install scrapy-lightpanda-middleware
13
+ ```
14
+
15
+ For development from this repository:
16
+
17
+ ```bash
18
+ python -m venv .venv
19
+ source .venv/bin/activate # Windows: .venv\Scripts\activate
20
+ python -m pip install -e '.[dev]'
21
+ ```
22
+
23
+ `lightpanda-py` bundles the Lightpanda executable for supported platforms.
24
+
25
+ ## Enable the middleware
26
+
27
+ Enable it in a spider's `custom_settings` or your Scrapy project's settings:
28
+
29
+ ```python
30
+ DOWNLOADER_MIDDLEWARES = {
31
+ "scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
32
+ }
33
+ ```
34
+
35
+ Keep this middleware after Scrapy's request-preparation middleware, but before HTTP cache middleware. Priority `800` places it after Scrapy's default headers, cookies, and proxy middleware and before HTTP cache. Opted-in requests are marked uncacheable so rendered and ordinary responses cannot share cache entries.
36
+
37
+ Opt in individual requests with `meta={"lightpanda": True}`:
38
+
39
+ ```python
40
+ yield scrapy.Request(
41
+ "https://example.com/javascript-page",
42
+ callback=self.parse,
43
+ meta={
44
+ "lightpanda": True,
45
+ "lightpanda_wait_until": "networkidle",
46
+ "lightpanda_wait_selector": "main article",
47
+ },
48
+ )
49
+ ```
50
+
51
+ The middleware accepts these optional per-request settings:
52
+
53
+ - `lightpanda_wait_until`: Lightpanda navigation readiness event, such as `load`, `domcontentloaded`, `networkalmostidle`, `networkidle`, or `done`.
54
+ - `lightpanda_wait_selector`: CSS selector to wait for before returning the rendered page.
55
+
56
+ Only `Accept`, `Accept-Language`, and `User-Agent` are forwarded. Repeated `Accept` and `Accept-Language` values are combined, and `User-Agent` uses Lightpanda's dedicated option. `Referer`, cookies, authorization, and custom headers are not forwarded, even with opt-in: Lightpanda applies CLI headers to every outgoing request, including cross-origin subrequests. Sensitive origin headers cause a clear error when the opt-in is set rather than risking disclosure.
57
+
58
+ An unauthenticated Scrapy proxy (`request.meta["proxy"]`) is forwarded to Lightpanda. Scrapy's Basic proxy credentials are reconstructed from `Proxy-Authorization` into the proxy URL only when `lightpanda_allow_sensitive_headers=True`; other proxy authorization schemes are not supported. This opt-in is only for proxy credentials. Credentialed proxy values become subprocess arguments and may be visible to same-user process inspection; Lightpanda errors are redacted when such credentials are in use. Inherited proxy environment variables can override Scrapy's decision, so the middleware fails clearly if they conflict rather than silently using or bypassing a proxy.
59
+
60
+ Scrapy's per-request `download_timeout` (seconds), or the configured `DOWNLOAD_TIMEOUT` when no request override is set, becomes Lightpanda's `http_timeout` in milliseconds. Positive sub-millisecond values round up to one millisecond. Results without an HTTP status are treated as retryable download timeouts. The timeout limits Lightpanda's network request but is not a hard deadline for its subprocess; cancelling the Scrapy worker thread does not guarantee that the subprocess is stopped. Lightpanda also rejects browser-impersonating User-Agent strings containing `Mozilla`; use a compatible custom value if setting `USER_AGENT`.
61
+
62
+ The returned object is a Scrapy `HtmlResponse`, so callbacks can use the usual `.css()`, `.xpath()`, `.text`, and `.url` APIs. The synchronous Lightpanda fetch runs in a worker thread rather than blocking Scrapy's reactor. Lightpanda errors are passed back through Scrapy's normal request failure handling.
63
+
64
+ Only opted-in GET requests are supported. An opted-in request using another HTTP method raises Scrapy's `NotSupported` exception; it is not silently sent through a different request path.
65
+
66
+ ## Example
67
+
68
+ [`examples/spider.py`](examples/spider.py) issues one normal Scrapy request and one Lightpanda-rendered request to the same URL:
69
+
70
+ ```bash
71
+ scrapy runspider examples/spider.py -a url=https://example.com
72
+ ```
73
+
74
+ Lightpanda is a lightweight browser engine, not a full desktop browser. JavaScript and browser feature coverage can differ from Chromium or Firefox, so verify the rendered output for your target site.
75
+
76
+ ## JavaScript-rendered quotes example
77
+
78
+ [`examples/quotes_spider.py`](examples/quotes_spider.py) visits [Quotes to Scrape's JavaScript demo](https://quotes.toscrape.com/js/), emits each quote's text, author, and tags, and follows the next-page link using Lightpanda for every page:
79
+
80
+ ```bash
81
+ scrapy runspider examples/quotes_spider.py
82
+ ```
83
+
84
+ Run it from this project directory after installation. It requires network access to the demo site.
85
+
86
+ ## Releases
87
+
88
+ The PyPI distribution is named `scrapy-lightpanda-middleware` (the Python import remains `scrapy_lightpanda`). Releases use PyPI Trusted Publishing; no PyPI API token is stored in GitHub.
89
+
90
+ Before the first release, configure a PyPI pending Trusted Publisher for:
91
+
92
+ - PyPI project: `scrapy-lightpanda-middleware`
93
+ - GitHub owner: `bisdak`
94
+ - Repository: `scrapy-lightpanda`
95
+ - Workflow filename: `publish.yml` (the file is `.github/workflows/publish.yml`)
96
+ - GitHub environment: `pypi`
97
+
98
+ Create the `pypi` environment in GitHub repository settings with a required reviewer and deployment tags restricted to `v*`; enable prevention of self-review and disallow administrator bypass where the repository plan permits. Protect changes to the release workflow with required PR review/branch protection. These controls matter because PyPI trusts the configured workflow identity and environment, not the workflow's event trigger.
99
+
100
+ Once the pending publisher and environment protections are configured and CI is green, publish a GitHub Release tagged `v0.1.0` to upload the first release. The release workflow verifies the tag matches `pyproject.toml` and publishes only after the full test matrix, wheel/sdist checks, and tests against the extracted source distribution pass. The shorter PyPI name `scrapy-lightpanda` belongs to a separate, unrelated project and must not be used.
@@ -0,0 +1,41 @@
1
+ """Scrape quotes from the JavaScript-rendered Quotes to Scrape demo."""
2
+
3
+ import scrapy
4
+ from scrapy import Request
5
+ from scrapy.http import Response
6
+
7
+
8
+ def _clean_text(value: str) -> str:
9
+ return " ".join(value.split())
10
+
11
+
12
+ class QuotesSpider(scrapy.Spider):
13
+ name = "quotes_js"
14
+ start_url = "https://quotes.toscrape.com/js/"
15
+
16
+ custom_settings = {
17
+ "DOWNLOADER_MIDDLEWARES": {
18
+ "scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
19
+ },
20
+ }
21
+
22
+ async def start(self):
23
+ yield self._start_request()
24
+
25
+ def start_requests(self):
26
+ yield self._start_request()
27
+
28
+ def _start_request(self) -> Request:
29
+ return Request(self.start_url, callback=self.parse, meta={"lightpanda": True})
30
+
31
+ def parse(self, response: Response):
32
+ for quote in response.css(".quote"):
33
+ yield {
34
+ "text": _clean_text(" ".join(quote.css(".text::text").getall())),
35
+ "author": _clean_text(" ".join(quote.css(".author::text").getall())),
36
+ "tags": [_clean_text(tag) for tag in quote.css(".tag::text").getall()],
37
+ }
38
+
39
+ next_page = response.css("li.next > a::attr(href)").get()
40
+ if next_page:
41
+ yield response.follow(next_page, callback=self.parse, meta={"lightpanda": True})
@@ -0,0 +1,35 @@
1
+ """Run with: scrapy runspider examples/spider.py -a url=https://example.com."""
2
+
3
+ import scrapy
4
+ from scrapy import Request
5
+
6
+
7
+ class LightpandaExampleSpider(scrapy.Spider):
8
+ name = "lightpanda_example"
9
+
10
+ custom_settings = {
11
+ "DOWNLOADER_MIDDLEWARES": {
12
+ "scrapy_lightpanda.middleware.LightpandaMiddleware": 800,
13
+ },
14
+ }
15
+
16
+ def __init__(self, url: str = "https://example.com", **kwargs):
17
+ super().__init__(**kwargs)
18
+ self.url = url
19
+
20
+ def start_requests(self):
21
+ yield Request(self.url, callback=self.parse, cb_kwargs={"rendered": False})
22
+ yield Request(
23
+ self.url,
24
+ callback=self.parse,
25
+ cb_kwargs={"rendered": True},
26
+ dont_filter=True,
27
+ meta={"lightpanda": True, "lightpanda_wait_until": "networkidle"},
28
+ )
29
+
30
+ def parse(self, response, rendered: bool):
31
+ yield {
32
+ "url": response.url,
33
+ "rendered_with_lightpanda": rendered,
34
+ "title": response.css("title::text").get(),
35
+ }
@@ -0,0 +1,19 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "scrapy-lightpanda-middleware"
7
+ version = "0.1.0"
8
+ description = "Opt-in Lightpanda downloader middleware for Scrapy"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = "BSD-3-Clause"
12
+ license-files = ["LICENSE"]
13
+ dependencies = ["Scrapy>=2.11,<3", "lightpanda-py"]
14
+
15
+ [project.optional-dependencies]
16
+ dev = ["pytest>=8.0", "build>=1.2", "twine>=6.0"]
17
+
18
+ [tool.setuptools.packages.find]
19
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,3 @@
1
+ """Opt-in Lightpanda downloader middleware for Scrapy."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,215 @@
1
+ """Scrapy downloader middleware backed by Lightpanda's ephemeral browser."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import base64
7
+ import binascii
8
+ import math
9
+ import os
10
+ from urllib.parse import quote_from_bytes, urlsplit, urlunsplit
11
+ from urllib.request import proxy_bypass
12
+
13
+ import lightpanda
14
+ from scrapy import Request, Spider
15
+ from scrapy.exceptions import NotSupported
16
+ from scrapy.http import Headers, HtmlResponse, Response
17
+ from twisted.internet.error import TimeoutError as TwistedTimeoutError
18
+
19
+ try:
20
+ from scrapy.exceptions import DownloadTimeoutError
21
+ except ImportError: # Scrapy < 2.20 uses Twisted timeout exceptions for retries.
22
+ DownloadTimeoutError = TwistedTimeoutError
23
+
24
+
25
+ class LightpandaMiddleware:
26
+ """Render requests marked with ``meta={"lightpanda": True}``."""
27
+
28
+ _safe_headers = {"accept", "accept-language", "user-agent"}
29
+ _sensitive_headers = {
30
+ "authorization",
31
+ "cookie",
32
+ "proxy-authorization",
33
+ "x-api-key",
34
+ "api-key",
35
+ "x-auth-token",
36
+ "x-access-token",
37
+ }
38
+ _proxy_environment = (
39
+ "http_proxy",
40
+ "HTTP_PROXY",
41
+ "https_proxy",
42
+ "HTTPS_PROXY",
43
+ "all_proxy",
44
+ "ALL_PROXY",
45
+ )
46
+
47
+ def __init__(self, default_download_timeout: float | None = None):
48
+ self.default_download_timeout = default_download_timeout
49
+
50
+ @classmethod
51
+ def from_crawler(cls, crawler) -> LightpandaMiddleware:
52
+ """Construct the middleware using Scrapy's downloader middleware API."""
53
+ return cls(crawler.settings.getfloat("DOWNLOAD_TIMEOUT"))
54
+
55
+ async def process_request(
56
+ self, request: Request, spider: Spider | None = None
57
+ ) -> Response | None:
58
+ """Fetch opted-in GET requests asynchronously from Scrapy's perspective."""
59
+ if not request.meta.get("lightpanda"):
60
+ return None
61
+ if request.method != "GET":
62
+ raise NotSupported("LightpandaMiddleware only supports opted-in GET requests")
63
+ proxy = request.meta.get("proxy")
64
+ allow_sensitive = request.meta.get("lightpanda_allow_sensitive_headers") is True
65
+ proxy_authorization = request.headers.get(b"Proxy-Authorization")
66
+ sensitive_origin_headers = self._sensitive_headers - {"proxy-authorization"}
67
+ for name in request.headers:
68
+ normalized_name = name.decode("latin-1").lower()
69
+ if allow_sensitive and normalized_name in sensitive_origin_headers:
70
+ raise NotSupported(
71
+ "Sensitive origin headers cannot be forwarded to Lightpanda; "
72
+ "Lightpanda applies CLI headers to cross-origin subrequests"
73
+ )
74
+ if proxy_authorization is not None and not allow_sensitive:
75
+ raise NotSupported(
76
+ "Credentialed Lightpanda proxies require "
77
+ "meta['lightpanda_allow_sensitive_headers']=True"
78
+ )
79
+ if proxy and self._proxy_has_credentials(proxy) and not allow_sensitive:
80
+ raise NotSupported(
81
+ "Credentialed Lightpanda proxies require "
82
+ "meta['lightpanda_allow_sensitive_headers']=True"
83
+ )
84
+ if proxy and proxy_authorization is not None:
85
+ proxy = self._proxy_with_authorization(proxy, proxy_authorization)
86
+ self._validate_proxy_environment(request.url, proxy)
87
+ request.meta["dont_cache"] = True
88
+ return await asyncio.to_thread(self._fetch_response, request, proxy)
89
+
90
+ @classmethod
91
+ def _validate_proxy_environment(cls, url: str, proxy: str | None) -> None:
92
+ if proxy is None and any(os.environ.get(name) for name in cls._proxy_environment):
93
+ raise NotSupported(
94
+ "Lightpanda inherits proxy environment variables; clear them or "
95
+ "set an explicit Scrapy proxy to avoid proxy environment conflicts"
96
+ )
97
+ if proxy is not None and proxy_bypass(urlsplit(url).hostname or ""):
98
+ raise NotSupported(
99
+ "The no_proxy environment bypasses this request's explicit Scrapy proxy"
100
+ )
101
+
102
+ @staticmethod
103
+ def _proxy_has_credentials(proxy: str) -> bool:
104
+ proxy_parts = urlsplit(proxy)
105
+ return proxy_parts.username is not None or proxy_parts.password is not None
106
+
107
+ @staticmethod
108
+ def _proxy_with_authorization(proxy: str, proxy_authorization: bytes) -> str:
109
+ try:
110
+ scheme, encoded_credentials = proxy_authorization.split(None, 1)
111
+ if scheme.lower() != b"basic":
112
+ raise NotSupported("Only Basic proxy authorization is supported by Lightpanda")
113
+ credentials = base64.b64decode(encoded_credentials, validate=True)
114
+ username, password = credentials.split(b":", 1)
115
+ except (ValueError, binascii.Error) as exc:
116
+ raise NotSupported("Invalid Basic Proxy-Authorization header") from exc
117
+
118
+ parts = urlsplit(proxy)
119
+ user_info = f"{quote_from_bytes(username, safe='')}:{quote_from_bytes(password, safe='')}@"
120
+ host_port = parts.netloc.rsplit("@", 1)[-1]
121
+ return urlunsplit(
122
+ (parts.scheme, user_info + host_port, parts.path, parts.query, parts.fragment)
123
+ )
124
+
125
+ def _fetch_response(self, request: Request, proxy: str | None) -> HtmlResponse:
126
+ options: dict[str, str | int | list[str]] = {}
127
+ wait_until = request.meta.get("lightpanda_wait_until")
128
+ wait_selector = request.meta.get("lightpanda_wait_selector")
129
+ if wait_until is not None:
130
+ options["wait_until"] = wait_until
131
+ if wait_selector is not None:
132
+ options["wait_selector"] = wait_selector
133
+
134
+ forwarded_headers: list[str] = []
135
+ for name in request.headers:
136
+ normalized_name = name.decode("latin-1").lower()
137
+ if normalized_name in {"proxy-authorization", "referer"}:
138
+ continue
139
+ if normalized_name not in self._safe_headers:
140
+ continue
141
+ values = request.headers.getlist(name)
142
+ if normalized_name == "user-agent":
143
+ if values:
144
+ options["user_agent"] = self._decode_header_value(values[0])
145
+ continue
146
+ if values:
147
+ header_values = [self._decode_header_value(value) for value in values]
148
+ forwarded_headers.append(
149
+ f"{name.decode('ascii')}: {', '.join(header_values)}"
150
+ )
151
+ if forwarded_headers:
152
+ options["http_header"] = forwarded_headers
153
+
154
+ if proxy:
155
+ options["http_proxy"] = proxy
156
+
157
+ timeout = request.meta.get("download_timeout", self.default_download_timeout)
158
+ if timeout is not None:
159
+ timeout_ms = float(timeout) * 1000
160
+ options["http_timeout"] = max(1, math.ceil(timeout_ms)) if timeout_ms > 0 else 0
161
+
162
+ has_proxy_credentials = bool(proxy and self._proxy_has_credentials(proxy))
163
+ try:
164
+ result = lightpanda.fetch(request.url, json=True, **options).json()
165
+ except Exception as exc:
166
+ if has_proxy_credentials:
167
+ raise RuntimeError(
168
+ "Lightpanda fetch failed while using a credentialed proxy; "
169
+ "error details redacted"
170
+ ) from None
171
+ if isinstance(exc, lightpanda.Error) and "OperationTimedout" in str(exc):
172
+ raise DownloadTimeoutError("Lightpanda request timed out") from None
173
+ raise
174
+
175
+ content = result.get("content", "")
176
+ body = content if isinstance(content, bytes) else str(content).encode("utf-8")
177
+ status_value = result.get("http_status")
178
+ if not status_value:
179
+ raise DownloadTimeoutError("Lightpanda did not receive an HTTP response")
180
+ status = int(status_value)
181
+ headers = self._response_headers(result.get("headers"))
182
+
183
+ return HtmlResponse(
184
+ url=result.get("url") or request.url,
185
+ status=status,
186
+ headers=headers,
187
+ body=body,
188
+ encoding="utf-8",
189
+ request=request,
190
+ )
191
+
192
+ @staticmethod
193
+ def _decode_header_value(value: bytes) -> str:
194
+ try:
195
+ return value.decode("ascii")
196
+ except UnicodeDecodeError as exc:
197
+ raise NotSupported(
198
+ "Lightpanda CLI header values must be ASCII; non-ASCII bytes "
199
+ "cannot be preserved through subprocess arguments"
200
+ ) from exc
201
+
202
+ @staticmethod
203
+ def _response_headers(raw_headers) -> Headers:
204
+ headers = Headers()
205
+ if isinstance(raw_headers, dict):
206
+ for name, value in raw_headers.items():
207
+ values = value if isinstance(value, list) else [value]
208
+ for item in values:
209
+ headers.appendlist(name, item)
210
+ else:
211
+ for header in raw_headers or []:
212
+ headers.appendlist(header["name"], header["value"])
213
+ for name in (b"Content-Encoding", b"Content-Length", b"Transfer-Encoding"):
214
+ headers.pop(name, None)
215
+ return headers