trustpilot-scraper 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- trustpilot_scraper-1.0.0/CHANGELOG.md +44 -0
- trustpilot_scraper-1.0.0/LICENSE +21 -0
- trustpilot_scraper-1.0.0/MANIFEST.in +2 -0
- trustpilot_scraper-1.0.0/PKG-INFO +196 -0
- trustpilot_scraper-1.0.0/README.md +157 -0
- trustpilot_scraper-1.0.0/pyproject.toml +81 -0
- trustpilot_scraper-1.0.0/setup.cfg +4 -0
- trustpilot_scraper-1.0.0/tests/fixtures/challenge.html +1 -0
- trustpilot_scraper-1.0.0/tests/fixtures/page1.html +1 -0
- trustpilot_scraper-1.0.0/tests/fixtures/page2.html +1 -0
- trustpilot_scraper-1.0.0/tests/test_scraper.py +239 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper/__init__.py +35 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper/cli.py +68 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper/py.typed +0 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper/scraper.py +281 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper.egg-info/PKG-INFO +196 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper.egg-info/SOURCES.txt +19 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper.egg-info/dependency_links.txt +1 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper.egg-info/entry_points.txt +2 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper.egg-info/requires.txt +11 -0
- trustpilot_scraper-1.0.0/trustpilot_scraper.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [1.0.0] - 2026-10-31
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
- **Packaging:** the 0.10 wheel on PyPI contained no Python code, because the modules sat at
|
|
14
|
+
the repo root (`scraper.py`, `init.py`) instead of in a `trustpilot_scraper/` package. The
|
|
15
|
+
package now installs and imports properly.
|
|
16
|
+
- Errors no longer fail silently. Network errors, HTTP 403/429, Trustpilot's bot-verification
|
|
17
|
+
page and pages without review data now raise `TrustpilotError`,
|
|
18
|
+
`TrustpilotBlockedError` or `TrustpilotParseError` instead of returning `[]`.
|
|
19
|
+
- Pagination stops at `filters.pagination.totalPages` instead of requesting pages until one
|
|
20
|
+
fails.
|
|
21
|
+
- De-duplication now uses the review id and keeps page order (it used to scramble the order).
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
- `max_pages`, `delay`, `extended`, `session` and `timeout` options on
|
|
25
|
+
`scrape_trustpilot_reviews`.
|
|
26
|
+
- `iter_trustpilot_reviews` (lazy generator), `parse_reviews_html`, `parse_next_data`,
|
|
27
|
+
`normalize_review` and `normalize_url`. You can pass a bare domain such as `www.amazon.com`.
|
|
28
|
+
- `extended=True` adds `ReviewId`, `ExperienceDate`, `Verified`, `Likes`, `Language`, `Reply`
|
|
29
|
+
and `ReplyDate`.
|
|
30
|
+
- `trustpilot-scraper` CLI with JSON, JSONL and CSV output.
|
|
31
|
+
- `__version__`, `py.typed`, type hints, an offline pytest suite with HTML fixtures, and CI on
|
|
32
|
+
Python 3.10 to 3.13 with ruff.
|
|
33
|
+
- `SECURITY.md`, `CODE_OF_CONDUCT.md`, `CONTRIBUTING.md`, `CITATION.cff`, issue/PR templates and
|
|
34
|
+
Dependabot.
|
|
35
|
+
|
|
36
|
+
### Changed
|
|
37
|
+
- Python 3.10+ required. Packaging moved to `pyproject.toml` (PEP 639 license metadata) and
|
|
38
|
+
`setup.py` was removed.
|
|
39
|
+
- `pandas` is no longer required (it was only used to format dates). Install
|
|
40
|
+
`trustpilot_scraper[pandas]` if you want it.
|
|
41
|
+
|
|
42
|
+
## [0.10] - 2024-04-16
|
|
43
|
+
|
|
44
|
+
- Last 0.x release.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Md Irfan Ali
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: trustpilot_scraper
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Scrape public Trustpilot company reviews into plain Python dicts, CSV or JSON, using the page's embedded __NEXT_DATA__ JSON.
|
|
5
|
+
Author-email: Irfan Ali <irfanali29@hotmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/irfanalidv/trustpilot_scraper
|
|
8
|
+
Project-URL: Documentation, https://github.com/irfanalidv/trustpilot_scraper#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/irfanalidv/trustpilot_scraper
|
|
10
|
+
Project-URL: Issues, https://github.com/irfanalidv/trustpilot_scraper/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/irfanalidv/trustpilot_scraper/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: trustpilot,reviews,scraper,web-scraping,customer-reviews,sentiment-analysis,review-analysis,market-research,competitor-analysis,nextjs,beautifulsoup
|
|
13
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
23
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: requests>=2.28
|
|
30
|
+
Requires-Dist: beautifulsoup4>=4.11
|
|
31
|
+
Provides-Extra: pandas
|
|
32
|
+
Requires-Dist: pandas>=1.5; extra == "pandas"
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
36
|
+
Requires-Dist: build>=1.0; extra == "dev"
|
|
37
|
+
Requires-Dist: twine>=5.0; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
39
|
+
|
|
40
|
+
<div align="center">
|
|
41
|
+
|
|
42
|
+
# trustpilot-scraper
|
|
43
|
+
|
|
44
|
+
**Turn a company's public Trustpilot reviews into clean Python dicts, CSV or JSON, in one call.**
|
|
45
|
+
|
|
46
|
+
[](https://pypi.org/project/trustpilot_scraper/)
|
|
47
|
+
[](https://pypi.org/project/trustpilot_scraper/)
|
|
48
|
+
[](https://github.com/irfanalidv/trustpilot_scraper/actions/workflows/tests.yml)
|
|
49
|
+
[](https://pepy.tech/project/trustpilot_scraper)
|
|
50
|
+
[](https://github.com/irfanalidv/trustpilot_scraper/blob/main/LICENSE)
|
|
51
|
+
[](https://github.com/astral-sh/ruff)
|
|
52
|
+
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
The `trustpilot_scraper` library is a Python package for scraping Trustpilot reviews. Give it a
|
|
56
|
+
Trustpilot review page (or just the company's domain) and it returns every review as a dictionary.
|
|
57
|
+
|
|
58
|
+
Businesses can use it to collect Trustpilot reviews for market analysis, sentiment analysis and
|
|
59
|
+
competitor benchmarking, and to learn from customer feedback to improve products, services and
|
|
60
|
+
online reputation.
|
|
61
|
+
|
|
62
|
+
> **Heads-up (October 2026):** Trustpilot currently puts an AWS WAF bot-verification page in front
|
|
63
|
+
> of its review pages, and plain HTTP clients such as `requests` often get **HTTP 403**. When that
|
|
64
|
+
> happens this library raises `TrustpilotBlockedError` rather than quietly returning an empty list.
|
|
65
|
+
> The library does not try to get around the check. If you are blocked, save the page from your
|
|
66
|
+
> browser and parse it with `parse_reviews_html()` (see [below](#parse-html-you-already-have)). The
|
|
67
|
+
> page's data format was checked against a live page on 2026-10-31.
|
|
68
|
+
|
|
69
|
+
## Features
|
|
70
|
+
|
|
71
|
+
- Reads the page's embedded `__NEXT_DATA__` JSON instead of brittle CSS selectors.
|
|
72
|
+
- Handles pagination automatically, stopping at the real last page (or at `max_pages`).
|
|
73
|
+
- Retrieves date, author, body, heading, rating and location, and with `extended=True` also the
|
|
74
|
+
review id, experience date, verification flag, likes, language and the company's reply.
|
|
75
|
+
- Removes duplicate reviews (by review id) and keeps page order.
|
|
76
|
+
- Waits politely between requests (2 s by default) and lets you pass your own `requests.Session`.
|
|
77
|
+
- Raises clear exceptions when something goes wrong: `TrustpilotBlockedError`,
|
|
78
|
+
`TrustpilotParseError` and `TrustpilotError`.
|
|
79
|
+
- Includes a `trustpilot-scraper` command that writes JSON, JSONL or CSV.
|
|
80
|
+
- Fully typed (`py.typed`) and light: only `requests` and `beautifulsoup4` are required.
|
|
81
|
+
|
|
82
|
+
## Installation
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install trustpilot_scraper
|
|
86
|
+
# optional: pandas, for working with results as a DataFrame
|
|
87
|
+
pip install "trustpilot_scraper[pandas]"
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Requires Python 3.10+.
|
|
91
|
+
|
|
92
|
+
## Quick start
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from trustpilot_scraper import scrape_trustpilot_reviews
|
|
96
|
+
|
|
97
|
+
reviews = scrape_trustpilot_reviews(
|
|
98
|
+
"https://www.trustpilot.com/review/www.example.com", # or just "www.example.com"
|
|
99
|
+
max_pages=2, # omit to fetch every page
|
|
100
|
+
delay=2.0, # seconds between requests
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
for review in reviews:
|
|
104
|
+
print(review)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Each review is a dict with the same keys as earlier versions:
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
{'Date': '2026-10-29', 'Author': 'Reviewer 1', 'Body': 'Synthetic review body number 1.',
|
|
111
|
+
'Heading': 'Synthetic heading 1', 'Rating': 5, 'Location': 'US'}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
With `extended=True` each dict also has `ReviewId`, `ExperienceDate`, `Verified`, `Likes`,
|
|
115
|
+
`Language`, `Reply` and `ReplyDate`.
|
|
116
|
+
|
|
117
|
+
### Parse HTML you already have
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from pathlib import Path
|
|
121
|
+
from trustpilot_scraper import parse_reviews_html, normalize_review
|
|
122
|
+
|
|
123
|
+
html = Path("amazon_page1.html").read_text(encoding="utf-8") # saved from your browser
|
|
124
|
+
reviews = [normalize_review(r, extended=True) for r in parse_reviews_html(html)]
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Handle blocking explicitly
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from trustpilot_scraper import scrape_trustpilot_reviews, TrustpilotBlockedError
|
|
131
|
+
|
|
132
|
+
try:
|
|
133
|
+
reviews = scrape_trustpilot_reviews("www.example.com", max_pages=1)
|
|
134
|
+
except TrustpilotBlockedError as exc:
|
|
135
|
+
print("Trustpilot refused the request:", exc)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Command line
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
trustpilot-scraper www.example.com --max-pages 3 -o reviews.csv
|
|
142
|
+
trustpilot-scraper https://www.trustpilot.com/review/www.example.com --extended -o reviews.jsonl
|
|
143
|
+
trustpilot-scraper www.example.com # JSON to stdout, first page only
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
The exit code is `1` and the error goes to stderr when a request is blocked or fails.
|
|
147
|
+
|
|
148
|
+
## Use cases
|
|
149
|
+
|
|
150
|
+
- **Sentiment analysis:** feed `Body` and `Rating` into your NLP pipeline.
|
|
151
|
+
- **Competitor benchmarking:** compare rating distributions across companies.
|
|
152
|
+
- **Customer-support QA:** use `extended=True` to see which reviews got a company reply.
|
|
153
|
+
- **Dashboards:** `pandas.DataFrame(reviews)` gives a ready-to-plot table.
|
|
154
|
+
|
|
155
|
+
## API overview
|
|
156
|
+
|
|
157
|
+
| Function | Purpose |
|
|
158
|
+
| --- | --- |
|
|
159
|
+
| `scrape_trustpilot_reviews(base_url, *, max_pages=None, delay=2.0, extended=False, session=None, timeout=30.0)` | Fetch all (or `max_pages`) pages and return a list of dicts. |
|
|
160
|
+
| `iter_trustpilot_reviews(...)` | The same, but as a lazy generator that fetches page by page. |
|
|
161
|
+
| `get_reviews_from_page(url, *, session=None, timeout=30.0)` | Fetch one page and return the raw Trustpilot review objects. |
|
|
162
|
+
| `parse_reviews_html(html)` | Pull the raw review objects out of an HTML string. |
|
|
163
|
+
| `parse_next_data(html)` | Return the whole decoded `__NEXT_DATA__` JSON. |
|
|
164
|
+
| `normalize_review(raw, *, extended=False)` | Turn a raw review object into the flat dict shown above. |
|
|
165
|
+
| `normalize_url(company)` | `"www.example.com"` becomes `"https://www.trustpilot.com/review/www.example.com"`. |
|
|
166
|
+
|
|
167
|
+
Exceptions: `TrustpilotError` (base class), `TrustpilotBlockedError` (HTTP 403/429 or a
|
|
168
|
+
verification page) and `TrustpilotParseError` (the page has no review data).
|
|
169
|
+
|
|
170
|
+
### Upgrading from 0.x
|
|
171
|
+
|
|
172
|
+
- `from trustpilot_scraper.scraper import scrape_trustpilot_reviews` still works, and the output
|
|
173
|
+
keys are unchanged.
|
|
174
|
+
- Version 0.10 on PyPI shipped a wheel without the Python package in it. 1.0.0 fixes the packaging.
|
|
175
|
+
- Errors now raise exceptions instead of returning `[]`.
|
|
176
|
+
- `pandas` is no longer a required dependency.
|
|
177
|
+
|
|
178
|
+
## Responsible use
|
|
179
|
+
|
|
180
|
+
Only scrape public pages, keep the request rate low (the default delay is 2 s), and follow
|
|
181
|
+
Trustpilot's terms of use and the data-protection law that applies to you.
|
|
182
|
+
|
|
183
|
+
## Contributing
|
|
184
|
+
|
|
185
|
+
Pull requests are welcome. See [CONTRIBUTING.md](https://github.com/irfanalidv/trustpilot_scraper/blob/main/CONTRIBUTING.md).
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
pip install -e ".[dev]"
|
|
189
|
+
ruff check . && ruff format --check . && pytest
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
The tests run offline against saved HTML fixtures in `tests/fixtures/`.
|
|
193
|
+
|
|
194
|
+
## License
|
|
195
|
+
|
|
196
|
+
MIT. See [LICENSE](https://github.com/irfanalidv/trustpilot_scraper/blob/main/LICENSE).
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# trustpilot-scraper
|
|
4
|
+
|
|
5
|
+
**Turn a company's public Trustpilot reviews into clean Python dicts, CSV or JSON, in one call.**
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/trustpilot_scraper/)
|
|
8
|
+
[](https://pypi.org/project/trustpilot_scraper/)
|
|
9
|
+
[](https://github.com/irfanalidv/trustpilot_scraper/actions/workflows/tests.yml)
|
|
10
|
+
[](https://pepy.tech/project/trustpilot_scraper)
|
|
11
|
+
[](https://github.com/irfanalidv/trustpilot_scraper/blob/main/LICENSE)
|
|
12
|
+
[](https://github.com/astral-sh/ruff)
|
|
13
|
+
|
|
14
|
+
</div>
|
|
15
|
+
|
|
16
|
+
The `trustpilot_scraper` library is a Python package for scraping Trustpilot reviews. Give it a
|
|
17
|
+
Trustpilot review page (or just the company's domain) and it returns every review as a dictionary.
|
|
18
|
+
|
|
19
|
+
Businesses can use it to collect Trustpilot reviews for market analysis, sentiment analysis and
|
|
20
|
+
competitor benchmarking, and to learn from customer feedback to improve products, services and
|
|
21
|
+
online reputation.
|
|
22
|
+
|
|
23
|
+
> **Heads-up (October 2026):** Trustpilot currently puts an AWS WAF bot-verification page in front
|
|
24
|
+
> of its review pages, and plain HTTP clients such as `requests` often get **HTTP 403**. When that
|
|
25
|
+
> happens this library raises `TrustpilotBlockedError` rather than quietly returning an empty list.
|
|
26
|
+
> The library does not try to get around the check. If you are blocked, save the page from your
|
|
27
|
+
> browser and parse it with `parse_reviews_html()` (see [below](#parse-html-you-already-have)). The
|
|
28
|
+
> page's data format was checked against a live page on 2026-10-31.
|
|
29
|
+
|
|
30
|
+
## Features
|
|
31
|
+
|
|
32
|
+
- Reads the page's embedded `__NEXT_DATA__` JSON instead of brittle CSS selectors.
|
|
33
|
+
- Handles pagination automatically, stopping at the real last page (or at `max_pages`).
|
|
34
|
+
- Retrieves date, author, body, heading, rating and location, and with `extended=True` also the
|
|
35
|
+
review id, experience date, verification flag, likes, language and the company's reply.
|
|
36
|
+
- Removes duplicate reviews (by review id) and keeps page order.
|
|
37
|
+
- Waits politely between requests (2 s by default) and lets you pass your own `requests.Session`.
|
|
38
|
+
- Raises clear exceptions when something goes wrong: `TrustpilotBlockedError`,
|
|
39
|
+
`TrustpilotParseError` and `TrustpilotError`.
|
|
40
|
+
- Includes a `trustpilot-scraper` command that writes JSON, JSONL or CSV.
|
|
41
|
+
- Fully typed (`py.typed`) and light: only `requests` and `beautifulsoup4` are required.
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install trustpilot_scraper
|
|
47
|
+
# optional: pandas, for working with results as a DataFrame
|
|
48
|
+
pip install "trustpilot_scraper[pandas]"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Requires Python 3.10+.
|
|
52
|
+
|
|
53
|
+
## Quick start
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from trustpilot_scraper import scrape_trustpilot_reviews
|
|
57
|
+
|
|
58
|
+
reviews = scrape_trustpilot_reviews(
|
|
59
|
+
"https://www.trustpilot.com/review/www.example.com", # or just "www.example.com"
|
|
60
|
+
max_pages=2, # omit to fetch every page
|
|
61
|
+
delay=2.0, # seconds between requests
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
for review in reviews:
|
|
65
|
+
print(review)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Each review is a dict with the same keys as earlier versions:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
{'Date': '2026-10-29', 'Author': 'Reviewer 1', 'Body': 'Synthetic review body number 1.',
|
|
72
|
+
'Heading': 'Synthetic heading 1', 'Rating': 5, 'Location': 'US'}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
With `extended=True` each dict also has `ReviewId`, `ExperienceDate`, `Verified`, `Likes`,
|
|
76
|
+
`Language`, `Reply` and `ReplyDate`.
|
|
77
|
+
|
|
78
|
+
### Parse HTML you already have
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
from pathlib import Path
|
|
82
|
+
from trustpilot_scraper import parse_reviews_html, normalize_review
|
|
83
|
+
|
|
84
|
+
html = Path("amazon_page1.html").read_text(encoding="utf-8") # saved from your browser
|
|
85
|
+
reviews = [normalize_review(r, extended=True) for r in parse_reviews_html(html)]
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### Handle blocking explicitly
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from trustpilot_scraper import scrape_trustpilot_reviews, TrustpilotBlockedError
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
reviews = scrape_trustpilot_reviews("www.example.com", max_pages=1)
|
|
95
|
+
except TrustpilotBlockedError as exc:
|
|
96
|
+
print("Trustpilot refused the request:", exc)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### Command line
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
trustpilot-scraper www.example.com --max-pages 3 -o reviews.csv
|
|
103
|
+
trustpilot-scraper https://www.trustpilot.com/review/www.example.com --extended -o reviews.jsonl
|
|
104
|
+
trustpilot-scraper www.example.com # JSON to stdout, first page only
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The exit code is `1` and the error goes to stderr when a request is blocked or fails.
|
|
108
|
+
|
|
109
|
+
## Use cases
|
|
110
|
+
|
|
111
|
+
- **Sentiment analysis:** feed `Body` and `Rating` into your NLP pipeline.
|
|
112
|
+
- **Competitor benchmarking:** compare rating distributions across companies.
|
|
113
|
+
- **Customer-support QA:** use `extended=True` to see which reviews got a company reply.
|
|
114
|
+
- **Dashboards:** `pandas.DataFrame(reviews)` gives a ready-to-plot table.
|
|
115
|
+
|
|
116
|
+
## API overview
|
|
117
|
+
|
|
118
|
+
| Function | Purpose |
|
|
119
|
+
| --- | --- |
|
|
120
|
+
| `scrape_trustpilot_reviews(base_url, *, max_pages=None, delay=2.0, extended=False, session=None, timeout=30.0)` | Fetch all (or `max_pages`) pages and return a list of dicts. |
|
|
121
|
+
| `iter_trustpilot_reviews(...)` | The same, but as a lazy generator that fetches page by page. |
|
|
122
|
+
| `get_reviews_from_page(url, *, session=None, timeout=30.0)` | Fetch one page and return the raw Trustpilot review objects. |
|
|
123
|
+
| `parse_reviews_html(html)` | Pull the raw review objects out of an HTML string. |
|
|
124
|
+
| `parse_next_data(html)` | Return the whole decoded `__NEXT_DATA__` JSON. |
|
|
125
|
+
| `normalize_review(raw, *, extended=False)` | Turn a raw review object into the flat dict shown above. |
|
|
126
|
+
| `normalize_url(company)` | `"www.example.com"` becomes `"https://www.trustpilot.com/review/www.example.com"`. |
|
|
127
|
+
|
|
128
|
+
Exceptions: `TrustpilotError` (base class), `TrustpilotBlockedError` (HTTP 403/429 or a
|
|
129
|
+
verification page) and `TrustpilotParseError` (the page has no review data).
|
|
130
|
+
|
|
131
|
+
### Upgrading from 0.x
|
|
132
|
+
|
|
133
|
+
- `from trustpilot_scraper.scraper import scrape_trustpilot_reviews` still works, and the output
|
|
134
|
+
keys are unchanged.
|
|
135
|
+
- Version 0.10 on PyPI shipped a wheel without the Python package in it. 1.0.0 fixes the packaging.
|
|
136
|
+
- Errors now raise exceptions instead of returning `[]`.
|
|
137
|
+
- `pandas` is no longer a required dependency.
|
|
138
|
+
|
|
139
|
+
## Responsible use
|
|
140
|
+
|
|
141
|
+
Only scrape public pages, keep the request rate low (the default delay is 2 s), and follow
|
|
142
|
+
Trustpilot's terms of use and the data-protection law that applies to you.
|
|
143
|
+
|
|
144
|
+
## Contributing
|
|
145
|
+
|
|
146
|
+
Pull requests are welcome. See [CONTRIBUTING.md](https://github.com/irfanalidv/trustpilot_scraper/blob/main/CONTRIBUTING.md).
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
pip install -e ".[dev]"
|
|
150
|
+
ruff check . && ruff format --check . && pytest
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The tests run offline against saved HTML fixtures in `tests/fixtures/`.
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT. See [LICENSE](https://github.com/irfanalidv/trustpilot_scraper/blob/main/LICENSE).
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "trustpilot_scraper"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Scrape public Trustpilot company reviews into plain Python dicts, CSV or JSON, using the page's embedded __NEXT_DATA__ JSON."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Irfan Ali", email = "irfanali29@hotmail.com" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"trustpilot",
|
|
16
|
+
"reviews",
|
|
17
|
+
"scraper",
|
|
18
|
+
"web-scraping",
|
|
19
|
+
"customer-reviews",
|
|
20
|
+
"sentiment-analysis",
|
|
21
|
+
"review-analysis",
|
|
22
|
+
"market-research",
|
|
23
|
+
"competitor-analysis",
|
|
24
|
+
"nextjs",
|
|
25
|
+
"beautifulsoup",
|
|
26
|
+
]
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Development Status :: 5 - Production/Stable",
|
|
29
|
+
"Intended Audience :: Developers",
|
|
30
|
+
"Intended Audience :: Science/Research",
|
|
31
|
+
"Operating System :: OS Independent",
|
|
32
|
+
"Programming Language :: Python :: 3",
|
|
33
|
+
"Programming Language :: Python :: 3.10",
|
|
34
|
+
"Programming Language :: Python :: 3.11",
|
|
35
|
+
"Programming Language :: Python :: 3.12",
|
|
36
|
+
"Programming Language :: Python :: 3.13",
|
|
37
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
38
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
39
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
40
|
+
"Typing :: Typed",
|
|
41
|
+
]
|
|
42
|
+
dependencies = [
|
|
43
|
+
"requests>=2.28",
|
|
44
|
+
"beautifulsoup4>=4.11",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[project.optional-dependencies]
|
|
48
|
+
pandas = ["pandas>=1.5"]
|
|
49
|
+
dev = [
|
|
50
|
+
"pytest>=7.4",
|
|
51
|
+
"ruff>=0.6",
|
|
52
|
+
"build>=1.0",
|
|
53
|
+
"twine>=5.0",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[project.scripts]
|
|
57
|
+
trustpilot-scraper = "trustpilot_scraper.cli:main"
|
|
58
|
+
|
|
59
|
+
[project.urls]
|
|
60
|
+
Homepage = "https://github.com/irfanalidv/trustpilot_scraper"
|
|
61
|
+
Documentation = "https://github.com/irfanalidv/trustpilot_scraper#readme"
|
|
62
|
+
Repository = "https://github.com/irfanalidv/trustpilot_scraper"
|
|
63
|
+
Issues = "https://github.com/irfanalidv/trustpilot_scraper/issues"
|
|
64
|
+
Changelog = "https://github.com/irfanalidv/trustpilot_scraper/blob/main/CHANGELOG.md"
|
|
65
|
+
|
|
66
|
+
[tool.setuptools.packages.find]
|
|
67
|
+
include = ["trustpilot_scraper*"]
|
|
68
|
+
|
|
69
|
+
[tool.setuptools.package-data]
|
|
70
|
+
trustpilot_scraper = ["py.typed"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff]
|
|
73
|
+
target-version = "py310"
|
|
74
|
+
line-length = 100
|
|
75
|
+
extend-exclude = ["*.md"]
|
|
76
|
+
|
|
77
|
+
[tool.ruff.lint]
|
|
78
|
+
select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
|
|
79
|
+
|
|
80
|
+
[tool.pytest.ini_options]
|
|
81
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<!doctype html><html lang="en"><head><meta charset="UTF-8" /><title>Verifying Connection</title><script src="https://example.edge.sdk.awswaf.com/x/y/challenge.js" defer></script></head><body><h1>Verifying your connection...</h1></body></html>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<!DOCTYPE html><html lang="en"><head><title>Example Reviews</title></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props": {"pageProps": {"pageUrl": "https://www.trustpilot.com/review/www.example.com", "businessUnit": {"id": "bu1", "displayName": "Example", "identifyingName": "www.example.com", "numberOfReviews": 5, "trustScore": 3.5, "stars": 3.5}, "reviews": [{"id": "rev0001", "filtered": false, "isPending": false, "text": "Synthetic review body number 1.", "rating": 5, "labels": {"merged": null, "verification": {"isVerified": true, "createdDateTime": "2026-10-29T10:00:00.000Z", "reviewSourceName": "Organic", "verificationSource": "invitation", "verificationLevel": "verified", "hasDachExclusion": false}}, "title": "Synthetic heading 1", "likes": 1, "source": "Organic", "dates": {"experiencedDate": "2026-10-28T00:00:00.000Z", "publishedDate": "2026-10-29T10:00:00.000Z", "updatedDate": null, "submittedDate": null}, "report": null, "hasUnhandledReports": false, "isCurrentlyUnderModeration": false, "consumer": {"id": "c1", "displayName": "Reviewer 1", "imageUrl": "", "numberOfReviews": 1, "countryCode": "US", "hasImage": false, "isVerified": false}, "reply": null, "consumersReviewCountOnSameDomain": 1, "consumersReviewCountOnSameLocation": null, "productReviews": [], "language": "en", "location": null}, {"id": "rev0002", "filtered": false, "isPending": false, "text": "Synthetic review body number 2.", "rating": 1, "labels": {"merged": null, "verification": {"isVerified": false, "createdDateTime": "2026-10-28T10:00:00.000Z", "reviewSourceName": "Organic", "verificationSource": "invitation", "verificationLevel": "not-verified", "hasDachExclusion": false}}, "title": "Synthetic heading 2", "likes": 2, "source": "Organic", "dates": {"experiencedDate": "2026-10-27T00:00:00.000Z", "publishedDate": "2026-10-28T10:00:00.000Z", "updatedDate": null, "submittedDate": null}, "report": null, "hasUnhandledReports": false, "isCurrentlyUnderModeration": false, "consumer": {"id": "c2", "displayName": "Reviewer 2", "imageUrl": "", "numberOfReviews": 1, "countryCode": "GB", "hasImage": false, "isVerified": false}, "reply": {"id": "rp1", "message": "Sorry to hear that.", "publishedDate": "2026-10-29T12:00:00.000Z", "updatedDate": null}, "consumersReviewCountOnSameDomain": 1, "consumersReviewCountOnSameLocation": null, "productReviews": [], "language": "en", "location": null}, {"id": "rev0003", "filtered": false, "isPending": false, "text": "Synthetic review body number 3.", "rating": 4, "labels": {"merged": null, "verification": {"isVerified": false, "createdDateTime": "2026-10-27T10:00:00.000Z", "reviewSourceName": "Organic", "verificationSource": "invitation", "verificationLevel": "not-verified", "hasDachExclusion": false}}, "title": "Synthetic heading 3", "likes": 3, "source": "Organic", "dates": {"experiencedDate": "2026-10-26T00:00:00.000Z", "publishedDate": "2026-10-27T10:00:00.000Z", "updatedDate": null, "submittedDate": null}, "report": null, "hasUnhandledReports": false, "isCurrentlyUnderModeration": false, "consumer": {"id": "c3", "displayName": "Reviewer 3", "imageUrl": "", "numberOfReviews": 1, "countryCode": "DE", "hasImage": false, "isVerified": false}, "reply": null, "consumersReviewCountOnSameDomain": 1, "consumersReviewCountOnSameLocation": null, "productReviews": [], "language": "en", "location": null}], "filters": {"hasActiveFilters": false, "totalNumberOfReviews": 5, "totalNumberOfFilteredReviews": 5, "pagination": {"currentPage": 1, "perPage": 3, "totalCount": 5, "totalPages": 2}}}}, "page": "/review/[businessUnit]", "query": {"businessUnit": "www.example.com"}, "buildId": "businessunitprofile-consumersite-2.8396.0"}</script></body></html>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<!DOCTYPE html><html lang="en"><head><title>Example Reviews</title></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props": {"pageProps": {"pageUrl": "https://www.trustpilot.com/review/www.example.com", "businessUnit": {"id": "bu1", "displayName": "Example", "identifyingName": "www.example.com", "numberOfReviews": 5, "trustScore": 3.5, "stars": 3.5}, "reviews": [{"id": "rev0003", "filtered": false, "isPending": false, "text": "Synthetic review body number 3.", "rating": 4, "labels": {"merged": null, "verification": {"isVerified": false, "createdDateTime": "2026-10-27T10:00:00.000Z", "reviewSourceName": "Organic", "verificationSource": "invitation", "verificationLevel": "not-verified", "hasDachExclusion": false}}, "title": "Synthetic heading 3", "likes": 3, "source": "Organic", "dates": {"experiencedDate": "2026-10-26T00:00:00.000Z", "publishedDate": "2026-10-27T10:00:00.000Z", "updatedDate": null, "submittedDate": null}, "report": null, "hasUnhandledReports": false, "isCurrentlyUnderModeration": false, "consumer": {"id": "c3", "displayName": "Reviewer 3", "imageUrl": "", "numberOfReviews": 1, "countryCode": "DE", "hasImage": false, "isVerified": false}, "reply": null, "consumersReviewCountOnSameDomain": 1, "consumersReviewCountOnSameLocation": null, "productReviews": [], "language": "en", "location": null}, {"id": "rev0004", "filtered": false, "isPending": false, "text": "Synthetic review body number 4.", "rating": 2, "labels": {"merged": null, "verification": {"isVerified": false, "createdDateTime": "2026-10-26T10:00:00.000Z", "reviewSourceName": "Organic", "verificationSource": "invitation", "verificationLevel": "not-verified", "hasDachExclusion": false}}, "title": "Synthetic heading 4", "likes": 4, "source": "Organic", "dates": {"experiencedDate": "2026-10-25T00:00:00.000Z", "publishedDate": "2026-10-26T10:00:00.000Z", "updatedDate": null, "submittedDate": null}, "report": null, "hasUnhandledReports": false, "isCurrentlyUnderModeration": false, "consumer": {"id": "c4", "displayName": "Reviewer 4", "imageUrl": "", "numberOfReviews": 1, "countryCode": "IN", "hasImage": false, "isVerified": false}, "reply": null, "consumersReviewCountOnSameDomain": 1, "consumersReviewCountOnSameLocation": null, "productReviews": [], "language": "en", "location": null}, {"id": "rev0005", "filtered": false, "isPending": false, "text": "Synthetic review body number 5.", "rating": 3, "labels": {"merged": null, "verification": {"isVerified": false, "createdDateTime": "2026-10-25T10:00:00.000Z", "reviewSourceName": "Organic", "verificationSource": "invitation", "verificationLevel": "not-verified", "hasDachExclusion": false}}, "title": "Synthetic heading 5", "likes": 5, "source": "Organic", "dates": {"experiencedDate": "2026-10-24T00:00:00.000Z", "publishedDate": "2026-10-25T10:00:00.000Z", "updatedDate": null, "submittedDate": null}, "report": null, "hasUnhandledReports": false, "isCurrentlyUnderModeration": false, "consumer": {"id": "c5", "displayName": "Reviewer 5", "imageUrl": "", "numberOfReviews": 1, "countryCode": "AU", "hasImage": false, "isVerified": false}, "reply": null, "consumersReviewCountOnSameDomain": 1, "consumersReviewCountOnSameLocation": null, "productReviews": [], "language": "en", "location": null}], "filters": {"hasActiveFilters": false, "totalNumberOfReviews": 5, "totalNumberOfFilteredReviews": 5, "pagination": {"currentPage": 2, "perPage": 3, "totalCount": 5, "totalPages": 2}}}}, "page": "/review/[businessUnit]", "query": {"businessUnit": "www.example.com"}, "buildId": "businessunitprofile-consumersite-2.8396.0"}</script></body></html>
|