brightdata-amazon-scraper 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amazon_scraper/__init__.py +15 -0
- amazon_scraper/__main__.py +79 -0
- amazon_scraper/scrape.py +183 -0
- brightdata_amazon_scraper-0.1.0.dist-info/METADATA +540 -0
- brightdata_amazon_scraper-0.1.0.dist-info/RECORD +8 -0
- brightdata_amazon_scraper-0.1.0.dist-info/WHEEL +4 -0
- brightdata_amazon_scraper-0.1.0.dist-info/entry_points.txt +2 -0
- brightdata_amazon_scraper-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""An Amazon scraper on Bright Data's Scraper API."""
|
|
2
|
+
|
|
3
|
+
from .scrape import Outcome, attribute, clean_asin, product_url, row_asin, rows, scrape, write
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"Outcome",
|
|
7
|
+
"attribute",
|
|
8
|
+
"clean_asin",
|
|
9
|
+
"product_url",
|
|
10
|
+
"row_asin",
|
|
11
|
+
"rows",
|
|
12
|
+
"scrape",
|
|
13
|
+
"write",
|
|
14
|
+
]
|
|
15
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""amazon-scraper B0CRMZHDG8 B085DVHQ57, or python -m amazon_scraper ..."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from brightdata import BrightDataError
|
|
9
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
|
|
10
|
+
|
|
11
|
+
from .scrape import Outcome, clean_asin, client_context, fetch, write
|
|
12
|
+
|
|
13
|
+
#: The SDK's own advice lists a Python parameter. A command has no such thing.
|
|
14
|
+
NO_TOKEN = (
|
|
15
|
+
"API token required but not found.\n"
|
|
16
|
+
" export BRIGHTDATA_API_TOKEN=YOUR_API_KEY token: https://brightdata.com/cp/setting/users\n"
|
|
17
|
+
" or run once: npx -p @brightdata/cli bdata login"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
#: A spinner and a running clock, so a minute of waiting looks alive.
|
|
21
|
+
WAITING = (SpinnerColumn(), TextColumn("{task.description}"), TimeElapsedColumn())
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(argv: list[str] | None = None) -> int:
|
|
25
|
+
parser = argparse.ArgumentParser(
|
|
26
|
+
prog="amazon-scraper",
|
|
27
|
+
description="Scrape Amazon products by ASIN, or by a product URL carrying one.",
|
|
28
|
+
)
|
|
29
|
+
parser.add_argument("asins", nargs="+", help="ASINs such as B0CRMZHDG8, or product URLs")
|
|
30
|
+
parser.add_argument("--out", default="amazon.json", help="output file, default %(default)s")
|
|
31
|
+
args = parser.parse_args(argv)
|
|
32
|
+
|
|
33
|
+
count = len(args.asins)
|
|
34
|
+
plural = "" if count == 1 else "s"
|
|
35
|
+
print(
|
|
36
|
+
f"Fetching {count} Amazon product{plural}: {', '.join(args.asins)}\n"
|
|
37
|
+
"One job for all of them. One credit per product.",
|
|
38
|
+
flush=True, # piped, an unflushed header would print after the error below
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
asins, outcomes_by_input = [], {}
|
|
42
|
+
for value in args.asins:
|
|
43
|
+
try:
|
|
44
|
+
asins.append(clean_asin(value))
|
|
45
|
+
except ValueError as exc:
|
|
46
|
+
outcomes_by_input[value] = str(exc)
|
|
47
|
+
|
|
48
|
+
found = {}
|
|
49
|
+
try:
|
|
50
|
+
with client_context() as client, Progress(*WAITING, transient=True) as bar:
|
|
51
|
+
label = f"{len(asins)} product{'' if len(asins) == 1 else 's'}"
|
|
52
|
+
if not bar.console.is_terminal:
|
|
53
|
+
print(f"asking {label}...", flush=True) # a log wants a line, not a spinner
|
|
54
|
+
task = bar.add_task(label)
|
|
55
|
+
for outcome in fetch(client, list(dict.fromkeys(asins))):
|
|
56
|
+
found[outcome.asin] = outcome
|
|
57
|
+
bar.remove_task(task)
|
|
58
|
+
except BrightDataError as exc:
|
|
59
|
+
# Almost always a missing token, which reads as a crash under a traceback.
|
|
60
|
+
print(NO_TOKEN if "token required" in str(exc) else exc, file=sys.stderr)
|
|
61
|
+
return 2
|
|
62
|
+
|
|
63
|
+
outcomes = []
|
|
64
|
+
for value in args.asins:
|
|
65
|
+
if value in outcomes_by_input:
|
|
66
|
+
outcomes.append(Outcome(value, error=outcomes_by_input[value]))
|
|
67
|
+
else:
|
|
68
|
+
outcomes.append(found[clean_asin(value)])
|
|
69
|
+
for outcome in outcomes:
|
|
70
|
+
print(outcome.line())
|
|
71
|
+
|
|
72
|
+
path = write(outcomes, args.out)
|
|
73
|
+
got = sum(1 for o in outcomes if o.ok)
|
|
74
|
+
print(f"\nSaved {got} of {len(outcomes)} products as JSON to {path}")
|
|
75
|
+
return 0 if all(o.ok for o in outcomes) else 1
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
sys.exit(main())
|
amazon_scraper/scrape.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Get Amazon products by ASIN, or by the URL that carries one.
|
|
2
|
+
|
|
3
|
+
ASINs -> scrape.amazon.products(urls) -> rows -> matched back by ASIN
|
|
4
|
+
|
|
5
|
+
Every ASIN asked for goes into one job. The API bills per record, not per job.
|
|
6
|
+
|
|
7
|
+
Rows are matched to inputs on the row's own `asin`, never on position. With a
|
|
8
|
+
list of URLs the SDK pairs rows to inputs by index, and the API returns them in
|
|
9
|
+
a different order each run, so the SDK's `ScrapeResult.url` can name one
|
|
10
|
+
product while `data` holds another (sdk-python#60).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
from collections.abc import Iterable
|
|
18
|
+
from contextlib import nullcontext
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from brightdata import SyncBrightDataClient
|
|
25
|
+
|
|
26
|
+
#: Seconds, the SDK's own unit and its default for Amazon. The Node twin counts in ms.
|
|
27
|
+
TIMEOUT = 240
|
|
28
|
+
|
|
29
|
+
#: Which Amazon site the ASINs are read from.
|
|
30
|
+
DOMAIN = "https://www.amazon.com"
|
|
31
|
+
|
|
32
|
+
_ASIN = re.compile(r"^[A-Z0-9]{10}$", re.I)
|
|
33
|
+
_ASIN_IN_PATH = re.compile(r"/(?:dp|gp/product|gp/aw/d)/([A-Z0-9]{10})", re.I)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def clean_asin(value: str) -> str:
|
|
37
|
+
"""Accept B0CRMZHDG8, a /dp/ URL, a /gp/product/ URL, or one with a query."""
|
|
38
|
+
raw = (value or "").strip()
|
|
39
|
+
found = _ASIN_IN_PATH.search(raw)
|
|
40
|
+
candidate = found.group(1) if found else raw.split("?", 1)[0].split("#", 1)[0].rstrip("/")
|
|
41
|
+
if not _ASIN.match(candidate):
|
|
42
|
+
raise ValueError(f"{value!r} is not an Amazon ASIN")
|
|
43
|
+
return candidate.upper()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def product_url(value: str) -> str:
|
|
47
|
+
return f"{DOMAIN}/dp/{clean_asin(value)}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def rows(result: Any) -> list[dict[str, Any]]:
|
|
51
|
+
"""Flatten a ScrapeResult, or a list of them, into plain dicts."""
|
|
52
|
+
if isinstance(result, list):
|
|
53
|
+
return [row for item in result for row in rows(item)]
|
|
54
|
+
data = getattr(result, "data", result)
|
|
55
|
+
if isinstance(data, dict):
|
|
56
|
+
data = [data]
|
|
57
|
+
if not isinstance(data, list):
|
|
58
|
+
return []
|
|
59
|
+
return [row for row in data if isinstance(row, dict)]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def envelope_error(result: Any) -> str | None:
|
|
63
|
+
"""A failed or timed out request carries no rows to explain itself.
|
|
64
|
+
|
|
65
|
+
On failure the SDK returns one ScrapeResult even for list input, so this is
|
|
66
|
+
the only shape that can say the whole job died.
|
|
67
|
+
"""
|
|
68
|
+
if isinstance(result, list) or getattr(result, "success", True):
|
|
69
|
+
return None
|
|
70
|
+
return str(getattr(result, "error", None) or getattr(result, "status", "request failed"))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def row_asin(row: dict[str, Any]) -> str:
|
|
74
|
+
"""The ASIN a row answers for: its asin field, else the one in its URL."""
|
|
75
|
+
direct = row.get("asin")
|
|
76
|
+
if isinstance(direct, str) and _ASIN.match(direct):
|
|
77
|
+
return direct.upper()
|
|
78
|
+
source = row.get("input_url") or (row.get("input") or {}).get("url") or row.get("url") or ""
|
|
79
|
+
found = _ASIN_IN_PATH.search(str(source))
|
|
80
|
+
return found.group(1).upper() if found else ""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class Outcome:
|
|
85
|
+
"""What happened to one ASIN."""
|
|
86
|
+
|
|
87
|
+
asin: str
|
|
88
|
+
product: dict[str, Any] | None = None
|
|
89
|
+
error: str | None = None
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def ok(self) -> bool:
|
|
93
|
+
return self.error is None
|
|
94
|
+
|
|
95
|
+
def line(self) -> str:
|
|
96
|
+
"""One line a reader can understand without having read the source."""
|
|
97
|
+
if not self.ok:
|
|
98
|
+
return f"failed {self.asin}: {self.error}"
|
|
99
|
+
title = str((self.product or {}).get("title") or "")[:48]
|
|
100
|
+
suffix = f" ({title})" if title else ""
|
|
101
|
+
return f"got {self.asin}: {len(self.product or {})} fields{suffix}"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def attribute(asins: list[str], found: list[dict[str, Any]]) -> list[Outcome]:
|
|
105
|
+
"""Match each row to the ASIN that asked for it, by the row's own identity."""
|
|
106
|
+
by_asin: dict[str, dict[str, Any]] = {}
|
|
107
|
+
for row in found:
|
|
108
|
+
asin = row_asin(row)
|
|
109
|
+
if asin in asins and asin not in by_asin:
|
|
110
|
+
by_asin[asin] = row
|
|
111
|
+
|
|
112
|
+
outcomes = []
|
|
113
|
+
for asin in asins:
|
|
114
|
+
row = by_asin.get(asin)
|
|
115
|
+
if row is None:
|
|
116
|
+
outcomes.append(Outcome(asin, error="the API returned no row for this ASIN"))
|
|
117
|
+
elif row.get("error"):
|
|
118
|
+
outcomes.append(Outcome(asin, error=str(row["error"])))
|
|
119
|
+
else:
|
|
120
|
+
outcomes.append(Outcome(asin, product=row))
|
|
121
|
+
return outcomes
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def client_context(client: Any = None) -> Any:
|
|
125
|
+
"""The client passed in, or one we open and own.
|
|
126
|
+
|
|
127
|
+
SyncBrightDataClient builds its event loop in __enter__, so it must be
|
|
128
|
+
entered. An unentered client fails with an AttributeError about
|
|
129
|
+
run_until_complete.
|
|
130
|
+
"""
|
|
131
|
+
# auto_create_zones defaults to True: the SDK tries to create Web Unlocker
|
|
132
|
+
# and SERP zones on startup, which this scraper never uses. Creating a zone
|
|
133
|
+
# needs a payment method, so leaving it on breaks the first run for free
|
|
134
|
+
# accounts.
|
|
135
|
+
return nullcontext(client) if client is not None else SyncBrightDataClient(
|
|
136
|
+
auto_create_zones=False
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def fetch(client: Any, asins: list[str], timeout: int = TIMEOUT) -> list[Outcome]:
|
|
141
|
+
"""One job for every ASIN. Never raises: a failure becomes Outcomes."""
|
|
142
|
+
if not asins:
|
|
143
|
+
return []
|
|
144
|
+
try:
|
|
145
|
+
# A list, even of one, so the SDK never collapses the answer to a dict.
|
|
146
|
+
result = client.scrape.amazon.products([f"{DOMAIN}/dp/{a}" for a in asins], timeout=timeout)
|
|
147
|
+
failed = envelope_error(result)
|
|
148
|
+
if failed:
|
|
149
|
+
return [Outcome(asin, error=failed) for asin in asins]
|
|
150
|
+
return attribute(asins, rows(result))
|
|
151
|
+
except Exception as exc: # one bad batch must not end the run with a traceback
|
|
152
|
+
return [Outcome(asin, error=f"{type(exc).__name__}: {exc}") for asin in asins]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def scrape(values: Iterable[str], client: Any = None) -> list[Outcome]:
|
|
156
|
+
"""One Outcome per input, in the order asked. A repeated ASIN is fetched once."""
|
|
157
|
+
values = list(values)
|
|
158
|
+
asins: list[str] = []
|
|
159
|
+
rejected: dict[str, Outcome] = {}
|
|
160
|
+
for value in values:
|
|
161
|
+
try:
|
|
162
|
+
asins.append(clean_asin(value))
|
|
163
|
+
except ValueError as exc:
|
|
164
|
+
rejected[str(value)] = Outcome(str(value), error=str(exc))
|
|
165
|
+
|
|
166
|
+
found: dict[str, Outcome] = {}
|
|
167
|
+
if asins:
|
|
168
|
+
with client_context(client) as opened:
|
|
169
|
+
found = {o.asin: o for o in fetch(opened, list(dict.fromkeys(asins)))}
|
|
170
|
+
|
|
171
|
+
return [rejected[str(v)] if str(v) in rejected else found[clean_asin(v)] for v in values]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def write(outcomes: list[Outcome], path: str | Path) -> Path:
|
|
175
|
+
"""Write one JSON file: when it ran, and the product found per ASIN."""
|
|
176
|
+
document = {
|
|
177
|
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
178
|
+
"products": [{"asin": o.asin, "product": o.product} for o in outcomes],
|
|
179
|
+
}
|
|
180
|
+
target = Path(path)
|
|
181
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
182
|
+
target.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8")
|
|
183
|
+
return target
|
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: brightdata-amazon-scraper
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Amazon scraper in Python: products, reviews, sellers and search as JSON, on the Bright Data Amazon Scraper API (Web Scraper API). No login, no browser. 5,000 free credits a month.
|
|
5
|
+
Project-URL: Homepage, https://github.com/brightdata/amazon-scraper
|
|
6
|
+
Project-URL: Documentation, https://docs.brightdata.com/products/scrapers/amazon/introduction
|
|
7
|
+
Project-URL: Source, https://github.com/brightdata/amazon-scraper
|
|
8
|
+
Project-URL: Issues, https://github.com/brightdata/amazon-scraper/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/brightdata/amazon-scraper/blob/main/CHANGELOG.md
|
|
10
|
+
Author: Bright Data
|
|
11
|
+
License: Apache-2.0
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: amazon,amazon-api,amazon-scraper,bright-data,ecommerce,products,reviews,scraper,web-scraping
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: brightdata-sdk<3,>=2.5.2
|
|
23
|
+
Requires-Dist: rich<16,>=13
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
26
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
[](https://brightdata.com/products/web-scraper/amazon?utm_source=github&utm_medium=referral&utm_content=amazon-scraper)
|
|
30
|
+
|
|
31
|
+
# Amazon scraper for Python
|
|
32
|
+
|
|
33
|
+
Scrape Amazon products, reviews, sellers and search results to JSON with Python. No Product Advertising API key, no login, no browser.
|
|
34
|
+
Built on the [Bright Data Amazon Scraper API](https://brightdata.com/products/web-scraper/amazon?utm_source=github&utm_medium=referral&utm_content=amazon-scraper). New accounts get 5,000 free credits a month.
|
|
35
|
+
|
|
36
|
+
[](https://github.com/brightdata/amazon-scraper/actions/workflows/live.yml)
|
|
37
|
+
[](https://github.com/brightdata/amazon-scraper/actions/workflows/live.yml) <!-- verified: rewritten by the weekly run -->
|
|
38
|
+
|
|
39
|
+
[Quickstart](#how-to-scrape-amazon-product-data-with-python) · [No PA-API](#scrape-amazon-without-the-product-advertising-api) · [CLI](#bulk-amazon-scraper-cli) · [Endpoints](#scrape-amazon-products-reviews-sellers-and-search-results) · [Data](#amazon-product-data-fields-json) · [Errors](#amazon-scraper-errors) · [MCP and Claude Code](#amazon-mcp-server-and-claude-code-skill) · [Free credits](#is-this-amazon-scraper-free) · [Docs](https://docs.brightdata.com/products/scrapers/amazon/introduction) · [Support](#support)
|
|
40
|
+
|
|
41
|
+
Amazon products, reviews, sellers and search results as JSON, in Python. No
|
|
42
|
+
login, no browser. Built on the
|
|
43
|
+
[Bright Data Amazon Scraper API](https://brightdata.com/products/web-scraper/amazon?utm_source=github&utm_medium=referral&utm_content=amazon-scraper).
|
|
44
|
+
|
|
45
|
+
Uses the [Bright Data Python SDK](https://github.com/brightdata/sdk-python).
|
|
46
|
+
Full API docs:
|
|
47
|
+
[Amazon Scraper API](https://docs.brightdata.com/products/scrapers/amazon/introduction).
|
|
48
|
+
|
|
49
|
+
Also here: a one-command CLI for products, and the
|
|
50
|
+
[Bright Data CLI](#amazon-mcp-server-and-claude-code-skill), which needs no Python at all.
|
|
51
|
+
|
|
52
|
+
## How to scrape Amazon product data with Python
|
|
53
|
+
|
|
54
|
+
Python 3.10 or newer.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install brightdata-sdk
|
|
58
|
+
export BRIGHTDATA_API_TOKEN=YOUR_API_KEY
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Get a token from the
|
|
62
|
+
[Bright Data control panel](https://brightdata.com/cp/setting/users). A `.env`
|
|
63
|
+
file in the project root works too, as long as the virtualenv is inside the
|
|
64
|
+
project.
|
|
65
|
+
|
|
66
|
+
Or skip the token. Run `npx -p @brightdata/cli bdata login` once: it opens a
|
|
67
|
+
browser, and from then on the SDK finds the stored credentials on its own, for
|
|
68
|
+
you and for any coding agent working in that terminal. Agents cannot click
|
|
69
|
+
through the login, so do it yourself first.
|
|
70
|
+
|
|
71
|
+
No account yet? [Create one](https://brightdata.com/cp/start); new accounts get
|
|
72
|
+
[5,000 free credits a month](https://docs.brightdata.com/general/account/billing-and-pricing/free-tier).
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from brightdata import SyncBrightDataClient
|
|
76
|
+
|
|
77
|
+
with SyncBrightDataClient(auto_create_zones=False) as client:
|
|
78
|
+
product = client.scrape.amazon.products("https://www.amazon.com/dp/B0CRMZHDG8").data
|
|
79
|
+
print(product["title"])
|
|
80
|
+
print(product.get("final_price"), product["currency"], "|", product["rating"], "stars |",
|
|
81
|
+
product["reviews_count"], "reviews")
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
STANLEY Quencher H2.0 Flow State Tumbler, 40 oz, Fuchsia
|
|
86
|
+
39.95 USD | 4.7 stars | 205036 reviews
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
One [credit](https://brightdata.com/pricing/web-scraper) per product, and the
|
|
90
|
+
API quotes about 7 seconds per input.
|
|
91
|
+
|
|
92
|
+
Pass `auto_create_zones=False` every time. Left on, the SDK creates zones on
|
|
93
|
+
startup for Web Unlocker and SERP, two other Bright Data products this scraper
|
|
94
|
+
never touches, and zone creation fails on accounts without a payment method
|
|
95
|
+
([sdk-python#57](https://github.com/brightdata/sdk-python/issues/57)).
|
|
96
|
+
|
|
97
|
+
`products` takes a URL, not an ASIN, so write an ASIN as a `/dp/` URL as above.
|
|
98
|
+
|
|
99
|
+
Read prices with `.get()`. An out-of-stock product comes back with
|
|
100
|
+
`is_available` set to `false` and no `final_price`, `initial_price`,
|
|
101
|
+
`buybox_prices` or `availability` at all, so `product["final_price"]` raises
|
|
102
|
+
`KeyError`.
|
|
103
|
+
|
|
104
|
+
## Scrape Amazon without the Product Advertising API
|
|
105
|
+
|
|
106
|
+
The Amazon Scraper API needs no Amazon account, no seller account and no
|
|
107
|
+
Product Advertising API (PA-API) access key. The only credential is a Bright
|
|
108
|
+
Data API key, set as `BRIGHTDATA_API_TOKEN` or stored by `bdata login`.
|
|
109
|
+
|
|
110
|
+
## Bulk Amazon scraper CLI
|
|
111
|
+
|
|
112
|
+
The command in this repo runs the Python quickstart for several ASINs at
|
|
113
|
+
once and writes one JSON file.
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
pip install git+https://github.com/brightdata/amazon-scraper
|
|
117
|
+
amazon-scraper B0CRMZHDG8 B085DVHQ57
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
```
|
|
121
|
+
Fetching 2 Amazon products: B0CRMZHDG8, B085DVHQ57
|
|
122
|
+
One job for all of them. One credit per product.
|
|
123
|
+
asking 2 products...
|
|
124
|
+
|
|
125
|
+
got B0CRMZHDG8: 100 fields (STANLEY Quencher H2.0 Flow State Tumbler, 40 oz,)
|
|
126
|
+
got B085DVHQ57: 99 fields (Owala FreeSip Stainless Steel Water Bottle 32 oz)
|
|
127
|
+
|
|
128
|
+
Saved 2 of 2 products as JSON to amazon.json
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
It takes a product URL just as happily as the ASIN inside it. The same ASIN
|
|
132
|
+
twice is fetched once, so a duplicate costs nothing.
|
|
133
|
+
|
|
134
|
+
In a terminal the `asking` line is replaced by this, updating in place, so you
|
|
135
|
+
can see it is working and how long it has been going:
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
⠹ 2 products 0:00:14
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
--out PATH output file, default amazon.json
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
`python -m amazon_scraper` works too.
|
|
146
|
+
|
|
147
|
+
Every ASIN you ask for goes into one job. The API bills per record, not per
|
|
148
|
+
job, so ten products cost the same wait as one.
|
|
149
|
+
|
|
150
|
+
Import it instead of running it, for `ok` and `error` per ASIN instead of raw
|
|
151
|
+
rows. `scrape` never raises for one bad ASIN; check `ok` before reading
|
|
152
|
+
`product`:
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from amazon_scraper import scrape
|
|
156
|
+
|
|
157
|
+
for outcome in scrape(["B0CRMZHDG8", "B0ZZZZZZZZ"]):
|
|
158
|
+
if outcome.ok:
|
|
159
|
+
print(f"{outcome.asin}: {outcome.product['title']}")
|
|
160
|
+
else:
|
|
161
|
+
print(f"{outcome.asin} failed: {outcome.error}")
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
```
|
|
165
|
+
B0CRMZHDG8: STANLEY Quencher H2.0 Flow State Tumbler, 40 oz, Fuchsia
|
|
166
|
+
B0ZZZZZZZZ failed: The navigation resulted in a dead page (404 status code)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Scrape Amazon products, reviews, sellers and search results
|
|
170
|
+
|
|
171
|
+
The command covers the first row of the table below. Every snippet here is
|
|
172
|
+
complete and needs only `brightdata-sdk`: paste it as is. Every one of them
|
|
173
|
+
runs in Actions each Monday, a smaller check runs every other day, and the
|
|
174
|
+
badge at the top is the latest result.
|
|
175
|
+
|
|
176
|
+
| you have | want | call |
|
|
177
|
+
| --- | --- | --- |
|
|
178
|
+
| a product URL | that product | `client.scrape.amazon.products(url)` |
|
|
179
|
+
| a keyword | a page of search results | `client.search.amazon.products(keyword=...)`, see the note below |
|
|
180
|
+
| a seller URL | that seller | `client.scrape.amazon.sellers(url)` |
|
|
181
|
+
| a product URL | its reviews | `client.scrape.amazon.reviews(url)`, one credit per review, and see below |
|
|
182
|
+
| a keyword, category URL, best sellers URL or UPC | full product records | no Python twin; see below |
|
|
183
|
+
|
|
184
|
+
The `timeout` argument is seconds, 240 by default for Amazon.
|
|
185
|
+
|
|
186
|
+
### Scrape Amazon reviews, and why the call is not run here
|
|
187
|
+
|
|
188
|
+
Reviews bill one credit per review. `reviews()` accepts `numOfReviews`,
|
|
189
|
+
`pastDays` and `keyWord`, and sends none of them: the request is built from the
|
|
190
|
+
URL alone ([sdk-python#62](https://github.com/brightdata/sdk-python/issues/62)).
|
|
191
|
+
So `reviews(url, numOfReviews=20)` asks for every review, and nothing warns you.
|
|
192
|
+
|
|
193
|
+
The API does support a cap, `max_reviews`, and Bright Data's own documented
|
|
194
|
+
sample sets it to 20. The SDK does not map `numOfReviews` onto it.
|
|
195
|
+
|
|
196
|
+
The product in the [quickstart](#how-to-scrape-amazon-product-data-with-python) reports 205,036 reviews. That is one call. Until
|
|
197
|
+
the SDK sends the cap, call the REST endpoint directly if you need a bounded
|
|
198
|
+
slice.
|
|
199
|
+
|
|
200
|
+
### Scrape Amazon search results by keyword
|
|
201
|
+
|
|
202
|
+
`client.search.amazon.products(keyword=...)` returns rows from the product
|
|
203
|
+
search dataset, about 30 fields each, one row per search result. It is not the
|
|
204
|
+
same thing as the Node twin's keyword discovery, which returns full 119-field
|
|
205
|
+
product records.
|
|
206
|
+
|
|
207
|
+
It has no parameter that caps how many rows come back, and it does not send
|
|
208
|
+
`pages_to_search`. It is not run here, because every snippet in this README
|
|
209
|
+
reruns weekly.
|
|
210
|
+
|
|
211
|
+
The API can also discover full product records by keyword, category URL, best
|
|
212
|
+
sellers URL and UPC. This SDK has none of the four; the JavaScript SDK has all
|
|
213
|
+
of them.
|
|
214
|
+
|
|
215
|
+
### Scrape multiple Amazon ASINs in one job
|
|
216
|
+
|
|
217
|
+
A list of URLs is one job, not one per product.
|
|
218
|
+
|
|
219
|
+
```python
|
|
220
|
+
from brightdata import SyncBrightDataClient
|
|
221
|
+
|
|
222
|
+
with SyncBrightDataClient(auto_create_zones=False) as client:
|
|
223
|
+
results = client.scrape.amazon.products([
|
|
224
|
+
"https://www.amazon.com/dp/B0CRMZHDG8",
|
|
225
|
+
"https://www.amazon.com/dp/B085DVHQ57",
|
|
226
|
+
])
|
|
227
|
+
for result in results:
|
|
228
|
+
row = result.data
|
|
229
|
+
print(row["asin"], "|", row["brand"], "|", row.get("final_price"), row["currency"])
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
```
|
|
233
|
+
B0CRMZHDG8 | STANLEY | 39.95 USD
|
|
234
|
+
B085DVHQ57 | Owala | 29.99 USD
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Read which product a row is from its own `asin`, as above, never from
|
|
238
|
+
`result.url`. The SDK pairs results with the URLs you passed by position, and
|
|
239
|
+
the API returns rows in a different order each run, so `result.url` can name
|
|
240
|
+
one product while `result.data` holds another. Every result is also marked
|
|
241
|
+
`success=True`, including a dead ASIN's error row
|
|
242
|
+
([sdk-python#60](https://github.com/brightdata/sdk-python/issues/60)).
|
|
243
|
+
|
|
244
|
+
### Trigger now, fetch later
|
|
245
|
+
|
|
246
|
+
For anything bigger than a few products, do not block a process for an hour.
|
|
247
|
+
Trigger, keep the snapshot id, fetch when ready. Snapshots stay downloadable
|
|
248
|
+
for 30 days.
|
|
249
|
+
|
|
250
|
+
```python
|
|
251
|
+
import time
|
|
252
|
+
|
|
253
|
+
from brightdata import SyncBrightDataClient
|
|
254
|
+
|
|
255
|
+
with SyncBrightDataClient(auto_create_zones=False) as client:
|
|
256
|
+
job = client.scrape.amazon.products_trigger("https://www.amazon.com/dp/B0CRMZHDG8")
|
|
257
|
+
print("snapshot:", job.snapshot_id)
|
|
258
|
+
while (status := client.scrape.amazon.products_status(job.snapshot_id)) not in ("ready", "failed"):
|
|
259
|
+
time.sleep(5)
|
|
260
|
+
print("status:", status)
|
|
261
|
+
record = client.scrape.amazon.products_fetch(job.snapshot_id)[0]
|
|
262
|
+
print("fetched:", record["asin"], "|", record["title"][:40])
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
```
|
|
266
|
+
snapshot: sd_muars9x1e0ejmq4cx
|
|
267
|
+
status: ready
|
|
268
|
+
fetched: B0CRMZHDG8 | STANLEY Quencher H2.0 Flow State Tumbler
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
## Amazon product data fields (JSON)
|
|
272
|
+
|
|
273
|
+
The fields most people want from a product:
|
|
274
|
+
|
|
275
|
+
```
|
|
276
|
+
asin title brand final_price currency rating reviews_count availability
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
The code hardcodes no field list. Whatever the API returns lands in
|
|
280
|
+
`result.data`, and in the command's file.
|
|
281
|
+
|
|
282
|
+
The API marks 3 of these fields as personal data, with `pii: true` in the
|
|
283
|
+
schema: `seller_name`, `zipcode` and `coupon`. `get_metadata()` drops that
|
|
284
|
+
flag ([sdk-python#61](https://github.com/brightdata/sdk-python/issues/61)), so
|
|
285
|
+
the table below reads the raw metadata endpoint instead.
|
|
286
|
+
|
|
287
|
+
<!-- fields:start -->
|
|
288
|
+
<details>
|
|
289
|
+
<summary>All 122 fields, with type and description</summary>
|
|
290
|
+
|
|
291
|
+
Regenerated every week from the dataset schema, via the raw metadata
|
|
292
|
+
endpoint, so it cannot go stale. A
|
|
293
|
+
product carries the fields that apply to it: the sample file has 97
|
|
294
|
+
of these 122, plus `timestamp` and `input`,
|
|
295
|
+
which the schema does not list.
|
|
296
|
+
|
|
297
|
+
| field | type | description |
|
|
298
|
+
| --- | --- | --- |
|
|
299
|
+
| `title` | text | Product title |
|
|
300
|
+
| `seller_name` | text | Personal data. Seller name |
|
|
301
|
+
| `brand` | text | Product brand |
|
|
302
|
+
| `description` | text | A brief description of the product |
|
|
303
|
+
| `initial_price` | price | Initial price |
|
|
304
|
+
| `currency` | text | Currency of the product |
|
|
305
|
+
| `availability` | text | Product availability |
|
|
306
|
+
| `reviews_count` | number | Number of reviews |
|
|
307
|
+
| `categories` | array | Product categories |
|
|
308
|
+
| `parent_asin` | text | Parent ASIN of the product |
|
|
309
|
+
| `asin` | text | Unique identifier for each product |
|
|
310
|
+
| `buybox_seller` | text | Seller in the buy box |
|
|
311
|
+
| `number_of_sellers` | number | Number of sellers for the product |
|
|
312
|
+
| `root_bs_rank` | number | Best sellers rank in the general category |
|
|
313
|
+
| `ISBN10` | text | ISBN-10 identifier for books |
|
|
314
|
+
| `answered_questions` | number | Number of answered questions |
|
|
315
|
+
| `domain` | url | URL of the product domain |
|
|
316
|
+
| `images_count` | number | Number of images |
|
|
317
|
+
| `url` | url | URL that links directly to the product |
|
|
318
|
+
| `video_count` | number | Number of videos |
|
|
319
|
+
| `image_url` | url | URL that links directly to the product image |
|
|
320
|
+
| `item_weight` | text | Weight of the product |
|
|
321
|
+
| `rating` | number | Product rating |
|
|
322
|
+
| `product_dimensions` | text | Dimensions of the product |
|
|
323
|
+
| `seller_id` | text | Unique identifier for each seller |
|
|
324
|
+
| `image` | url | URL that links directly to the product image |
|
|
325
|
+
| `date_first_available` | text | Date when the product first became available |
|
|
326
|
+
| `discount` | text | Product discount information |
|
|
327
|
+
| `model_number` | text | Model number of the product |
|
|
328
|
+
| `manufacturer` | text | Manufacturer of the product |
|
|
329
|
+
| `department` | text | Department to which the product belongs |
|
|
330
|
+
| `plus_content` | boolean | Boolean indicating the presence of additional content |
|
|
331
|
+
| `upc` | text | Universal Product Code |
|
|
332
|
+
| `video` | boolean | Boolean indicating the presence of videos |
|
|
333
|
+
| `top_review` | text | Top review for the product |
|
|
334
|
+
| `final_price_high` | price | Highest value of the final price when it is a range |
|
|
335
|
+
| `final_price` | price | Final price of the product |
|
|
336
|
+
| `variations` | array | Details about the same product in different variations |
|
|
337
|
+
| `delivery` | array | Delivery-related information |
|
|
338
|
+
| `features` | array | Product features |
|
|
339
|
+
| `format` | array | Books format-related information |
|
|
340
|
+
| `buybox_prices` | object | Product price details |
|
|
341
|
+
| `input_asin` | text | Input ASIN (currently inactive) |
|
|
342
|
+
| `ingredients` | text | Ingredients of the product, relevant mostly for food products |
|
|
343
|
+
| `origin_url` | url | Source page URL used to extract this record |
|
|
344
|
+
| `bought_past_month` | number | Units bought in the past month (as shown by Amazon) |
|
|
345
|
+
| `is_available` | boolean | Indication if the product is still available |
|
|
346
|
+
| `root_bs_category` | text | best seller root category |
|
|
347
|
+
| `bs_category` | text | best seller category |
|
|
348
|
+
| `bs_rank` | number | best seller rank in the specific category |
|
|
349
|
+
| `badge` | text | product badge. for example: #1 Best Seller or Amazons Choice |
|
|
350
|
+
| `subcategory_rank` | array | Best Sellers rank entries by subcategory |
|
|
351
|
+
| `amazon_choice` | boolean | Specifies if the product is amazons choice |
|
|
352
|
+
| `images` | array | URLs of the product images |
|
|
353
|
+
| `product_details` | array | Full product details |
|
|
354
|
+
| `prices_breakdown` | object | Breakdown of list/typical pricing and deal status |
|
|
355
|
+
| `country_of_origin` | text | Country of origin of the product |
|
|
356
|
+
| `from_the_brand` | array | Brand-provided promotional media shown on the page |
|
|
357
|
+
| `product_description` | array | Media embedded in the product description section |
|
|
358
|
+
| `seller_url` | url | Seller storefront/profile URL on Amazon |
|
|
359
|
+
| `customer_says` | text | customer_says |
|
|
360
|
+
| `sustainability_features` | array | Sustainability badges/certifications with references |
|
|
361
|
+
| `climate_pledge_friendly` | boolean | Whether the product shows the Climate Pledge Friendly badge |
|
|
362
|
+
| `videos` | array | URLs of the products videos |
|
|
363
|
+
| `other_sellers_prices` | array | Offers from other sellers for the same product |
|
|
364
|
+
| `downloadable_videos` | array | Direct media URL |
|
|
365
|
+
| `editorial_reviews` | array | The Editorial Reviews of the book |
|
|
366
|
+
| `about_the_author` | text | About the author information |
|
|
367
|
+
| `zipcode` | text | Personal data. ZIP/postal code used for delivery and availability estimates |
|
|
368
|
+
| `coupon` | text | Personal data. coupon |
|
|
369
|
+
| `sponsered` | boolean | sponsored |
|
|
370
|
+
| `store_url` | url | The products store URL |
|
|
371
|
+
| `ships_from` | text | Where the item ships from |
|
|
372
|
+
| `city` | text | City related to shipping or seller/location context |
|
|
373
|
+
| `customers_say` | object | Amazons Customers say summary extracted from reviews |
|
|
374
|
+
| `max_quantity_available` | number | Maximum quantity allowed to add to cart |
|
|
375
|
+
| `variations_values` | array | Variations and their possible values |
|
|
376
|
+
| `language` | text | Language of the product page/content |
|
|
377
|
+
| `return_policy` | text | Return policy text shown on the product page |
|
|
378
|
+
| `inactive_buy_box` | object | Price information when the Buy Box is unavailable/inactive |
|
|
379
|
+
| `buybox_seller_rating` | number | The rating of the buy box seller |
|
|
380
|
+
| `premium_brand` | boolean | Is it premium brand |
|
|
381
|
+
| `amazon_prime` | boolean | Does it have amazon prime delivery |
|
|
382
|
+
| `coupon_description` | text | coupon description |
|
|
383
|
+
| `all_badges` | array | all badges |
|
|
384
|
+
| `sponsored` | boolean | Amazon Sponsored flag |
|
|
385
|
+
| `variant_id` | text | Unique identifier for the specific variant |
|
|
386
|
+
| `product_category` | text | Full breadcrumb path joined with separator |
|
|
387
|
+
| `category_tree` | array | Category hierarchy as array of objects with name and url |
|
|
388
|
+
| `availability_date` | text | Expected availability date for out-of-stock items |
|
|
389
|
+
| `listing_has_variations` | boolean | Whether the listing has multiple variants |
|
|
390
|
+
| `variant_attributes` | array | Current variant attributes as name/value pairs |
|
|
391
|
+
| `variants` | array | Structured variant options grouped by type |
|
|
392
|
+
| `seller_privacy_policy` | text | URL to seller privacy policy |
|
|
393
|
+
| `seller_tos` | text | URL to seller terms of service |
|
|
394
|
+
| `return_window` | number | Number of days for return window |
|
|
395
|
+
| `target_countries` | array | Countries where product ships to |
|
|
396
|
+
| `store_country` | text | store_country |
|
|
397
|
+
| `category_urls` | array | category breadcrumbs |
|
|
398
|
+
| `all_variations` | boolean | nput field for collecting all variations |
|
|
399
|
+
| `safety_information` | text | Safety Information |
|
|
400
|
+
| `subcategory_link` | array | Best Sellers link entries by subcategory |
|
|
401
|
+
| `all_inactive_buy_box` | array | all inactive buy box information |
|
|
402
|
+
| `is_frequently_returned_item_badge` | boolean | Indicates whether the badge is present |
|
|
403
|
+
| `frequently_returned_item_message` | text | The text shown inside the warning box |
|
|
404
|
+
| `is_customers_usually_keep` | boolean | Indicates whether customers usually keep this item |
|
|
405
|
+
| `title_badge` | text | Title of the badge |
|
|
406
|
+
| `review_images` | array | review_images |
|
|
407
|
+
| `review_videos` | array | review_videos |
|
|
408
|
+
| `also_viewed` | array | Customers also viewed |
|
|
409
|
+
| `similar_items` | array | Consider a similar item |
|
|
410
|
+
| `bought_past_month_text` | text | Units bought in the past month (as displayed by Amazon), in text format. |
|
|
411
|
+
| `is_high_price` | boolean | Indicates whether the product is tagged as high price. |
|
|
412
|
+
| `title_highlight` | text | Captures the supplementary marketing/highlight text that Amazon renders directly after the product title on the product page |
|
|
413
|
+
| `title_clean` | text | The actual product title as defined by the seller/brand, excluding any Amazon-rendered highlight text that may appear next to/after it on the page |
|
|
414
|
+
| `customers_say_topics` | array | Structured breakdown of the "Customers say" section |
|
|
415
|
+
| `brand_url` | url | Product brand URL |
|
|
416
|
+
| `variant_condition` | text | The condition of a product variant (e.g., New, Renewed, Refurbished - Excellent). |
|
|
417
|
+
| `is_aplus_premium` | boolean | Indicates whether the product is Premium A+ |
|
|
418
|
+
| `buying_option_type` | text | Buying option type |
|
|
419
|
+
| `amazon_checkout_promotion` | object | Product's checkout promotion |
|
|
420
|
+
| `selected_format` | text | Selected format |
|
|
421
|
+
|
|
422
|
+
</details>
|
|
423
|
+
<!-- fields:end -->
|
|
424
|
+
|
|
425
|
+
<details>
|
|
426
|
+
<summary>The start of a real output file, from <code>amazon-scraper B0CRMZHDG8</code></summary>
|
|
427
|
+
|
|
428
|
+
```json
|
|
429
|
+
{
|
|
430
|
+
"generated_at": "2026-09-21T04:54:18.105606+00:00",
|
|
431
|
+
"products": [
|
|
432
|
+
{
|
|
433
|
+
"asin": "B0CRMZHDG8",
|
|
434
|
+
"product": {
|
|
435
|
+
"title": "STANLEY Quencher H2.0 Flow State Tumbler, 40 oz, Fuchsia",
|
|
436
|
+
"seller_name": "Avrix Brands",
|
|
437
|
+
"brand": "STANLEY",
|
|
438
|
+
"description": "Constructed of recycled stainless steel for sustainable sipping, our 40 oz Quencher H2.0 offers maximum hydration with fewer refills. Commuting, studio workouts, day trips or your front porch\u2014you\u2019ll want this tumbler by your side. Thanks to Stanley\u2019s vacuum insulation, your water will stay ice-cold, hour after hour. The advanced FlowState\u2122 lid features a rotating cover with three positions: a straw opening designed to resist splashes while holding the reusable straw in place, a drink opening, and a full-cover top. The ergonomic handle includes comfort-grip inserts for easy carrying, and the narrow base fits just about any car cup holder.",
|
|
439
|
+
"initial_price": 45,
|
|
440
|
+
"currency": "USD",
|
|
441
|
+
"availability": "In Stock",
|
|
442
|
+
"reviews_count": 205036,
|
|
443
|
+
"categories": [
|
|
444
|
+
"Home & Kitchen",
|
|
445
|
+
"Kitchen & Dining",
|
|
446
|
+
"Storage & Organization",
|
|
447
|
+
"Thermoses",
|
|
448
|
+
...
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
The whole file, one product with every field, is
|
|
452
|
+
[examples/sample_output.json](https://github.com/brightdata/amazon-scraper/blob/main/examples/sample_output.json).
|
|
453
|
+
|
|
454
|
+
</details>
|
|
455
|
+
|
|
456
|
+
## Amazon scraper errors
|
|
457
|
+
|
|
458
|
+
| you see | what it means |
|
|
459
|
+
| --- | --- |
|
|
460
|
+
| `API token required but not found.` | Exit 2, before any request. Set the token. |
|
|
461
|
+
| `failed ASIN: The navigation resulted in a dead page (404 status code)` | Exit 1. No such product, usually a typo in the ASIN. |
|
|
462
|
+
| `failed ASIN: the API returned no row for this ASIN` | Exit 1. The job came back without a row for that input. Run it again. |
|
|
463
|
+
| `failed ASIN: timeout` | Exit 1. A request gives up after 240 seconds. Run it again. |
|
|
464
|
+
| `failed ASIN: Failed to fetch results: Request timeout after 30 seconds` | Exit 1. The job finished, but downloading it took longer than 30 seconds. Run it again. |
|
|
465
|
+
|
|
466
|
+
Any failure exits 1, so a run is safe to gate a script on.
|
|
467
|
+
|
|
468
|
+
From the SDK, the same conditions look like this:
|
|
469
|
+
|
|
470
|
+
| you see | what it means |
|
|
471
|
+
| --- | --- |
|
|
472
|
+
| `AuthenticationError: Unauthorized (401)` | The token is set but wrong. |
|
|
473
|
+
| `result.success` is `False`, `result.status` is `"timeout"` | The SDK gave up waiting for the job, 240 seconds by default. Pass a larger `timeout` to the call, or run it again. |
|
|
474
|
+
| `Failed to fetch results: Request timeout after 30 seconds` | The job finished; downloading it did not. That is the client's own limit on each HTTP request, `SyncBrightDataClient(timeout=30)`, not the `timeout` you pass to the call. Raising the call's `timeout` does not help. |
|
|
475
|
+
| a dict in `result.data` with an `error` key | The API's answer for one input. `result.success` still says `True`; read the row. |
|
|
476
|
+
|
|
477
|
+
## Amazon MCP server and Claude Code skill
|
|
478
|
+
|
|
479
|
+
No Python, nothing installed. Paste both lines; the first opens a browser
|
|
480
|
+
once, or use `bdata login --device` over SSH and in CI:
|
|
481
|
+
|
|
482
|
+
```bash
|
|
483
|
+
npx -p @brightdata/cli bdata login
|
|
484
|
+
npx -p @brightdata/cli bdata pipelines run gd_l7q7dkf244hwjntr0 collect_by_url "https://www.amazon.com/dp/B0CRMZHDG8"
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
This is the CLI 0.4.0 syntax; the older `bdata pipelines amazon_product` form
|
|
488
|
+
was removed in that release. `bdata pipelines search amazon` finds the scraper
|
|
489
|
+
ids. The Amazon ones are `gd_l7q7dkf244hwjntr0` (products),
|
|
490
|
+
`gd_le8e811kzy4ggddlq` (reviews) and `gd_lwdb4vjm1ehb499uxs` (search), all with
|
|
491
|
+
the `collect_by_url` method. `bdata pipelines info <id> collect_by_url` lists a
|
|
492
|
+
scraper's input fields. Each takes URLs, prints JSON, and costs one credit per
|
|
493
|
+
record.
|
|
494
|
+
|
|
495
|
+
`npx skills add brightdata/skills` teaches Claude Code, Cursor and Codex these
|
|
496
|
+
commands and the docs, so plain language works afterwards. Full guide:
|
|
497
|
+
[Bright Data for your coding agent](https://docs.brightdata.com/quickstart-coding-agent).
|
|
498
|
+
|
|
499
|
+
No terminal, for a hosted assistant? The
|
|
500
|
+
[Bright Data MCP server](https://github.com/brightdata/brightdata-mcp#which-tool-to-use)
|
|
501
|
+
has Amazon tools in its `ecommerce` group, which is off unless you ask for it:
|
|
502
|
+
|
|
503
|
+
https://mcp.brightdata.com/mcp?token=YOUR_API_TOKEN&groups=ecommerce
|
|
504
|
+
|
|
505
|
+
An agent can also open the account itself, no signup form:
|
|
506
|
+
[agent registration](https://brightdata.com/auth.md). Everything else Bright
|
|
507
|
+
Data connects to, from LangChain to Zapier and n8n:
|
|
508
|
+
[integrations](https://docs.brightdata.com/integrations/introduction).
|
|
509
|
+
|
|
510
|
+
## Is this Amazon scraper free?
|
|
511
|
+
|
|
512
|
+
New Bright Data accounts get
|
|
513
|
+
[5,000 free credits a month](https://docs.brightdata.com/general/account/billing-and-pricing/free-tier).
|
|
514
|
+
Products and search results cost one credit per record, and reviews one
|
|
515
|
+
credit per review, so the free credits cover 5,000 products a month.
|
|
516
|
+
[Pricing](https://brightdata.com/pricing/web-scraper) lists the rates after that.
|
|
517
|
+
|
|
518
|
+
## Support
|
|
519
|
+
|
|
520
|
+
Bugs in this repo:
|
|
521
|
+
[open an issue](https://github.com/brightdata/amazon-scraper/issues), and
|
|
522
|
+
[CONTRIBUTING.md](https://github.com/brightdata/amazon-scraper/blob/main/CONTRIBUTING.md) says what to put in it.
|
|
523
|
+
Anything about the API, your account or your credits:
|
|
524
|
+
[Bright Data support](https://brightdata.zendesk.com/hc/en-us/requests/new).
|
|
525
|
+
|
|
526
|
+
## License
|
|
527
|
+
|
|
528
|
+
[Apache 2.0](https://github.com/brightdata/amazon-scraper/blob/main/LICENSE).
|
|
529
|
+
|
|
530
|
+
## Amazon scraper for Node.js
|
|
531
|
+
|
|
532
|
+
The same Amazon scraper is available for Node.js: [brightdata/amazon-scraper-node](https://github.com/brightdata/amazon-scraper-node).
|
|
533
|
+
|
|
534
|
+
## More open source scrapers from Bright Data
|
|
535
|
+
|
|
536
|
+
Other Python scrapers built the same way, with live-verified examples:
|
|
537
|
+
|
|
538
|
+
- [URL to Markdown (Fetch API) for Python](https://github.com/brightdata/fetch-api-scraper)
|
|
539
|
+
- [TikTok scraper for Python](https://github.com/brightdata/tiktok-scraper)
|
|
540
|
+
- [ChatGPT scraper for Python](https://github.com/brightdata/chatgpt-scraper)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
amazon_scraper/__init__.py,sha256=yrB0nDtpWVPau46i5t-ZjKgNVBneX6-VAo6WXrBecsk,311
|
|
2
|
+
amazon_scraper/__main__.py,sha256=M5pv3-jXRbHXkNYIJb5K5h0x3Y3Bn5bf_6ABdGrODuU,3008
|
|
3
|
+
amazon_scraper/scrape.py,sha256=vT3NKfQE9J6bQM2JCz10XiUN5UqV7SSRFmw__G4mNkw,6800
|
|
4
|
+
brightdata_amazon_scraper-0.1.0.dist-info/METADATA,sha256=gZp_Shey0GmQqUvAZ3f7LosYzrkj7bmXYsTiPJLqk8o,26729
|
|
5
|
+
brightdata_amazon_scraper-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
6
|
+
brightdata_amazon_scraper-0.1.0.dist-info/entry_points.txt,sha256=IQVfMumFb9Hz2Hb9rqfTcnN3OEshhDScDU3ab9COUlo,64
|
|
7
|
+
brightdata_amazon_scraper-0.1.0.dist-info/licenses/LICENSE,sha256=yVuuHRzgI17MzTVgt3LsHvuX80innw--CmNPDCzO_iw,11358
|
|
8
|
+
brightdata_amazon_scraper-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
202
|
+
|