brightdata-amazon-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brightdata_amazon_scraper-0.1.0/.env.example +3 -0
- brightdata_amazon_scraper-0.1.0/.github/banner.png +0 -0
- brightdata_amazon_scraper-0.1.0/.github/workflows/ci.yml +124 -0
- brightdata_amazon_scraper-0.1.0/.github/workflows/live.yml +439 -0
- brightdata_amazon_scraper-0.1.0/.github/workflows/publish.yml +38 -0
- brightdata_amazon_scraper-0.1.0/.gitignore +7 -0
- brightdata_amazon_scraper-0.1.0/AGENTS.md +111 -0
- brightdata_amazon_scraper-0.1.0/CHANGELOG.md +8 -0
- brightdata_amazon_scraper-0.1.0/CONTRIBUTING.md +27 -0
- brightdata_amazon_scraper-0.1.0/LICENSE +202 -0
- brightdata_amazon_scraper-0.1.0/PKG-INFO +540 -0
- brightdata_amazon_scraper-0.1.0/README.md +512 -0
- brightdata_amazon_scraper-0.1.0/examples/sample_output.json +4289 -0
- brightdata_amazon_scraper-0.1.0/pyproject.toml +52 -0
- brightdata_amazon_scraper-0.1.0/src/amazon_scraper/__init__.py +15 -0
- brightdata_amazon_scraper-0.1.0/src/amazon_scraper/__main__.py +79 -0
- brightdata_amazon_scraper-0.1.0/src/amazon_scraper/scrape.py +183 -0
- brightdata_amazon_scraper-0.1.0/tests/test_scrape.py +325 -0
|
Binary file
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Runs by itself on every push and pull request.
|
|
2
|
+
# Never calls the Instagram API, so it needs no token and costs nothing.
|
|
3
|
+
name: Tests (automatic, no API calls, free)
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch: # a Run workflow button, to re-run the tests without a commit
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
tests:
|
|
15
|
+
name: Run the tests and the code style check
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
timeout-minutes: 10
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
20
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
- name: Install the package and its test tools
|
|
24
|
+
run: pip install -e ".[dev]"
|
|
25
|
+
- name: Check code style with ruff
|
|
26
|
+
run: ruff check .
|
|
27
|
+
- name: Run the tests
|
|
28
|
+
run: pytest
|
|
29
|
+
- name: README headings are well formed
|
|
30
|
+
# A section rewrite once produced "## The data## The data" and shipped;
|
|
31
|
+
# a heading with a second run of hashes mid-line is never intended.
|
|
32
|
+
run: |
|
|
33
|
+
if grep -nE '^#+ .*#{2,} ' README.md; then
|
|
34
|
+
echo "::error::malformed heading in README.md"; exit 1
|
|
35
|
+
fi
|
|
36
|
+
|
|
37
|
+
# These install steps shipped broken once, because they were only ever run
|
|
38
|
+
# from a working copy that already had the package. This job follows the
|
|
39
|
+
# README on an empty machine and stops where the tool asks for a token, which
|
|
40
|
+
# proves the clone, the install and the command without spending money.
|
|
41
|
+
coldstart:
|
|
42
|
+
name: Follow the README from scratch on an empty machine
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
timeout-minutes: 10
|
|
45
|
+
steps:
|
|
46
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
47
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
48
|
+
with:
|
|
49
|
+
python-version: "3.12"
|
|
50
|
+
- name: Copy the install steps out of the README and run them
|
|
51
|
+
run: |
|
|
52
|
+
set -o pipefail
|
|
53
|
+
python - <<'PY' > "$RUNNER_TEMP/quickstart.sh"
|
|
54
|
+
import os
|
|
55
|
+
import pathlib
|
|
56
|
+
import re
|
|
57
|
+
import sys
|
|
58
|
+
|
|
59
|
+
readme = pathlib.Path("README.md").read_text(encoding="utf-8")
|
|
60
|
+
|
|
61
|
+
def section(title):
|
|
62
|
+
return readme.split(f"## {title}", 1)[1].split("\n## ", 1)[0]
|
|
63
|
+
|
|
64
|
+
def block(text, lang):
|
|
65
|
+
return re.search(r"```" + lang + r"\n(.*?)```", text, re.S).group(1)
|
|
66
|
+
|
|
67
|
+
quick = section("How to scrape Amazon product data with Python")
|
|
68
|
+
install, snippet = block(quick, "bash"), block(quick, "python")
|
|
69
|
+
command = block(section("Bulk Amazon scraper CLI"), "bash")
|
|
70
|
+
for required, where in (("pip install brightdata-sdk", install),
|
|
71
|
+
("SyncBrightDataClient", snippet),
|
|
72
|
+
("amazon-scraper", command)):
|
|
73
|
+
if required not in where:
|
|
74
|
+
sys.exit(f"the README no longer contains {required!r} where the cold start expects it")
|
|
75
|
+
|
|
76
|
+
steps = ["# --- Quickstart: the SDK path ---"]
|
|
77
|
+
steps += [l for l in install.splitlines() if l.startswith("pip install")]
|
|
78
|
+
pathlib.Path(os.environ["RUNNER_TEMP"], "quickstart_snippet.py").write_text(snippet, encoding="utf-8")
|
|
79
|
+
steps += [
|
|
80
|
+
"set +e",
|
|
81
|
+
"python quickstart_snippet.py > out.txt 2> err.txt",
|
|
82
|
+
"code=$?",
|
|
83
|
+
"set -e",
|
|
84
|
+
'test "$code" != "0" || { echo "the snippet ran without a token?"; cat out.txt; exit 1; }',
|
|
85
|
+
'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
|
|
86
|
+
'echo "Quickstart: installs, and stops at the token, as documented."',
|
|
87
|
+
"# --- Or run it as a command: the CLI path ---",
|
|
88
|
+
]
|
|
89
|
+
for line in command.splitlines():
|
|
90
|
+
if line.startswith("amazon-scraper"):
|
|
91
|
+
continue # run below, so the exit code can be checked
|
|
92
|
+
if line.startswith("pip install") and "git+" in line:
|
|
93
|
+
# The repository is public, so this installs the exact line a
|
|
94
|
+
# reader pastes. On a private fork the URL is unreachable, and
|
|
95
|
+
# the check falls back to the checkout on its own.
|
|
96
|
+
url = line.split("git+", 1)[1].strip() if "git+" in line else ""
|
|
97
|
+
line = "\n".join([
|
|
98
|
+
f'repo="{url}"',
|
|
99
|
+
'code=$(curl -s -o /dev/null -w "%{http_code}" "$repo" || echo 000)',
|
|
100
|
+
'if [ "$code" = "200" ]; then',
|
|
101
|
+
f' echo "$repo is public: installing the exact README line"',
|
|
102
|
+
f' {line}',
|
|
103
|
+
"else",
|
|
104
|
+
' echo "NOTE: $repo returns HTTP $code, so it is not public yet."',
|
|
105
|
+
' echo "Installing from this checkout instead."',
|
|
106
|
+
' pip install "$GITHUB_WORKSPACE"',
|
|
107
|
+
"fi",
|
|
108
|
+
])
|
|
109
|
+
steps.append(line)
|
|
110
|
+
steps += [
|
|
111
|
+
"set +e",
|
|
112
|
+
"amazon-scraper B0CRMZHDG8 > out.txt 2> err.txt",
|
|
113
|
+
"code=$?",
|
|
114
|
+
"set -e",
|
|
115
|
+
'test "$code" = "2" || { echo "expected exit 2, got $code"; cat err.txt; exit 1; }',
|
|
116
|
+
'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
|
|
117
|
+
'echo "Success: a new user can follow both README paths from scratch."',
|
|
118
|
+
]
|
|
119
|
+
print("\n".join(steps))
|
|
120
|
+
PY
|
|
121
|
+
echo "--- these commands came out of README.md ---"
|
|
122
|
+
cat "$RUNNER_TEMP/quickstart.sh"
|
|
123
|
+
echo "--- running them on this empty machine ---"
|
|
124
|
+
cd "$RUNNER_TEMP" && bash -e "$RUNNER_TEMP/quickstart.sh"
|
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
# The only workflow that calls the real API. Needs the BRIGHTDATA_API_TOKEN
|
|
2
|
+
# secret and spends about one credit per product.
|
|
3
|
+
#
|
|
4
|
+
# Three ways it runs:
|
|
5
|
+
# on every push to main: one product and the README's links, one credit
|
|
6
|
+
# and about a minute, so a broken commit shows up at once. A newer push
|
|
7
|
+
# cancels a run still going for an older commit.
|
|
8
|
+
# by hand, from the Actions tab, to scrape whatever products you choose
|
|
9
|
+
# every Monday at 07:20 UTC: one product, the async path, and the
|
|
10
|
+
# field table, about two credits
|
|
11
|
+
# in the same Monday run: every code block in the README,
|
|
12
|
+
# about 12 credits
|
|
13
|
+
#
|
|
14
|
+
# A slow day at the API shows up as a red run. Rerun it before reading more
|
|
15
|
+
# into it.
|
|
16
|
+
#
|
|
17
|
+
# The Monday run ends by rewriting the "last verified" badge line at
|
|
18
|
+
# the top of README.md with today's date, and committing it if it changed. Push
|
|
19
|
+
# runs never do this, so your own push is never followed by a bot commit. Only
|
|
20
|
+
# the main repository commits; a copy of the repository just runs the checks.
|
|
21
|
+
name: Live check
|
|
22
|
+
|
|
23
|
+
run-name: "${{ github.event.schedule == '20 7 * * 1' && 'Weekly check: does every README block still work?' || github.event_name == 'schedule' && 'Daily check: is anything broken?' || github.event_name == 'push' && format('Push check: {0}', github.event.head_commit.message) || format('Scrape {0}', inputs.asins) }}"
|
|
24
|
+
|
|
25
|
+
on:
|
|
26
|
+
push:
|
|
27
|
+
branches: [main]
|
|
28
|
+
schedule:
|
|
29
|
+
- cron: "20 7 * * 1" # Monday: the whole README. Staggered across the
|
|
30
|
+
# scraper repos (06:00, 06:20, 06:40, 07:00, 07:20) so four full matrices never
|
|
31
|
+
# hit one account at once, which is what trips "too many requests".
|
|
32
|
+
# The two `== '20 7 * * 1'` checks below must match this line exactly.
|
|
33
|
+
workflow_dispatch:
|
|
34
|
+
inputs:
|
|
35
|
+
asins:
|
|
36
|
+
description: Amazon ASINs or product URLs, separated by spaces.
|
|
37
|
+
default: B0CRMZHDG8 B085DVHQ57
|
|
38
|
+
validate:
|
|
39
|
+
description: Also run every README block, as the Monday run does
|
|
40
|
+
type: boolean
|
|
41
|
+
default: false
|
|
42
|
+
only:
|
|
43
|
+
description: Block numbers to run, comma separated, for a cheap check. Empty means all.
|
|
44
|
+
default: ""
|
|
45
|
+
|
|
46
|
+
permissions:
|
|
47
|
+
contents: write # the weekly check commits the regenerated field table
|
|
48
|
+
|
|
49
|
+
concurrency:
|
|
50
|
+
group: live-${{ github.event_name }}-${{ github.ref }}
|
|
51
|
+
cancel-in-progress: ${{ github.event_name == 'push' }} # only the newest commit matters
|
|
52
|
+
|
|
53
|
+
jobs:
|
|
54
|
+
scrape:
|
|
55
|
+
name: Fetch the products and attach them to this run
|
|
56
|
+
runs-on: ubuntu-latest
|
|
57
|
+
timeout-minutes: 20 # fetch, async path, table; the README blocks have their own jobs
|
|
58
|
+
env:
|
|
59
|
+
BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
|
|
60
|
+
# On the weekly schedule there are no inputs, so fall back to one cheap product.
|
|
61
|
+
ASINS: ${{ inputs.asins || 'B0CRMZHDG8' }}
|
|
62
|
+
# the async path and the field table run on every scheduled run and on request
|
|
63
|
+
CHECK: ${{ github.event_name == 'schedule' || inputs.validate }}
|
|
64
|
+
steps:
|
|
65
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
66
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
67
|
+
with:
|
|
68
|
+
python-version: "3.12"
|
|
69
|
+
- name: Install the package and the Bright Data CLI
|
|
70
|
+
run: |
|
|
71
|
+
pip install -e .
|
|
72
|
+
npm i -g @brightdata/cli@0.4.0 # pinned: this job holds the API secret
|
|
73
|
+
|
|
74
|
+
- name: Do the README's links still resolve?
|
|
75
|
+
run: |
|
|
76
|
+
public=$(curl -s -o /dev/null -w '%{http_code}' https://github.com/brightdata/amazon-scraper)
|
|
77
|
+
failed=0
|
|
78
|
+
for url in $(grep -ohE 'https?://[^ )>"`]+' README.md AGENTS.md | sed 's/[.,]$//' | sort -u); do
|
|
79
|
+
case "$url" in
|
|
80
|
+
*amazon.com/*) continue ;; # rate-limits runners; the weekly README run fetches these products through the API anyway
|
|
81
|
+
*zendesk.com/*) continue ;; # answers 403 to anything that is not a browser
|
|
82
|
+
*mcp.brightdata.com/*) continue ;; # carries a placeholder token; nothing to check without a real one
|
|
83
|
+
*api.brightdata.com/*) continue ;; # needs the token; the field-table step below calls it for real
|
|
84
|
+
*github.com/brightdata/amazon-scraper*) [ "$public" = "200" ] || continue ;; # unreachable on a private fork
|
|
85
|
+
esac
|
|
86
|
+
code=$(curl -s -o /dev/null -L --retry 3 --max-time 30 -A "Mozilla/5.0" -w '%{http_code}' "$url" || echo 000)
|
|
87
|
+
if [ "$code" = "200" ]; then echo "200 $url"; else echo "::error::$code $url"; failed=1; fi
|
|
88
|
+
done
|
|
89
|
+
[ "$failed" = 0 ] || exit 1
|
|
90
|
+
|
|
91
|
+
- name: Fetch the products from Amazon
|
|
92
|
+
run: |
|
|
93
|
+
test -n "$BRIGHTDATA_API_TOKEN" || {
|
|
94
|
+
echo "No API token. Add BRIGHTDATA_API_TOKEN under" >&2
|
|
95
|
+
echo "Settings, Secrets and variables, Actions, then start this again." >&2
|
|
96
|
+
exit 1
|
|
97
|
+
}
|
|
98
|
+
# One retry, only on an API timeout. A slow day is not a broken build.
|
|
99
|
+
set +e
|
|
100
|
+
python -m amazon_scraper $ASINS 2>&1 | tee run.log
|
|
101
|
+
code=${PIPESTATUS[0]}
|
|
102
|
+
if [ "$code" != "0" ] && grep -q "timeout" run.log; then
|
|
103
|
+
echo "::warning::The API timed out. Waiting 30s and trying once more."
|
|
104
|
+
sleep 30
|
|
105
|
+
python -m amazon_scraper $ASINS 2>&1 | tee run.log
|
|
106
|
+
code=${PIPESTATUS[0]}
|
|
107
|
+
fi
|
|
108
|
+
exit "$code"
|
|
109
|
+
|
|
110
|
+
- name: Show what was found
|
|
111
|
+
if: always()
|
|
112
|
+
run: |
|
|
113
|
+
set -o pipefail
|
|
114
|
+
test -f amazon.json || exit 0
|
|
115
|
+
python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
|
|
116
|
+
import json
|
|
117
|
+
|
|
118
|
+
doc = json.load(open("amazon.json"))
|
|
119
|
+
print(f"Run at {doc['generated_at']}\n")
|
|
120
|
+
print("| asin | fields | price | rating | reviews |")
|
|
121
|
+
print("| --- | --- | --- | --- | --- |")
|
|
122
|
+
for entry in doc["products"]:
|
|
123
|
+
p = entry["product"] or {}
|
|
124
|
+
print(
|
|
125
|
+
f"| {entry['asin']} | {len(p)} | {p.get('final_price', '-')} {p.get('currency', '')} "
|
|
126
|
+
f"| {p.get('rating', '-')} | {p.get('reviews_count', '-')} |"
|
|
127
|
+
)
|
|
128
|
+
first = next((e["product"] for e in doc["products"] if e["product"]), None)
|
|
129
|
+
if first:
|
|
130
|
+
print("\n<details><summary>First record in full</summary>\n")
|
|
131
|
+
print("```json")
|
|
132
|
+
print(json.dumps(first, indent=2, ensure_ascii=False))
|
|
133
|
+
print("```\n")
|
|
134
|
+
print("</details>")
|
|
135
|
+
PY
|
|
136
|
+
|
|
137
|
+
# The two smoke checks. The products path is already covered by the step above.
|
|
138
|
+
- name: Does the async trigger, status, fetch path still work?
|
|
139
|
+
if: env.CHECK == 'true'
|
|
140
|
+
run: |
|
|
141
|
+
python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
|
|
142
|
+
import time
|
|
143
|
+
|
|
144
|
+
from brightdata import SyncBrightDataClient
|
|
145
|
+
|
|
146
|
+
url = "https://www.amazon.com/dp/B0CRMZHDG8"
|
|
147
|
+
with SyncBrightDataClient(auto_create_zones=False) as c:
|
|
148
|
+
job = c.scrape.amazon.products_trigger(url)
|
|
149
|
+
sid = job.snapshot_id
|
|
150
|
+
for _ in range(72): # up to 6 minutes
|
|
151
|
+
status = c.scrape.amazon.products_status(sid)
|
|
152
|
+
if status in ("ready", "failed", "canceled"):
|
|
153
|
+
break
|
|
154
|
+
time.sleep(5)
|
|
155
|
+
assert status == "ready", f"snapshot {sid} ended as {status}"
|
|
156
|
+
rows = c.scrape.amazon.products_fetch(sid)
|
|
157
|
+
rows = rows if isinstance(rows, list) else [rows]
|
|
158
|
+
assert rows and rows[0].get("asin"), "fetch returned no record"
|
|
159
|
+
print(f"\nAsync path ok: trigger, status, fetch returned {len(rows)} record(s).")
|
|
160
|
+
PY
|
|
161
|
+
|
|
162
|
+
- name: Regenerate the README field table from the live schema
|
|
163
|
+
if: always() && env.CHECK == 'true' && github.repository == 'brightdata/amazon-scraper'
|
|
164
|
+
run: |
|
|
165
|
+
python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
|
|
166
|
+
import json
|
|
167
|
+
import os
|
|
168
|
+
import pathlib
|
|
169
|
+
import re
|
|
170
|
+
import urllib.request
|
|
171
|
+
|
|
172
|
+
readme = pathlib.Path("README.md")
|
|
173
|
+
text = readme.read_text(encoding="utf-8")
|
|
174
|
+
start, end = "<!-- fields:start -->", "<!-- fields:end -->"
|
|
175
|
+
assert start in text and end in text, "README field-table markers are missing"
|
|
176
|
+
|
|
177
|
+
# The raw endpoint, not get_metadata(): the SDK drops the pii flag on
|
|
178
|
+
# every field when it parses this response (sdk-python#61).
|
|
179
|
+
request = urllib.request.Request(
|
|
180
|
+
"https://api.brightdata.com/datasets/gd_l7q7dkf244hwjntr0/metadata",
|
|
181
|
+
headers={"Authorization": f"Bearer {os.environ['BRIGHTDATA_API_TOKEN']}"},
|
|
182
|
+
)
|
|
183
|
+
fields = json.load(urllib.request.urlopen(request, timeout=60))["fields"]
|
|
184
|
+
sample = json.load(open("examples/sample_output.json"))["products"][0]["product"]
|
|
185
|
+
# A product carries a subset of the schema, and the API also returns
|
|
186
|
+
# fields the schema never lists, so these two counts differ.
|
|
187
|
+
known = sum(1 for name in fields if name in sample)
|
|
188
|
+
extra = [k for k in sample if k not in fields]
|
|
189
|
+
|
|
190
|
+
rows = ["| field | type | description |", "| --- | --- | --- |"]
|
|
191
|
+
for name, spec in fields.items():
|
|
192
|
+
desc = (spec.get("description") or "").replace(" — ", ": ").replace("—", ":")
|
|
193
|
+
desc = " ".join(desc.split()).replace("`", "'").replace("|", "/") # no fences, no table breaks
|
|
194
|
+
pii = "Personal data. " if spec.get("pii") else ""
|
|
195
|
+
rows.append(f"| `{name}` | {spec.get('type')} | {pii}{desc} |")
|
|
196
|
+
tail = f", plus {' and '.join(f'`{k}`' for k in extra)}," if extra else "."
|
|
197
|
+
block = "\n".join([
|
|
198
|
+
start,
|
|
199
|
+
"<details>",
|
|
200
|
+
f"<summary>All {len(fields)} fields, with type and description</summary>",
|
|
201
|
+
"",
|
|
202
|
+
"Regenerated every week from the dataset schema, via the raw metadata",
|
|
203
|
+
"endpoint, so it cannot go stale. A",
|
|
204
|
+
f"product carries the fields that apply to it: the sample file has {known}",
|
|
205
|
+
f"of these {len(fields)}{tail}",
|
|
206
|
+
*(["which the schema does not list."] if extra else []),
|
|
207
|
+
"",
|
|
208
|
+
*rows,
|
|
209
|
+
"",
|
|
210
|
+
"</details>",
|
|
211
|
+
end,
|
|
212
|
+
])
|
|
213
|
+
new = re.sub(re.escape(start) + r".*?" + re.escape(end), lambda _: block, text, flags=re.S)
|
|
214
|
+
readme.write_text(new, encoding="utf-8")
|
|
215
|
+
state = "updated" if new != text else "already current"
|
|
216
|
+
print(f"\nSchema: {len(fields)} fields. README table {state}.")
|
|
217
|
+
PY
|
|
218
|
+
if git diff --quiet README.md; then
|
|
219
|
+
echo "Nothing to commit."
|
|
220
|
+
exit 0
|
|
221
|
+
fi
|
|
222
|
+
# Never commit a README the tests would reject.
|
|
223
|
+
pip install -q -e ".[dev]" && pytest -q
|
|
224
|
+
git config user.name "anil-bd"
|
|
225
|
+
git config user.email "210667061+anil-bd@users.noreply.github.com"
|
|
226
|
+
git add README.md
|
|
227
|
+
git commit -m "Regenerate the field table from the live schema (automated weekly check)"
|
|
228
|
+
git pull --rebase origin main
|
|
229
|
+
git push
|
|
230
|
+
echo "Committed the regenerated table."
|
|
231
|
+
|
|
232
|
+
- name: Attach amazon.json so you can download it
|
|
233
|
+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
234
|
+
if: always()
|
|
235
|
+
with:
|
|
236
|
+
name: amazon-json
|
|
237
|
+
path: amazon.json
|
|
238
|
+
if-no-files-found: warn
|
|
239
|
+
|
|
240
|
+
# Every code block in the README, each in its own job. Mondays, and on request.
|
|
241
|
+
plan:
|
|
242
|
+
name: List the README blocks
|
|
243
|
+
if: github.event.schedule == '20 7 * * 1' || inputs.validate == true
|
|
244
|
+
runs-on: ubuntu-latest
|
|
245
|
+
timeout-minutes: 5
|
|
246
|
+
outputs:
|
|
247
|
+
blocks: ${{ steps.list.outputs.blocks }}
|
|
248
|
+
steps:
|
|
249
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
250
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
251
|
+
with:
|
|
252
|
+
python-version: "3.12"
|
|
253
|
+
- id: list
|
|
254
|
+
env:
|
|
255
|
+
ONLY: ${{ inputs.only }}
|
|
256
|
+
run: |
|
|
257
|
+
python - <<'PY' >> "$GITHUB_OUTPUT"
|
|
258
|
+
import os
|
|
259
|
+
import json
|
|
260
|
+
import pathlib
|
|
261
|
+
import re
|
|
262
|
+
|
|
263
|
+
readme = pathlib.Path("README.md").read_text(encoding="utf-8")
|
|
264
|
+
# never execute anything inside the generated field table: its text comes from
|
|
265
|
+
# the API, and a fence in a description must not become a command
|
|
266
|
+
readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
|
|
267
|
+
SKIP = ("pip install", "export ", "npm i", "npm install",
|
|
268
|
+
"npx -p @brightdata/cli bdata login", "bdata login")
|
|
269
|
+
blocks, heading, seen = [], "README", {}
|
|
270
|
+
for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
|
|
271
|
+
if m.group(1):
|
|
272
|
+
heading = m.group(1).strip() # a block is named after its section, not its first line
|
|
273
|
+
continue
|
|
274
|
+
lang, code = m.group(2), m.group(3)
|
|
275
|
+
lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")]
|
|
276
|
+
if lang == "bash":
|
|
277
|
+
lines = [l for l in lines if not l.startswith(SKIP)]
|
|
278
|
+
if not lines:
|
|
279
|
+
continue
|
|
280
|
+
code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines)
|
|
281
|
+
seen[heading] = seen.get(heading, 0) + 1
|
|
282
|
+
title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
|
|
283
|
+
blocks.append({"lang": lang, "title": title[:50], "code": code})
|
|
284
|
+
wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()}
|
|
285
|
+
out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]} for i, b in enumerate(blocks) if not wanted or i + 1 in wanted]
|
|
286
|
+
print("blocks=" + json.dumps(out))
|
|
287
|
+
PY
|
|
288
|
+
|
|
289
|
+
block:
|
|
290
|
+
name: "${{ matrix.n }}. ${{ matrix.title }}"
|
|
291
|
+
needs: plan
|
|
292
|
+
runs-on: ubuntu-latest
|
|
293
|
+
timeout-minutes: 15 # a block gives up at 8 minutes; past this it is stuck
|
|
294
|
+
strategy:
|
|
295
|
+
fail-fast: false
|
|
296
|
+
# Three at a time. The API stalls in waves, and it throttles an account
|
|
297
|
+
# that sends many requests at once. The cap does not fix either;
|
|
298
|
+
# it limits how much of the matrix one bad stretch can take down.
|
|
299
|
+
max-parallel: 3
|
|
300
|
+
matrix:
|
|
301
|
+
include: ${{ fromJson(needs.plan.outputs.blocks) }}
|
|
302
|
+
env:
|
|
303
|
+
BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
|
|
304
|
+
steps:
|
|
305
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
306
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
307
|
+
with:
|
|
308
|
+
python-version: "3.12"
|
|
309
|
+
- name: Install the package and, for CLI blocks, the Bright Data CLI
|
|
310
|
+
run: |
|
|
311
|
+
pip install -e .
|
|
312
|
+
if [ "${{ matrix.lang }}" = "bash" ]; then npm i -g @brightdata/cli@0.4.0; fi
|
|
313
|
+
- name: Run block ${{ matrix.n }} exactly as the README shows it
|
|
314
|
+
env:
|
|
315
|
+
N: ${{ matrix.n }}
|
|
316
|
+
run: |
|
|
317
|
+
set -o pipefail # without this, tee's exit code hides a failing block
|
|
318
|
+
python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
|
|
319
|
+
import os
|
|
320
|
+
import subprocess
|
|
321
|
+
import sys
|
|
322
|
+
import time
|
|
323
|
+
import pathlib
|
|
324
|
+
import re
|
|
325
|
+
|
|
326
|
+
readme = pathlib.Path("README.md").read_text(encoding="utf-8")
|
|
327
|
+
readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
|
|
328
|
+
SKIP = ("pip install", "export ", "npm i", "npm install",
|
|
329
|
+
"npx -p @brightdata/cli bdata login", "bdata login")
|
|
330
|
+
blocks, heading, seen = [], "README", {}
|
|
331
|
+
for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
|
|
332
|
+
if m.group(1):
|
|
333
|
+
heading = m.group(1).strip()
|
|
334
|
+
continue
|
|
335
|
+
lang, code = m.group(2), m.group(3)
|
|
336
|
+
lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")]
|
|
337
|
+
if lang == "bash":
|
|
338
|
+
lines = [l for l in lines if not l.startswith(SKIP)]
|
|
339
|
+
if not lines:
|
|
340
|
+
continue
|
|
341
|
+
code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines)
|
|
342
|
+
seen[heading] = seen.get(heading, 0) + 1
|
|
343
|
+
title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
|
|
344
|
+
blocks.append({"lang": lang, "title": title[:50], "code": code})
|
|
345
|
+
b = blocks[int(os.environ["N"]) - 1]
|
|
346
|
+
argv = ["bash", "-e", "-c", b["code"]] if b["lang"] == "bash" else [sys.executable, "-c", b["code"]]
|
|
347
|
+
secret = os.environ.get("BRIGHTDATA_API_TOKEN", "")
|
|
348
|
+
status = None
|
|
349
|
+
for attempt in (1, 2):
|
|
350
|
+
started = time.time()
|
|
351
|
+
try:
|
|
352
|
+
run = subprocess.run(argv, capture_output=True, text=True, timeout=480)
|
|
353
|
+
except subprocess.TimeoutExpired:
|
|
354
|
+
run = subprocess.CompletedProcess(argv, 1, "", "block timeout: no result within 8 minutes")
|
|
355
|
+
secs = round(time.time() - started)
|
|
356
|
+
status = run.returncode
|
|
357
|
+
out, err = run.stdout.strip(), run.stderr.strip()
|
|
358
|
+
if secret: # summaries are not secret-masked the way logs are
|
|
359
|
+
out, err = out.replace(secret, "***"), err.replace(secret, "***")
|
|
360
|
+
ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err)
|
|
361
|
+
if ok or ((out or err) and "timeout" not in (out + err).lower()):
|
|
362
|
+
break
|
|
363
|
+
time.sleep(30) # the API timed out or answered with nothing at all; one retry
|
|
364
|
+
# A block that exits 0 having printed nothing is not a crash: the call
|
|
365
|
+
# ran and returned no rows. Say so, or the next reader hunts for one.
|
|
366
|
+
if ok:
|
|
367
|
+
shown = [(out.splitlines() or [""])[0]]
|
|
368
|
+
elif status == 0 and not out and not err:
|
|
369
|
+
shown = ["exited 0 and printed nothing: the call returned no rows"]
|
|
370
|
+
else:
|
|
371
|
+
# The last line alone hides the cause. For the CLI it is the
|
|
372
|
+
# "Saved 1 of 2" summary, and the line saying which input failed
|
|
373
|
+
# and why sits above it. Show the failure lines, else the tail.
|
|
374
|
+
lines = [l for l in (out + "\n" + err).splitlines() if l.strip()]
|
|
375
|
+
reasons = [l for l in lines if l.startswith("failed ") or "Error" in l]
|
|
376
|
+
shown = (reasons or lines or [f"no output, exit {status}"])[-6:]
|
|
377
|
+
body = "\n".join(f" {line[:160]}" for line in shown)
|
|
378
|
+
print(f"\n`{b['title']}`: {'ok' if ok else 'FAILED'} in {secs}s\n\n{body}")
|
|
379
|
+
sys.exit(0 if ok else 1)
|
|
380
|
+
PY
|
|
381
|
+
|
|
382
|
+
badge:
|
|
383
|
+
name: Put today's date on the README's "last verified" badge
|
|
384
|
+
needs: [scrape, plan, block]
|
|
385
|
+
# scheduled and manual runs only, and only in the main repository: a push is
|
|
386
|
+
# never followed by a bot commit, and a copy never diverges from the original.
|
|
387
|
+
#
|
|
388
|
+
# `inputs.only` empty as well: a run of one block that passes is not the
|
|
389
|
+
# README verified. A scheduled run has no inputs, where `inputs.only` is null
|
|
390
|
+
# and this comparison is still true, so the Monday run sets it.
|
|
391
|
+
if: always() && !cancelled() && github.event_name != 'push' && inputs.only == '' && github.repository == 'brightdata/amazon-scraper'
|
|
392
|
+
permissions:
|
|
393
|
+
contents: write # commits the badge line
|
|
394
|
+
actions: read # counts this run's job results
|
|
395
|
+
runs-on: ubuntu-latest
|
|
396
|
+
timeout-minutes: 5
|
|
397
|
+
steps:
|
|
398
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
399
|
+
with:
|
|
400
|
+
# The tip of main, not this run's commit. A re-run checks out the
|
|
401
|
+
# commit the run first started on, but an earlier attempt of the same
|
|
402
|
+
# run may already have pushed a badge edit to this very line. Editing
|
|
403
|
+
# from that stale base makes the rebase below stop on a conflict. From
|
|
404
|
+
# the tip, the line is replaced, never merged. This broke the Node twin
|
|
405
|
+
# on 2026-09-18 when a failed run was re-run.
|
|
406
|
+
ref: main
|
|
407
|
+
- name: Rewrite one badge line, commit only if it changed
|
|
408
|
+
env:
|
|
409
|
+
GH_TOKEN: ${{ github.token }}
|
|
410
|
+
run: |
|
|
411
|
+
# Two states: red only when every check in this run failed. One passing
|
|
412
|
+
# check keeps the badge green; the run itself still shows which failed.
|
|
413
|
+
read -r passed failed < <(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
|
|
414
|
+
--jq '[.jobs[] | select((.name | startswith("Put today") or startswith("Open or update") or . == "List the README blocks") | not)] as $j | "\([$j[] | select(.conclusion == "success")] | length) \([$j[] | select(.conclusion == "failure")] | length)"') || true
|
|
415
|
+
echo "checks passed: ${passed:-0}, failed: ${failed:-0}"
|
|
416
|
+
if [ "${passed:-0}" -gt 0 ]; then
|
|
417
|
+
label="last verified"; color="brightgreen"
|
|
418
|
+
else
|
|
419
|
+
label="last check failed"; color="red"
|
|
420
|
+
fi
|
|
421
|
+
today=$(date -u +'%-d %b %Y')
|
|
422
|
+
line="[](https://github.com/brightdata/amazon-scraper/actions/workflows/live.yml) <!-- verified: rewritten by the weekly run -->"
|
|
423
|
+
python3 - "$line" <<'PY'
|
|
424
|
+
import pathlib, re, sys
|
|
425
|
+
p = pathlib.Path("README.md"); t = p.read_text(encoding="utf-8")
|
|
426
|
+
new, n = re.subn(r"^\[!\[last [^\]]*\]\(https://img\.shields\.io/badge/[^)]*\)\]\([^)]*\) <!-- verified:[^\n]*$", sys.argv[1], t, count=1, flags=re.M)
|
|
427
|
+
if n != 1:
|
|
428
|
+
sys.exit("the badge line is missing from README.md")
|
|
429
|
+
p.write_text(new, encoding="utf-8")
|
|
430
|
+
PY
|
|
431
|
+
if git diff --quiet; then echo "badge already shows $today"; exit 0; fi
|
|
432
|
+
git config user.name "anil-bd"
|
|
433
|
+
git config user.email "210667061+anil-bd@users.noreply.github.com"
|
|
434
|
+
# $label, not a fixed "Last verified": the log should not claim a
|
|
435
|
+
# check passed on a day the badge went red.
|
|
436
|
+
git commit -qam "${label^} $today (automated weekly check)"
|
|
437
|
+
git pull -q --rebase origin main
|
|
438
|
+
git push -q origin main
|
|
439
|
+
echo "badge now shows: $label, $today"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Publishes the package to PyPI when a GitHub Release is published.
|
|
2
|
+
# Uses PyPI Trusted Publishing (OIDC), so the repo stores no PyPI token.
|
|
3
|
+
# Release steps: bump __version__ in src/amazon_scraper/__init__.py,
|
|
4
|
+
# add a CHANGELOG.md entry, then publish a GitHub Release tagged v<version>.
|
|
5
|
+
name: Publish to PyPI
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
release:
|
|
9
|
+
types: [published]
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
publish:
|
|
16
|
+
name: Build and publish to PyPI
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
timeout-minutes: 10
|
|
19
|
+
environment:
|
|
20
|
+
name: pypi
|
|
21
|
+
url: https://pypi.org/p/brightdata-amazon-scraper
|
|
22
|
+
permissions:
|
|
23
|
+
id-token: write # required for Trusted Publishing
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
26
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
27
|
+
with:
|
|
28
|
+
python-version: "3.12"
|
|
29
|
+
- name: Check the release tag matches the package version
|
|
30
|
+
run: |
|
|
31
|
+
version=$(python -c "import re;print(re.search(r'__version__ = \"(.+)\"', open('src/amazon_scraper/__init__.py').read())[1])")
|
|
32
|
+
if [ "${GITHUB_REF_NAME#v}" != "$version" ]; then
|
|
33
|
+
echo "::error::tag $GITHUB_REF_NAME does not match __version__ $version"; exit 1
|
|
34
|
+
fi
|
|
35
|
+
- name: Build the sdist and wheel
|
|
36
|
+
run: pip install build twine && python -m build && twine check --strict dist/*
|
|
37
|
+
- name: Publish to PyPI
|
|
38
|
+
uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
|