brightdata-amazon-scraper 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ # Copy to .env in the project root. The SDK finds it when the virtualenv is inside the project.
2
+ # Token: https://brightdata.com/cp/setting/users
3
+ BRIGHTDATA_API_TOKEN=
@@ -0,0 +1,124 @@
1
+ # Runs by itself on every push and pull request.
2
+ # Never calls the Instagram API, so it needs no token and costs nothing.
3
+ name: Tests (automatic, no API calls, free)
4
+
5
+ on:
6
+ push:
7
+ pull_request:
8
+ workflow_dispatch: # a Run workflow button, to re-run the tests without a commit
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ tests:
15
+ name: Run the tests and the code style check
16
+ runs-on: ubuntu-latest
17
+ timeout-minutes: 10
18
+ steps:
19
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
20
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
21
+ with:
22
+ python-version: "3.12"
23
+ - name: Install the package and its test tools
24
+ run: pip install -e ".[dev]"
25
+ - name: Check code style with ruff
26
+ run: ruff check .
27
+ - name: Run the tests
28
+ run: pytest
29
+ - name: README headings are well formed
30
+ # A section rewrite once produced "## The data## The data" and shipped;
31
+ # a heading with a second run of hashes mid-line is never intended.
32
+ run: |
33
+ if grep -nE '^#+ .*#{2,} ' README.md; then
34
+ echo "::error::malformed heading in README.md"; exit 1
35
+ fi
36
+
37
+ # These install steps shipped broken once, because they were only ever run
38
+ # from a working copy that already had the package. This job follows the
39
+ # README on an empty machine and stops where the tool asks for a token, which
40
+ # proves the clone, the install and the command without spending money.
41
+ coldstart:
42
+ name: Follow the README from scratch on an empty machine
43
+ runs-on: ubuntu-latest
44
+ timeout-minutes: 10
45
+ steps:
46
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
47
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
48
+ with:
49
+ python-version: "3.12"
50
+ - name: Copy the install steps out of the README and run them
51
+ run: |
52
+ set -o pipefail
53
+ python - <<'PY' > "$RUNNER_TEMP/quickstart.sh"
54
+ import os
55
+ import pathlib
56
+ import re
57
+ import sys
58
+
59
+ readme = pathlib.Path("README.md").read_text(encoding="utf-8")
60
+
61
+ def section(title):
62
+ return readme.split(f"## {title}", 1)[1].split("\n## ", 1)[0]
63
+
64
+ def block(text, lang):
65
+ return re.search(r"```" + lang + r"\n(.*?)```", text, re.S).group(1)
66
+
67
+ quick = section("How to scrape Amazon product data with Python")
68
+ install, snippet = block(quick, "bash"), block(quick, "python")
69
+ command = block(section("Bulk Amazon scraper CLI"), "bash")
70
+ for required, where in (("pip install brightdata-sdk", install),
71
+ ("SyncBrightDataClient", snippet),
72
+ ("amazon-scraper", command)):
73
+ if required not in where:
74
+ sys.exit(f"the README no longer contains {required!r} where the cold start expects it")
75
+
76
+ steps = ["# --- Quickstart: the SDK path ---"]
77
+ steps += [l for l in install.splitlines() if l.startswith("pip install")]
78
+ pathlib.Path(os.environ["RUNNER_TEMP"], "quickstart_snippet.py").write_text(snippet, encoding="utf-8")
79
+ steps += [
80
+ "set +e",
81
+ "python quickstart_snippet.py > out.txt 2> err.txt",
82
+ "code=$?",
83
+ "set -e",
84
+ 'test "$code" != "0" || { echo "the snippet ran without a token?"; cat out.txt; exit 1; }',
85
+ 'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
86
+ 'echo "Quickstart: installs, and stops at the token, as documented."',
87
+ "# --- Or run it as a command: the CLI path ---",
88
+ ]
89
+ for line in command.splitlines():
90
+ if line.startswith("amazon-scraper"):
91
+ continue # run below, so the exit code can be checked
92
+ if line.startswith("pip install") and "git+" in line:
93
+ # The repository is public, so this installs the exact line a
94
+ # reader pastes. On a private fork the URL is unreachable, and
95
+ # the check falls back to the checkout on its own.
96
+ url = line.split("git+", 1)[1].strip() if "git+" in line else ""
97
+ line = "\n".join([
98
+ f'repo="{url}"',
99
+ 'code=$(curl -s -o /dev/null -w "%{http_code}" "$repo" || echo 000)',
100
+ 'if [ "$code" = "200" ]; then',
101
+ f' echo "$repo is public: installing the exact README line"',
102
+ f' {line}',
103
+ "else",
104
+ ' echo "NOTE: $repo returns HTTP $code, so it is not public yet."',
105
+ ' echo "Installing from this checkout instead."',
106
+ ' pip install "$GITHUB_WORKSPACE"',
107
+ "fi",
108
+ ])
109
+ steps.append(line)
110
+ steps += [
111
+ "set +e",
112
+ "amazon-scraper B0CRMZHDG8 > out.txt 2> err.txt",
113
+ "code=$?",
114
+ "set -e",
115
+ 'test "$code" = "2" || { echo "expected exit 2, got $code"; cat err.txt; exit 1; }',
116
+ 'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
117
+ 'echo "Success: a new user can follow both README paths from scratch."',
118
+ ]
119
+ print("\n".join(steps))
120
+ PY
121
+ echo "--- these commands came out of README.md ---"
122
+ cat "$RUNNER_TEMP/quickstart.sh"
123
+ echo "--- running them on this empty machine ---"
124
+ cd "$RUNNER_TEMP" && bash -e "$RUNNER_TEMP/quickstart.sh"
@@ -0,0 +1,439 @@
1
+ # The only workflow that calls the real API. Needs the BRIGHTDATA_API_TOKEN
2
+ # secret and spends about one credit per product.
3
+ #
4
+ # Three ways it runs:
5
+ # on every push to main: one product and the README's links, one credit
6
+ # and about a minute, so a broken commit shows up at once. A newer push
7
+ # cancels a run still going for an older commit.
8
+ # by hand, from the Actions tab, to scrape whatever products you choose
9
+ # every Monday at 07:20 UTC: one product, the async path, and the
10
+ # field table, about two credits
11
+ # in the same Monday run: every code block in the README,
12
+ # about 12 credits
13
+ #
14
+ # A slow day at the API shows up as a red run. Rerun it before reading more
15
+ # into it.
16
+ #
17
+ # The Monday run ends by rewriting the "last verified" badge line at
18
+ # the top of README.md with today's date, and committing it if it changed. Push
19
+ # runs never do this, so your own push is never followed by a bot commit. Only
20
+ # the main repository commits; a copy of the repository just runs the checks.
21
+ name: Live check
22
+
23
+ run-name: "${{ github.event.schedule == '20 7 * * 1' && 'Weekly check: does every README block still work?' || github.event_name == 'schedule' && 'Daily check: is anything broken?' || github.event_name == 'push' && format('Push check: {0}', github.event.head_commit.message) || format('Scrape {0}', inputs.asins) }}"
24
+
25
+ on:
26
+ push:
27
+ branches: [main]
28
+ schedule:
29
+ - cron: "20 7 * * 1" # Monday: the whole README. Staggered across the
30
+ # scraper repos (06:00, 06:20, 06:40, 07:00, 07:20) so four full matrices never
31
+ # hit one account at once, which is what trips "too many requests".
32
+ # The two `== '20 7 * * 1'` checks below must match this line exactly.
33
+ workflow_dispatch:
34
+ inputs:
35
+ asins:
36
+ description: Amazon ASINs or product URLs, separated by spaces.
37
+ default: B0CRMZHDG8 B085DVHQ57
38
+ validate:
39
+ description: Also run every README block, as the Monday run does
40
+ type: boolean
41
+ default: false
42
+ only:
43
+ description: Block numbers to run, comma separated, for a cheap check. Empty means all.
44
+ default: ""
45
+
46
+ permissions:
47
+ contents: write # the weekly check commits the regenerated field table
48
+
49
+ concurrency:
50
+ group: live-${{ github.event_name }}-${{ github.ref }}
51
+ cancel-in-progress: ${{ github.event_name == 'push' }} # only the newest commit matters
52
+
53
+ jobs:
54
+ scrape:
55
+ name: Fetch the products and attach them to this run
56
+ runs-on: ubuntu-latest
57
+ timeout-minutes: 20 # fetch, async path, table; the README blocks have their own jobs
58
+ env:
59
+ BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
60
+ # On the weekly schedule there are no inputs, so fall back to one cheap product.
61
+ ASINS: ${{ inputs.asins || 'B0CRMZHDG8' }}
62
+ # the async path and the field table run on every scheduled run and on request
63
+ CHECK: ${{ github.event_name == 'schedule' || inputs.validate }}
64
+ steps:
65
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
66
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
67
+ with:
68
+ python-version: "3.12"
69
+ - name: Install the package and the Bright Data CLI
70
+ run: |
71
+ pip install -e .
72
+ npm i -g @brightdata/cli@0.4.0 # pinned: this job holds the API secret
73
+
74
+ - name: Do the README's links still resolve?
75
+ run: |
76
+ public=$(curl -s -o /dev/null -w '%{http_code}' https://github.com/brightdata/amazon-scraper)
77
+ failed=0
78
+ for url in $(grep -ohE 'https?://[^ )>"`]+' README.md AGENTS.md | sed 's/[.,]$//' | sort -u); do
79
+ case "$url" in
80
+ *amazon.com/*) continue ;; # rate-limits runners; the weekly README run fetches these products through the API anyway
81
+ *zendesk.com/*) continue ;; # answers 403 to anything that is not a browser
82
+ *mcp.brightdata.com/*) continue ;; # carries a placeholder token; nothing to check without a real one
83
+ *api.brightdata.com/*) continue ;; # needs the token; the field-table step below calls it for real
84
+ *github.com/brightdata/amazon-scraper*) [ "$public" = "200" ] || continue ;; # unreachable on a private fork
85
+ esac
86
+ code=$(curl -s -o /dev/null -L --retry 3 --max-time 30 -A "Mozilla/5.0" -w '%{http_code}' "$url" || echo 000)
87
+ if [ "$code" = "200" ]; then echo "200 $url"; else echo "::error::$code $url"; failed=1; fi
88
+ done
89
+ [ "$failed" = 0 ] || exit 1
90
+
91
+ - name: Fetch the products from Amazon
92
+ run: |
93
+ test -n "$BRIGHTDATA_API_TOKEN" || {
94
+ echo "No API token. Add BRIGHTDATA_API_TOKEN under" >&2
95
+ echo "Settings, Secrets and variables, Actions, then start this again." >&2
96
+ exit 1
97
+ }
98
+ # One retry, only on an API timeout. A slow day is not a broken build.
99
+ set +e
100
+ python -m amazon_scraper $ASINS 2>&1 | tee run.log
101
+ code=${PIPESTATUS[0]}
102
+ if [ "$code" != "0" ] && grep -q "timeout" run.log; then
103
+ echo "::warning::The API timed out. Waiting 30s and trying once more."
104
+ sleep 30
105
+ python -m amazon_scraper $ASINS 2>&1 | tee run.log
106
+ code=${PIPESTATUS[0]}
107
+ fi
108
+ exit "$code"
109
+
110
+ - name: Show what was found
111
+ if: always()
112
+ run: |
113
+ set -o pipefail
114
+ test -f amazon.json || exit 0
115
+ python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
116
+ import json
117
+
118
+ doc = json.load(open("amazon.json"))
119
+ print(f"Run at {doc['generated_at']}\n")
120
+ print("| asin | fields | price | rating | reviews |")
121
+ print("| --- | --- | --- | --- | --- |")
122
+ for entry in doc["products"]:
123
+ p = entry["product"] or {}
124
+ print(
125
+ f"| {entry['asin']} | {len(p)} | {p.get('final_price', '-')} {p.get('currency', '')} "
126
+ f"| {p.get('rating', '-')} | {p.get('reviews_count', '-')} |"
127
+ )
128
+ first = next((e["product"] for e in doc["products"] if e["product"]), None)
129
+ if first:
130
+ print("\n<details><summary>First record in full</summary>\n")
131
+ print("```json")
132
+ print(json.dumps(first, indent=2, ensure_ascii=False))
133
+ print("```\n")
134
+ print("</details>")
135
+ PY
136
+
137
+ # The two smoke checks. The products path is already covered by the step above.
138
+ - name: Does the async trigger, status, fetch path still work?
139
+ if: env.CHECK == 'true'
140
+ run: |
141
+ python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
142
+ import time
143
+
144
+ from brightdata import SyncBrightDataClient
145
+
146
+ url = "https://www.amazon.com/dp/B0CRMZHDG8"
147
+ with SyncBrightDataClient(auto_create_zones=False) as c:
148
+ job = c.scrape.amazon.products_trigger(url)
149
+ sid = job.snapshot_id
150
+ for _ in range(72): # up to 6 minutes
151
+ status = c.scrape.amazon.products_status(sid)
152
+ if status in ("ready", "failed", "canceled"):
153
+ break
154
+ time.sleep(5)
155
+ assert status == "ready", f"snapshot {sid} ended as {status}"
156
+ rows = c.scrape.amazon.products_fetch(sid)
157
+ rows = rows if isinstance(rows, list) else [rows]
158
+ assert rows and rows[0].get("asin"), "fetch returned no record"
159
+ print(f"\nAsync path ok: trigger, status, fetch returned {len(rows)} record(s).")
160
+ PY
161
+
162
+ - name: Regenerate the README field table from the live schema
163
+ if: always() && env.CHECK == 'true' && github.repository == 'brightdata/amazon-scraper'
164
+ run: |
165
+ python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
166
+ import json
167
+ import os
168
+ import pathlib
169
+ import re
170
+ import urllib.request
171
+
172
+ readme = pathlib.Path("README.md")
173
+ text = readme.read_text(encoding="utf-8")
174
+ start, end = "<!-- fields:start -->", "<!-- fields:end -->"
175
+ assert start in text and end in text, "README field-table markers are missing"
176
+
177
+ # The raw endpoint, not get_metadata(): the SDK drops the pii flag on
178
+ # every field when it parses this response (sdk-python#61).
179
+ request = urllib.request.Request(
180
+ "https://api.brightdata.com/datasets/gd_l7q7dkf244hwjntr0/metadata",
181
+ headers={"Authorization": f"Bearer {os.environ['BRIGHTDATA_API_TOKEN']}"},
182
+ )
183
+ fields = json.load(urllib.request.urlopen(request, timeout=60))["fields"]
184
+ sample = json.load(open("examples/sample_output.json"))["products"][0]["product"]
185
+ # A product carries a subset of the schema, and the API also returns
186
+ # fields the schema never lists, so these two counts differ.
187
+ known = sum(1 for name in fields if name in sample)
188
+ extra = [k for k in sample if k not in fields]
189
+
190
+ rows = ["| field | type | description |", "| --- | --- | --- |"]
191
+ for name, spec in fields.items():
192
+ desc = (spec.get("description") or "").replace(" — ", ": ").replace("—", ":")
193
+ desc = " ".join(desc.split()).replace("`", "'").replace("|", "/") # no fences, no table breaks
194
+ pii = "Personal data. " if spec.get("pii") else ""
195
+ rows.append(f"| `{name}` | {spec.get('type')} | {pii}{desc} |")
196
+ tail = f", plus {' and '.join(f'`{k}`' for k in extra)}," if extra else "."
197
+ block = "\n".join([
198
+ start,
199
+ "<details>",
200
+ f"<summary>All {len(fields)} fields, with type and description</summary>",
201
+ "",
202
+ "Regenerated every week from the dataset schema, via the raw metadata",
203
+ "endpoint, so it cannot go stale. A",
204
+ f"product carries the fields that apply to it: the sample file has {known}",
205
+ f"of these {len(fields)}{tail}",
206
+ *(["which the schema does not list."] if extra else []),
207
+ "",
208
+ *rows,
209
+ "",
210
+ "</details>",
211
+ end,
212
+ ])
213
+ new = re.sub(re.escape(start) + r".*?" + re.escape(end), lambda _: block, text, flags=re.S)
214
+ readme.write_text(new, encoding="utf-8")
215
+ state = "updated" if new != text else "already current"
216
+ print(f"\nSchema: {len(fields)} fields. README table {state}.")
217
+ PY
218
+ if git diff --quiet README.md; then
219
+ echo "Nothing to commit."
220
+ exit 0
221
+ fi
222
+ # Never commit a README the tests would reject.
223
+ pip install -q -e ".[dev]" && pytest -q
224
+ git config user.name "anil-bd"
225
+ git config user.email "210667061+anil-bd@users.noreply.github.com"
226
+ git add README.md
227
+ git commit -m "Regenerate the field table from the live schema (automated weekly check)"
228
+ git pull --rebase origin main
229
+ git push
230
+ echo "Committed the regenerated table."
231
+
232
+ - name: Attach amazon.json so you can download it
233
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
234
+ if: always()
235
+ with:
236
+ name: amazon-json
237
+ path: amazon.json
238
+ if-no-files-found: warn
239
+
240
+ # Every code block in the README, each in its own job. Mondays, and on request.
241
+ plan:
242
+ name: List the README blocks
243
+ if: github.event.schedule == '20 7 * * 1' || inputs.validate == true
244
+ runs-on: ubuntu-latest
245
+ timeout-minutes: 5
246
+ outputs:
247
+ blocks: ${{ steps.list.outputs.blocks }}
248
+ steps:
249
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
250
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
251
+ with:
252
+ python-version: "3.12"
253
+ - id: list
254
+ env:
255
+ ONLY: ${{ inputs.only }}
256
+ run: |
257
+ python - <<'PY' >> "$GITHUB_OUTPUT"
258
+ import os
259
+ import json
260
+ import pathlib
261
+ import re
262
+
263
+ readme = pathlib.Path("README.md").read_text(encoding="utf-8")
264
+ # never execute anything inside the generated field table: its text comes from
265
+ # the API, and a fence in a description must not become a command
266
+ readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
267
+ SKIP = ("pip install", "export ", "npm i", "npm install",
268
+ "npx -p @brightdata/cli bdata login", "bdata login")
269
+ blocks, heading, seen = [], "README", {}
270
+ for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
271
+ if m.group(1):
272
+ heading = m.group(1).strip() # a block is named after its section, not its first line
273
+ continue
274
+ lang, code = m.group(2), m.group(3)
275
+ lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")]
276
+ if lang == "bash":
277
+ lines = [l for l in lines if not l.startswith(SKIP)]
278
+ if not lines:
279
+ continue
280
+ code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines)
281
+ seen[heading] = seen.get(heading, 0) + 1
282
+ title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
283
+ blocks.append({"lang": lang, "title": title[:50], "code": code})
284
+ wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()}
285
+ out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]} for i, b in enumerate(blocks) if not wanted or i + 1 in wanted]
286
+ print("blocks=" + json.dumps(out))
287
+ PY
288
+
289
+ block:
290
+ name: "${{ matrix.n }}. ${{ matrix.title }}"
291
+ needs: plan
292
+ runs-on: ubuntu-latest
293
+ timeout-minutes: 15 # a block gives up at 8 minutes; past this it is stuck
294
+ strategy:
295
+ fail-fast: false
296
+ # Three at a time. The API stalls in waves, and it throttles an account
297
+ # that sends many requests at once. The cap does not fix either;
298
+ # it limits how much of the matrix one bad stretch can take down.
299
+ max-parallel: 3
300
+ matrix:
301
+ include: ${{ fromJson(needs.plan.outputs.blocks) }}
302
+ env:
303
+ BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
304
+ steps:
305
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
306
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
307
+ with:
308
+ python-version: "3.12"
309
+ - name: Install the package and, for CLI blocks, the Bright Data CLI
310
+ run: |
311
+ pip install -e .
312
+ if [ "${{ matrix.lang }}" = "bash" ]; then npm i -g @brightdata/cli@0.4.0; fi
313
+ - name: Run block ${{ matrix.n }} exactly as the README shows it
314
+ env:
315
+ N: ${{ matrix.n }}
316
+ run: |
317
+ set -o pipefail # without this, tee's exit code hides a failing block
318
+ python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
319
+ import os
320
+ import subprocess
321
+ import sys
322
+ import time
323
+ import pathlib
324
+ import re
325
+
326
+ readme = pathlib.Path("README.md").read_text(encoding="utf-8")
327
+ readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
328
+ SKIP = ("pip install", "export ", "npm i", "npm install",
329
+ "npx -p @brightdata/cli bdata login", "bdata login")
330
+ blocks, heading, seen = [], "README", {}
331
+ for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
332
+ if m.group(1):
333
+ heading = m.group(1).strip()
334
+ continue
335
+ lang, code = m.group(2), m.group(3)
336
+ lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")]
337
+ if lang == "bash":
338
+ lines = [l for l in lines if not l.startswith(SKIP)]
339
+ if not lines:
340
+ continue
341
+ code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines)
342
+ seen[heading] = seen.get(heading, 0) + 1
343
+ title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
344
+ blocks.append({"lang": lang, "title": title[:50], "code": code})
345
+ b = blocks[int(os.environ["N"]) - 1]
346
+ argv = ["bash", "-e", "-c", b["code"]] if b["lang"] == "bash" else [sys.executable, "-c", b["code"]]
347
+ secret = os.environ.get("BRIGHTDATA_API_TOKEN", "")
348
+ status = None
349
+ for attempt in (1, 2):
350
+ started = time.time()
351
+ try:
352
+ run = subprocess.run(argv, capture_output=True, text=True, timeout=480)
353
+ except subprocess.TimeoutExpired:
354
+ run = subprocess.CompletedProcess(argv, 1, "", "block timeout: no result within 8 minutes")
355
+ secs = round(time.time() - started)
356
+ status = run.returncode
357
+ out, err = run.stdout.strip(), run.stderr.strip()
358
+ if secret: # summaries are not secret-masked the way logs are
359
+ out, err = out.replace(secret, "***"), err.replace(secret, "***")
360
+ ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err)
361
+ if ok or ((out or err) and "timeout" not in (out + err).lower()):
362
+ break
363
+ time.sleep(30) # the API timed out or answered with nothing at all; one retry
364
+ # A block that exits 0 having printed nothing is not a crash: the call
365
+ # ran and returned no rows. Say so, or the next reader hunts for one.
366
+ if ok:
367
+ shown = [(out.splitlines() or [""])[0]]
368
+ elif status == 0 and not out and not err:
369
+ shown = ["exited 0 and printed nothing: the call returned no rows"]
370
+ else:
371
+ # The last line alone hides the cause. For the CLI it is the
372
+ # "Saved 1 of 2" summary, and the line saying which input failed
373
+ # and why sits above it. Show the failure lines, else the tail.
374
+ lines = [l for l in (out + "\n" + err).splitlines() if l.strip()]
375
+ reasons = [l for l in lines if l.startswith("failed ") or "Error" in l]
376
+ shown = (reasons or lines or [f"no output, exit {status}"])[-6:]
377
+ body = "\n".join(f" {line[:160]}" for line in shown)
378
+ print(f"\n`{b['title']}`: {'ok' if ok else 'FAILED'} in {secs}s\n\n{body}")
379
+ sys.exit(0 if ok else 1)
380
+ PY
381
+
382
+ badge:
383
+ name: Put today's date on the README's "last verified" badge
384
+ needs: [scrape, plan, block]
385
+ # scheduled and manual runs only, and only in the main repository: a push is
386
+ # never followed by a bot commit, and a copy never diverges from the original.
387
+ #
388
+ # `inputs.only` empty as well: a run of one block that passes is not the
389
+ # README verified. A scheduled run has no inputs, where `inputs.only` is null
390
+ # and this comparison is still true, so the Monday run sets it.
391
+ if: always() && !cancelled() && github.event_name != 'push' && inputs.only == '' && github.repository == 'brightdata/amazon-scraper'
392
+ permissions:
393
+ contents: write # commits the badge line
394
+ actions: read # counts this run's job results
395
+ runs-on: ubuntu-latest
396
+ timeout-minutes: 5
397
+ steps:
398
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
399
+ with:
400
+ # The tip of main, not this run's commit. A re-run checks out the
401
+ # commit the run first started on, but an earlier attempt of the same
402
+ # run may already have pushed a badge edit to this very line. Editing
403
+ # from that stale base makes the rebase below stop on a conflict. From
404
+ # the tip, the line is replaced, never merged. This broke the Node twin
405
+ # on 2026-09-18 when a failed run was re-run.
406
+ ref: main
407
+ - name: Rewrite one badge line, commit only if it changed
408
+ env:
409
+ GH_TOKEN: ${{ github.token }}
410
+ run: |
411
+ # Two states: red only when every check in this run failed. One passing
412
+ # check keeps the badge green; the run itself still shows which failed.
413
+ read -r passed failed < <(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
414
+ --jq '[.jobs[] | select((.name | startswith("Put today") or startswith("Open or update") or . == "List the README blocks") | not)] as $j | "\([$j[] | select(.conclusion == "success")] | length) \([$j[] | select(.conclusion == "failure")] | length)"') || true
415
+ echo "checks passed: ${passed:-0}, failed: ${failed:-0}"
416
+ if [ "${passed:-0}" -gt 0 ]; then
417
+ label="last verified"; color="brightgreen"
418
+ else
419
+ label="last check failed"; color="red"
420
+ fi
421
+ today=$(date -u +'%-d %b %Y')
422
+ line="[![${label}](https://img.shields.io/badge/${label// /%20}-${today// /%20}-${color})](https://github.com/brightdata/amazon-scraper/actions/workflows/live.yml) <!-- verified: rewritten by the weekly run -->"
423
+ python3 - "$line" <<'PY'
424
+ import pathlib, re, sys
425
+ p = pathlib.Path("README.md"); t = p.read_text(encoding="utf-8")
426
+ new, n = re.subn(r"^\[!\[last [^\]]*\]\(https://img\.shields\.io/badge/[^)]*\)\]\([^)]*\) <!-- verified:[^\n]*$", sys.argv[1], t, count=1, flags=re.M)
427
+ if n != 1:
428
+ sys.exit("the badge line is missing from README.md")
429
+ p.write_text(new, encoding="utf-8")
430
+ PY
431
+ if git diff --quiet; then echo "badge already shows $today"; exit 0; fi
432
+ git config user.name "anil-bd"
433
+ git config user.email "210667061+anil-bd@users.noreply.github.com"
434
+ # $label, not a fixed "Last verified": the log should not claim a
435
+ # check passed on a day the badge went red.
436
+ git commit -qam "${label^} $today (automated weekly check)"
437
+ git pull -q --rebase origin main
438
+ git push -q origin main
439
+ echo "badge now shows: $label, $today"
@@ -0,0 +1,38 @@
1
+ # Publishes the package to PyPI when a GitHub Release is published.
2
+ # Uses PyPI Trusted Publishing (OIDC), so the repo stores no PyPI token.
3
+ # Release steps: bump __version__ in src/amazon_scraper/__init__.py,
4
+ # add a CHANGELOG.md entry, then publish a GitHub Release tagged v<version>.
5
+ name: Publish to PyPI
6
+
7
+ on:
8
+ release:
9
+ types: [published]
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ publish:
16
+ name: Build and publish to PyPI
17
+ runs-on: ubuntu-latest
18
+ timeout-minutes: 10
19
+ environment:
20
+ name: pypi
21
+ url: https://pypi.org/p/brightdata-amazon-scraper
22
+ permissions:
23
+ id-token: write # required for Trusted Publishing
24
+ steps:
25
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
26
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
27
+ with:
28
+ python-version: "3.12"
29
+ - name: Check the release tag matches the package version
30
+ run: |
31
+ version=$(python -c "import re;print(re.search(r'__version__ = \"(.+)\"', open('src/amazon_scraper/__init__.py').read())[1])")
32
+ if [ "${GITHUB_REF_NAME#v}" != "$version" ]; then
33
+ echo "::error::tag $GITHUB_REF_NAME does not match __version__ $version"; exit 1
34
+ fi
35
+ - name: Build the sdist and wheel
36
+ run: pip install build twine && python -m build && twine check --strict dist/*
37
+ - name: Publish to PyPI
38
+ uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
@@ -0,0 +1,7 @@
1
+ .env
2
+ .venv/
3
+ __pycache__/
4
+ *.egg-info/
5
+ .pytest_cache/
6
+ .ruff_cache/
7
+ amazon.json