brightdata-chatgpt-scraper 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ # Copy to .env, then load it into the shell: set -a; . ./.env; set +a
2
+ # Token: https://brightdata.com/cp/setting/users
3
+ BRIGHTDATA_API_TOKEN=
@@ -0,0 +1,72 @@
1
+ """Rewrite the README's field tables from the published scraper schemas.
2
+
3
+ python .github/scripts/field_tables.py
4
+
5
+ No token needed: this reads
6
+ https://docs.brightdata.com/scrapers-full.json, the list the Bright Data MCP
7
+ server also loads: every scraper's output fields, with type and description.
8
+ """
9
+
10
+ import json
11
+ import pathlib
12
+ import re
13
+ import urllib.request
14
+
15
+ ROOT = pathlib.Path(__file__).resolve().parents[2]
16
+ README = ROOT / "README.md"
17
+ START, END = "<!-- fields:start -->", "<!-- fields:end -->"
18
+ SOURCE = "https://docs.brightdata.com/scrapers-full.json"
19
+
20
+ #: The README's order. Each scraper's collect_by_url method lists its full record.
21
+ SCRAPERS = [
22
+ ("gd_m7aof0k82r803d5bjm", "ChatGPT answer"),
23
+ ]
24
+
25
+
26
+ def table(fields):
27
+ rows = ["| field | type | description |", "| --- | --- | --- |"]
28
+ for field in fields:
29
+ desc = (field.get("description") or "").replace(" — ", ": ").replace("—", ":")
30
+ # no fences, no table breaks
31
+ desc = " ".join(desc.split()).replace("`", "'").replace("|", "/")
32
+ rows.append(f"| `{field['name']}` | {field.get('type', '')} | {desc} |")
33
+ return rows
34
+
35
+
36
+ def main():
37
+ text = README.read_text(encoding="utf-8")
38
+ assert START in text and END in text, "README field-table markers are missing"
39
+ with urllib.request.urlopen(SOURCE, timeout=120) as response:
40
+ published = {s["id"]: s for s in json.load(response)}
41
+ sample = json.loads((ROOT / "examples" / "sample_output.json").read_text(encoding="utf-8"))
42
+ first = sample["prompts"][0]["answer"]
43
+ lines = [
44
+ START,
45
+ "Regenerated every week from the published schema,",
46
+ f"[scrapers-full.json]({SOURCE}), so they cannot go stale. A record",
47
+ "carries the fields that apply to it, plus `timestamp` and `input`; the",
48
+ f"sample file's first answer has {len(first)} keys.",
49
+ ]
50
+ counts = []
51
+ for dataset, label in SCRAPERS:
52
+ fields = published[dataset]["scrapers"]["collect_by_url"]["output_fields"]
53
+ counts.append(len(fields))
54
+ lines += [
55
+ "",
56
+ "<details>",
57
+ f"<summary>All {len(fields)} {label} fields, <code>{dataset}</code></summary>",
58
+ "",
59
+ *table(fields),
60
+ "",
61
+ "</details>",
62
+ ]
63
+ lines.append(END)
64
+ new = re.sub(re.escape(START) + r".*?" + re.escape(END), lambda _: "\n".join(lines), text,
65
+ flags=re.S)
66
+ README.write_text(new, encoding="utf-8")
67
+ state = "updated" if new != text else "already current"
68
+ print(f"\nSchemas: {len(SCRAPERS)} scrapers, {sum(counts)} fields. Tables {state}.")
69
+
70
+
71
+ if __name__ == "__main__":
72
+ main()
@@ -0,0 +1,138 @@
1
+ """The README's code blocks, as the live check runs them.
2
+
3
+ python readme_blocks.py list # JSON matrix for the block jobs (ONLY filters)
4
+ python readme_blocks.py run N # run block N with the real token, one retry
5
+ python readme_blocks.py cli-syntax # every README CLI line with a dummy key
6
+
7
+ One parser for all three, so the list and the runner can never disagree on
8
+ what block N is.
9
+ """
10
+
11
+ import json
12
+ import os
13
+ import pathlib
14
+ import re
15
+ import subprocess
16
+ import sys
17
+ import time
18
+
19
+ README = pathlib.Path(__file__).resolve().parents[2] / "README.md"
20
+ SKIP = ("pip install", "export ", "npm i", "npm install",
21
+ "npx -p @brightdata/cli bdata login", "bdata login")
22
+ CLI = os.environ.get("BDATA_CLI", "") # the pinned CLI version; empty means whatever is installed
23
+
24
+
25
+ def blocks():
26
+ readme = README.read_text(encoding="utf-8")
27
+ # never execute anything inside the generated field table: its text comes from
28
+ # the API, and a fence in a description must not become a command
29
+ readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
30
+ found, heading, seen = [], "README", {}
31
+ for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
32
+ if m.group(1):
33
+ heading = m.group(1).strip() # a block is named after its section, not its first line
34
+ continue
35
+ lang, code = m.group(2), m.group(3)
36
+ lines = [line for line in code.splitlines()
37
+ if line.strip() and not line.strip().startswith("#")]
38
+ if lang == "bash":
39
+ lines = [line for line in lines if not line.startswith(SKIP)]
40
+ if not lines:
41
+ continue
42
+ code = "\n".join(lines)
43
+ seen[heading] = seen.get(heading, 0) + 1
44
+ title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
45
+ found.append({"lang": lang, "title": title[:50], "code": code})
46
+ return found
47
+
48
+
49
+ def cli_lines(code, key):
50
+ """The CLI cannot open a browser here, so the key goes in by flag, and an
51
+ unpinned npx is held to the version the job installed."""
52
+ out = []
53
+ for line in code.splitlines():
54
+ if CLI:
55
+ line = line.replace("npx -p @brightdata/cli ", f"npx -p @brightdata/cli@{CLI} ")
56
+ if "bdata pipelines" in line:
57
+ line += f" -k {key}"
58
+ out.append(line)
59
+ return "\n".join(out)
60
+
61
+
62
+ def list_blocks():
63
+ wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()}
64
+ out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]}
65
+ for i, b in enumerate(blocks()) if not wanted or i + 1 in wanted]
66
+ print("blocks=" + json.dumps(out))
67
+
68
+
69
+ def run_block(n):
70
+ b = blocks()[n - 1]
71
+ if b["lang"] == "bash":
72
+ # only the shell ever sees the key's value
73
+ argv = ["bash", "-e", "-c", cli_lines(b["code"], '"$BRIGHTDATA_API_TOKEN"')]
74
+ else:
75
+ argv = [sys.executable, "-c", b["code"]]
76
+ secret = os.environ.get("BRIGHTDATA_API_TOKEN", "")
77
+ for _attempt in (1, 2):
78
+ started = time.time()
79
+ try:
80
+ run = subprocess.run(argv, capture_output=True, text=True, timeout=1080)
81
+ except subprocess.TimeoutExpired:
82
+ run = subprocess.CompletedProcess(
83
+ argv, 1, "", "block timeout: no result within 18 minutes")
84
+ secs = round(time.time() - started)
85
+ out, err = run.stdout.strip(), run.stderr.strip()
86
+ if secret: # summaries are not secret-masked the way logs are
87
+ out, err = out.replace(secret, "***"), err.replace(secret, "***")
88
+ ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err)
89
+ if ok or ((out or err) and "timeout" not in (out + err).lower()):
90
+ break
91
+ time.sleep(30) # the API timed out or answered with nothing at all; one retry
92
+ if ok:
93
+ shown = (out.splitlines() or [""])[:1]
94
+ else:
95
+ # the last few lines: a CLI error often puts the cause above its hint
96
+ shown = (err.splitlines() or out.splitlines() or ["no output"])[-6:]
97
+ body = "\n".join(" " + line[:160] for line in shown)
98
+ state = "ok" if ok else "FAILED"
99
+ print(f"\n`{b['title']}`: {state} in {secs}s (exit {run.returncode})\n\n{body}")
100
+ sys.exit(0 if ok else 1)
101
+
102
+
103
+ def cli_syntax():
104
+ """Run every README CLI line with a key the API rejects. A line the CLI
105
+ still understands reaches the API and gets a 401; a line it no longer
106
+ understands fails before that. Free, and needs no secret."""
107
+ failed = 0
108
+ print("| block | CLI line | result |\n| --- | --- | --- |")
109
+ for i, b in enumerate(blocks(), 1):
110
+ if b["lang"] != "bash":
111
+ continue
112
+ for line in b["code"].splitlines():
113
+ if "bdata pipelines" not in line:
114
+ continue
115
+ run = subprocess.run(["bash", "-c", cli_lines(line, "invalid-key-for-syntax-check")],
116
+ capture_output=True, text=True, timeout=120)
117
+ text = run.stdout + run.stderr
118
+ ok = "401" in text or "Invalid credentials" in text
119
+ last = (run.stderr.strip().splitlines() or run.stdout.strip().splitlines()
120
+ or ["no output"])
121
+ why = "reached the API" if ok else " / ".join(x.strip() for x in last[-3:])[:200]
122
+ print(f"| {i} | `{line[:90]}` | {'ok' if ok else 'FAILED'}: {why.replace('|', '/')} |")
123
+ if not ok:
124
+ failed = 1
125
+ print(f"::error::README block {i}: the CLI rejects `{line}`", file=sys.stderr)
126
+ sys.exit(failed)
127
+
128
+
129
+ if __name__ == "__main__":
130
+ mode = sys.argv[1] if len(sys.argv) > 1 else ""
131
+ if mode == "list":
132
+ list_blocks()
133
+ elif mode == "run":
134
+ run_block(int(sys.argv[2]))
135
+ elif mode == "cli-syntax":
136
+ cli_syntax()
137
+ else:
138
+ sys.exit(__doc__)
@@ -0,0 +1,129 @@
1
+ # Runs by itself on every push and pull request.
2
+ # Never calls the ChatGPT Scraper API, so it needs no token and costs nothing.
3
+ name: Tests (automatic, no API calls, free)
4
+
5
+ on:
6
+ push:
7
+ pull_request:
8
+ workflow_dispatch: # a Run workflow button, to re-run the tests without a commit
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ concurrency: # a newer push to the same branch or PR replaces the run still going
14
+ group: ci-${{ github.event_name }}-${{ github.ref }}
15
+ cancel-in-progress: true
16
+
17
+ jobs:
18
+ tests:
19
+ name: Run the tests and the code style check
20
+ runs-on: ubuntu-latest
21
+ timeout-minutes: 10
22
+ steps:
23
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
24
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
25
+ with:
26
+ python-version: "3.12"
27
+ cache: pip
28
+ cache-dependency-path: pyproject.toml
29
+ - name: Install the package and its test tools
30
+ run: pip install -e ".[dev]"
31
+ - name: Check code style with ruff
32
+ run: ruff check .
33
+ - name: Run the tests
34
+ run: pytest
35
+ - name: README headings are well formed
36
+ # A section rewrite once produced "## The data## The data" and shipped;
37
+ # a heading with a second run of hashes mid-line is never intended.
38
+ run: |
39
+ if grep -nE '^#+ .*#{2,} ' README.md; then
40
+ echo "::error::malformed heading in README.md"; exit 1
41
+ fi
42
+
43
+ # These install steps shipped broken once, because they were only ever run
44
+ # from a working copy that already had the package. This job follows the
45
+ # README on an empty machine and stops where the tool asks for a token, which
46
+ # proves the clone, the install and the command without spending money.
47
+ coldstart:
48
+ name: Follow the README from scratch on an empty machine
49
+ runs-on: ubuntu-latest
50
+ timeout-minutes: 10
51
+ steps:
52
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
53
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
54
+ with:
55
+ python-version: "3.12"
56
+ - name: Copy the install steps out of the README and run them
57
+ run: |
58
+ set -o pipefail
59
+ python - <<'PY' > "$RUNNER_TEMP/quickstart.sh"
60
+ import os
61
+ import pathlib
62
+ import re
63
+ import sys
64
+
65
+ readme = pathlib.Path("README.md").read_text(encoding="utf-8")
66
+
67
+ def section(title):
68
+ return readme.split(f"## {title}", 1)[1].split("\n## ", 1)[0]
69
+
70
+ def block(text, lang):
71
+ return re.search(r"```" + lang + r"\n(.*?)```", text, re.S).group(1)
72
+
73
+ quick = section("How to scrape ChatGPT results with Python")
74
+ snippet = block(quick, "python")
75
+ command = block(section("Bulk ChatGPT prompt scraper CLI"), "bash")
76
+ for required, where in (("BRIGHTDATA_API_TOKEN", snippet),
77
+ ("urllib.request", snippet),
78
+ ("chatgpt-scraper", command)):
79
+ if required not in where:
80
+ sys.exit(f"the README no longer contains {required!r} where the cold start expects it")
81
+
82
+ steps = ["# --- Quickstart: the standard library path, nothing to install ---"]
83
+ pathlib.Path(os.environ["RUNNER_TEMP"], "quickstart_snippet.py").write_text(snippet, encoding="utf-8")
84
+ steps += [
85
+ "set +e",
86
+ "python quickstart_snippet.py > out.txt 2> err.txt",
87
+ "code=$?",
88
+ "set -e",
89
+ 'test "$code" != "0" || { echo "the snippet ran without a token?"; cat out.txt; exit 1; }',
90
+ 'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
91
+ 'echo "Quickstart: runs on a bare Python, and stops at the token, as documented."',
92
+ "# --- Or run it as a command: the CLI path ---",
93
+ ]
94
+ for line in command.splitlines():
95
+ if line.startswith("chatgpt-scraper"):
96
+ continue # run below, so the exit code can be checked
97
+ if line.startswith("pip install") and "git+" in line:
98
+ # The repository is public, so this installs the exact line a
99
+ # reader pastes. On a private fork the URL is unreachable, and
100
+ # the check falls back to the checkout on its own.
101
+ url = line.split("git+", 1)[1].strip() if "git+" in line else ""
102
+ line = "\n".join([
103
+ f'repo="{url}"',
104
+ 'code=$(curl -s -o /dev/null -w "%{http_code}" "$repo" || echo 000)',
105
+ 'if [ "$code" = "200" ]; then',
106
+ f' echo "$repo is public: installing the exact README line"',
107
+ f' {line}',
108
+ "else",
109
+ ' echo "NOTE: $repo returns HTTP $code, so it is not public yet."',
110
+ ' echo "Installing from this checkout instead."',
111
+ ' pip install "$GITHUB_WORKSPACE"',
112
+ "fi",
113
+ ])
114
+ steps.append(line)
115
+ steps += [
116
+ "set +e",
117
+ 'chatgpt-scraper "What is the capital of Australia?" > out.txt 2> err.txt',
118
+ "code=$?",
119
+ "set -e",
120
+ 'test "$code" = "2" || { echo "expected exit 2, got $code"; cat err.txt; exit 1; }',
121
+ 'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
122
+ 'echo "Success: a new user can follow both README paths from scratch."',
123
+ ]
124
+ print("\n".join(steps))
125
+ PY
126
+ echo "--- these commands came out of README.md ---"
127
+ cat "$RUNNER_TEMP/quickstart.sh"
128
+ echo "--- running them on this empty machine ---"
129
+ cd "$RUNNER_TEMP" && bash -e "$RUNNER_TEMP/quickstart.sh"
@@ -0,0 +1,332 @@
1
+ # The only workflow that calls the real API. Needs the BRIGHTDATA_API_TOKEN
2
+ # secret and is billed per answer.
3
+ #
4
+ # Three ways it runs:
5
+ # on every push to main: one answer to one prompt and the README's links,
6
+ # one record and about two minutes, so a broken commit shows up at once.
7
+ # A newer push cancels a run still going for an older commit.
8
+ # by hand, from the Actions tab, to ask whatever prompts you choose
9
+ # every Monday at 08:00 UTC: one answer, the async path, and the
10
+ # field tables, about two records
11
+ # in the same Monday run: every code block in the README,
12
+ # about 15 records and 10 minutes
13
+ #
14
+ # Every run also checks, for free and with no secret, that the newest Bright
15
+ # Data CLI still accepts every CLI line in the README. Readers run `npx -p
16
+ # @brightdata/cli`, which is always the newest release, while the jobs that hold
17
+ # the secret run the version pinned in BDATA_CLI below. When it warns about a
18
+ # new release, bump BDATA_CLI.
19
+ #
20
+ # A slow day at the API shows up as a red run. Rerun it before reading more
21
+ # into it.
22
+ #
23
+ # The Monday run ends by rewriting the "last verified" badge line at
24
+ # the top of README.md with today's date, and committing it if it changed. Push
25
+ # runs never do this, so your own push is never followed by a bot commit. Only
26
+ # the main repository commits; a copy of the repository just runs the checks.
27
+ name: Live check
28
+
29
+ run-name: "${{ github.event.schedule == '0 8 * * 1' && 'Weekly check: does every README block still work?' || github.event_name == 'schedule' && 'Daily check: is anything broken?' || github.event_name == 'push' && format('Push check: {0}', github.event.head_commit.message) || format('Ask {0}', inputs.prompts) }}"
30
+
31
+ on:
32
+ push:
33
+ branches: [main]
34
+ schedule:
35
+ - cron: "0 8 * * 1" # Monday: the whole README
36
+ workflow_dispatch:
37
+ inputs:
38
+ prompts:
39
+ description: Prompts, separated by commas
40
+ default: Best CRM software for small businesses, Best free photo editing app
41
+ validate:
42
+ description: Also run every README block, as the Monday run does
43
+ type: boolean
44
+ default: false
45
+ only:
46
+ description: Block numbers to run, comma separated, for a cheap check. Empty means all.
47
+ default: ""
48
+
49
+ permissions:
50
+ contents: write # the weekly check commits the regenerated field tables
51
+
52
+ env:
53
+ BDATA_CLI: "0.4.0" # pinned for the jobs that hold the API secret; bump after cli-syntax passes
54
+
55
+ concurrency:
56
+ group: live-${{ github.event_name }}-${{ github.ref }}
57
+ cancel-in-progress: ${{ github.event_name == 'push' }} # only the newest commit matters
58
+
59
+ jobs:
60
+ scrape:
61
+ name: Ask ChatGPT and attach the answers to this run
62
+ runs-on: ubuntu-latest
63
+ timeout-minutes: 45 # an answer took one to five minutes live; the README blocks have their own jobs
64
+ env:
65
+ BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
66
+ # On the weekly schedule there are no inputs, so fall back to one cheap prompt.
67
+ PROMPTS: ${{ inputs.prompts || 'Best CRM software for small businesses' }}
68
+ # the async path and the field tables run on every scheduled run and on request
69
+ CHECK: ${{ github.event_name == 'schedule' || inputs.validate }}
70
+ steps:
71
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
72
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
73
+ with:
74
+ python-version: "3.12"
75
+ cache: pip
76
+ cache-dependency-path: pyproject.toml
77
+ - name: Install the package
78
+ run: pip install -e .
79
+
80
+ - name: Do the README's links still resolve?
81
+ run: |
82
+ public=$(curl -s -o /dev/null -w '%{http_code}' https://github.com/brightdata/chatgpt-scraper)
83
+ failed=0
84
+ # links in prose only: fenced blocks hold code and scraped output, and the
85
+ # weekly run executes the code blocks anyway
86
+ for url in $(awk '/^```/{fenced=!fenced; next} !fenced' README.md AGENTS.md | grep -ohE 'https?://[^ )<>"`\\]+' | sed 's/[.,:]*$//' | sort -u); do
87
+ case "$url" in
88
+ *chatgpt.com/*) continue ;; # answers bot traffic with a challenge page
89
+ *zendesk.com/*) continue ;; # answers 403 to anything that is not a browser
90
+ *mcp.brightdata.com/*) continue ;; # carries a placeholder token; nothing to check without a real one
91
+ *api.brightdata.com/*) continue ;; # API endpoints answer 401 without a key; the README blocks call them
92
+ *github.com/brightdata/chatgpt-scraper*) [ "$public" = "200" ] || continue ;; # this repo and its Node twin, unreachable while private
93
+ esac
94
+ code=$(curl -s -o /dev/null -L --retry 3 --max-time 30 -A "Mozilla/5.0" -w '%{http_code}' "$url" || echo 000)
95
+ if [ "$code" = "200" ]; then echo "200 $url"; else echo "::error::$code $url"; failed=1; fi
96
+ done
97
+ [ "$failed" = 0 ] || exit 1
98
+ - name: Ask ChatGPT
99
+ run: |
100
+ test -n "$BRIGHTDATA_API_TOKEN" || {
101
+ echo "No API token. Add BRIGHTDATA_API_TOKEN under" >&2
102
+ echo "Settings, Secrets and variables, Actions, then start this again." >&2
103
+ exit 1
104
+ }
105
+ mapfile -t KW < <(python3 -c 'import os; [print(k.strip()) for k in os.environ["PROMPTS"].split(",") if k.strip()]')
106
+ # One retry, only on an API timeout. A slow day is not a broken build.
107
+ set +e
108
+ python -m chatgpt_scraper "${KW[@]}" 2>&1 | tee run.log
109
+ code=${PIPESTATUS[0]}
110
+ if [ "$code" != "0" ] && grep -qi "timeout" run.log; then
111
+ echo "::warning::The API timed out. Waiting 30s and trying once more."
112
+ sleep 30
113
+ python -m chatgpt_scraper "${KW[@]}" 2>&1 | tee run.log
114
+ code=${PIPESTATUS[0]}
115
+ fi
116
+ exit "$code"
117
+
118
+ - name: Show what was found
119
+ if: always()
120
+ run: |
121
+ set -o pipefail
122
+ test -f chatgpt.json || exit 0
123
+ python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
124
+ import json
125
+
126
+ doc = json.load(open("chatgpt.json"))
127
+ print(f"Run at {doc['generated_at']}, country {doc['country']}\n")
128
+ print("| prompt | characters | citations | model |")
129
+ print("| --- | --- | --- | --- |")
130
+ for entry in doc["prompts"]:
131
+ answer = entry["answer"] or {}
132
+ print(
133
+ f"| {entry['prompt']} | {len(answer.get('answer_text') or '')} "
134
+ f"| {len(answer.get('citations') or [])} | {answer.get('model')} |"
135
+ )
136
+ first = next((e["answer"] for e in doc["prompts"] if e["answer"]), None)
137
+ if first:
138
+ print("\n<details><summary>First record in full</summary>\n")
139
+ print("```json")
140
+ print(json.dumps(first, indent=2, ensure_ascii=False))
141
+ print("```\n")
142
+ print("</details>")
143
+ PY
144
+
145
+ # The two smoke checks. A live request is already covered by the step above.
146
+ - name: Does the async trigger, status, fetch path still work?
147
+ if: env.CHECK == 'true'
148
+ run: |
149
+ set -o pipefail
150
+ python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
151
+ import time
152
+
153
+ from chatgpt_scraper import CHATGPT, fetch, status, trigger
154
+
155
+ sid = trigger(CHATGPT, [{"url": "https://chatgpt.com/", "prompt": "What is the capital of Australia?"}])
156
+ for _ in range(90): # up to 15 minutes
157
+ state = status(sid)
158
+ if state in ("ready", "failed"):
159
+ break
160
+ time.sleep(10)
161
+ assert state == "ready", f"snapshot {sid} ended as {state}"
162
+ rows = fetch(sid)
163
+ assert rows and rows[0].get("answer_text"), f"fetch returned no answer: {rows[:1]}"
164
+ print(f"\nAsync path ok: trigger, status, fetch returned {len(rows)} record(s).")
165
+ PY
166
+
167
+ - name: Regenerate the README field tables from the live schema
168
+ if: always() && env.CHECK == 'true' && github.repository == 'brightdata/chatgpt-scraper' && github.ref == 'refs/heads/main'
169
+ run: |
170
+ python .github/scripts/field_tables.py | tee -a "$GITHUB_STEP_SUMMARY"
171
+ if git diff --quiet README.md; then
172
+ echo "Nothing to commit."
173
+ exit 0
174
+ fi
175
+ # Never commit a README the tests would reject.
176
+ pip install -q -e ".[dev]" && pytest -q
177
+ git config user.name "anil-bd"
178
+ git config user.email "210667061+anil-bd@users.noreply.github.com"
179
+ git add README.md
180
+ git commit -m "Regenerate the field tables from the live schema (automated weekly check)"
181
+ git pull --rebase origin main
182
+ git push
183
+ echo "Committed the regenerated tables."
184
+
185
+ - name: Attach chatgpt.json so you can download it
186
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
187
+ if: always()
188
+ with:
189
+ name: chatgpt-json
190
+ path: chatgpt.json
191
+ if-no-files-found: warn
192
+
193
+ # Every code block in the README, each in its own job, all at once. Mondays,
194
+ # and on request. Wall clock is the slowest block instead of the sum of all.
195
+ plan:
196
+ name: List the README blocks
197
+ if: github.event.schedule == '0 8 * * 1' || inputs.validate == true
198
+ runs-on: ubuntu-latest
199
+ outputs:
200
+ blocks: ${{ steps.list.outputs.blocks }}
201
+ steps:
202
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
203
+ - id: list
204
+ env:
205
+ ONLY: ${{ inputs.only }}
206
+ run: python .github/scripts/readme_blocks.py list >> "$GITHUB_OUTPUT"
207
+
208
+ block:
209
+ name: "${{ matrix.n }}. ${{ matrix.title }}"
210
+ needs: plan
211
+ runs-on: ubuntu-latest
212
+ timeout-minutes: 45 # a snippet waits up to 15 minutes, and gets one retry; past this is stuck
213
+ strategy:
214
+ fail-fast: false
215
+ matrix:
216
+ include: ${{ fromJson(needs.plan.outputs.blocks) }}
217
+ env:
218
+ BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
219
+ steps:
220
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
221
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
222
+ with:
223
+ python-version: "3.12"
224
+ cache: pip
225
+ cache-dependency-path: pyproject.toml
226
+ - name: Install the package and, for CLI blocks, the Bright Data CLI
227
+ env:
228
+ LANG_: ${{ matrix.lang }}
229
+ run: |
230
+ pip install -e .
231
+ if [ "$LANG_" = "bash" ]; then npm i -g "@brightdata/cli@$BDATA_CLI"; fi
232
+ - name: Run block ${{ matrix.n }} exactly as the README shows it
233
+ env:
234
+ N: ${{ matrix.n }}
235
+ run: |
236
+ set -o pipefail # without this, tee's exit code hides a failing block
237
+ python .github/scripts/readme_blocks.py run "$N" | tee -a "$GITHUB_STEP_SUMMARY"
238
+
239
+ cli-syntax:
240
+ name: Does the newest CLI still accept the README's commands?
241
+ runs-on: ubuntu-latest
242
+ timeout-minutes: 10
243
+ # no secret here on purpose: this job runs whatever npm calls latest
244
+ steps:
245
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
246
+ - name: Install the newest CLI, as a reader's npx would
247
+ run: |
248
+ npm i -g @brightdata/cli@latest
249
+ latest=$(bdata --version)
250
+ echo "Newest CLI: $latest. Pinned for the live jobs: $BDATA_CLI."
251
+ if [ "$latest" != "$BDATA_CLI" ]; then
252
+ echo "::warning::@brightdata/cli $latest is out; the live jobs still pin $BDATA_CLI. Bump BDATA_CLI in live.yml once this job passes."
253
+ fi
254
+ - name: Run each README CLI line with a key the API rejects
255
+ env:
256
+ BDATA_CLI: "" # use the newest, not the pin
257
+ run: |
258
+ set -o pipefail
259
+ python3 .github/scripts/readme_blocks.py cli-syntax | tee -a "$GITHUB_STEP_SUMMARY"
260
+
261
+ badge:
262
+ name: Put today's date on the README's "last verified" badge
263
+ needs: [scrape, plan, block, cli-syntax]
264
+ # scheduled and manual runs on main only, and only in the main repository: a
265
+ # push is never followed by a bot commit, a copy never diverges from the
266
+ # original, and a manual run on a branch never tries to push main
267
+ # `inputs.only` empty as well: one block passing is not the README verified.
268
+ if: always() && !cancelled() && github.event_name != 'push' && inputs.only == '' && github.repository == 'brightdata/chatgpt-scraper' && github.ref == 'refs/heads/main'
269
+ permissions:
270
+ contents: write # commits the badge line
271
+ actions: read # counts this run's job results
272
+ runs-on: ubuntu-latest
273
+ timeout-minutes: 5
274
+ steps:
275
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
276
+ - name: Rewrite one badge line, commit only if it changed
277
+ env:
278
+ GH_TOKEN: ${{ github.token }}
279
+ run: |
280
+ # Two states: red only when every check in this run failed. One passing
281
+ # check keeps the badge green; the run itself still shows which failed.
282
+ read -r passed failed < <(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
283
+ --jq '[.jobs[] | select((.name | startswith("Put today") or startswith("Open or update") or . == "List the README blocks") | not)] as $j | "\([$j[] | select(.conclusion == "success")] | length) \([$j[] | select(.conclusion == "failure")] | length)"') || true
284
+ echo "checks passed: ${passed:-0}, failed: ${failed:-0}"
285
+ if [ "${passed:-0}" -gt 0 ]; then
286
+ label="last verified"; color="brightgreen"
287
+ else
288
+ label="last check failed"; color="red"
289
+ fi
290
+ today=$(date -u +'%-d %b %Y')
291
+ line="[![${label}](https://img.shields.io/badge/${label// /%20}-${today// /%20}-${color})](https://github.com/brightdata/chatgpt-scraper/actions/workflows/live.yml) <!-- verified: rewritten by the weekly run -->"
292
+ python3 - "$line" <<'PY'
293
+ import pathlib, re, sys
294
+ p = pathlib.Path("README.md"); t = p.read_text(encoding="utf-8")
295
+ new, n = re.subn(r"^\[!\[last [^\]]*\]\(https://img\.shields\.io/badge/[^)]*\)\]\([^)]*\) <!-- verified:[^\n]*$", sys.argv[1], t, count=1, flags=re.M)
296
+ if n != 1:
297
+ sys.exit("the badge line is missing from README.md")
298
+ p.write_text(new, encoding="utf-8")
299
+ PY
300
+ if git diff --quiet; then echo "badge already shows $today"; exit 0; fi
301
+ git config user.name "anil-bd"
302
+ git config user.email "210667061+anil-bd@users.noreply.github.com"
303
+ git commit -qam "${label^} $today (automated weekly check)"
304
+ git pull -q --rebase origin main
305
+ git push -q origin main
306
+ echo "badge now shows: $label, $today"
307
+
308
+ # A red scheduled run used to show only as a red badge. Open one issue and keep
309
+ # adding to it, so a failure reaches the people watching the repository.
310
+ alert:
311
+ name: Open or update the live-check issue
312
+ needs: [scrape, block, cli-syntax]
313
+ if: failure() && github.event_name == 'schedule' && github.repository == 'brightdata/chatgpt-scraper'
314
+ runs-on: ubuntu-latest
315
+ timeout-minutes: 5
316
+ permissions:
317
+ issues: write
318
+ steps:
319
+ - env:
320
+ GH_TOKEN: ${{ github.token }}
321
+ GH_REPO: ${{ github.repository }}
322
+ RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
323
+ RESULTS: "scrape ${{ needs.scrape.result }}, README blocks ${{ needs.block.result }}, CLI syntax ${{ needs.cli-syntax.result }}"
324
+ run: |
325
+ title="Live check is failing"
326
+ body="$(date -u +'%-d %b %Y'): $RESULTS. $RUN"
327
+ n=$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number')
328
+ if [ -n "$n" ]; then
329
+ gh issue comment "$n" --body "$body"
330
+ else
331
+ gh issue create --title "$title" --body "$body"
332
+ fi