brightdata-chatgpt-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brightdata_chatgpt_scraper-0.1.0/.env.example +3 -0
- brightdata_chatgpt_scraper-0.1.0/.github/scripts/field_tables.py +72 -0
- brightdata_chatgpt_scraper-0.1.0/.github/scripts/readme_blocks.py +138 -0
- brightdata_chatgpt_scraper-0.1.0/.github/workflows/ci.yml +129 -0
- brightdata_chatgpt_scraper-0.1.0/.github/workflows/live.yml +332 -0
- brightdata_chatgpt_scraper-0.1.0/.github/workflows/publish.yml +38 -0
- brightdata_chatgpt_scraper-0.1.0/.gitignore +8 -0
- brightdata_chatgpt_scraper-0.1.0/AGENTS.md +65 -0
- brightdata_chatgpt_scraper-0.1.0/CHANGELOG.md +8 -0
- brightdata_chatgpt_scraper-0.1.0/CONTRIBUTING.md +27 -0
- brightdata_chatgpt_scraper-0.1.0/LICENSE +202 -0
- brightdata_chatgpt_scraper-0.1.0/PKG-INFO +520 -0
- brightdata_chatgpt_scraper-0.1.0/README.md +494 -0
- brightdata_chatgpt_scraper-0.1.0/examples/sample_output.json +76 -0
- brightdata_chatgpt_scraper-0.1.0/pyproject.toml +55 -0
- brightdata_chatgpt_scraper-0.1.0/src/chatgpt_scraper/__init__.py +34 -0
- brightdata_chatgpt_scraper-0.1.0/src/chatgpt_scraper/__main__.py +97 -0
- brightdata_chatgpt_scraper-0.1.0/src/chatgpt_scraper/scrape.py +258 -0
- brightdata_chatgpt_scraper-0.1.0/tests/test_scrape.py +297 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Rewrite the README's field tables from the published scraper schemas.
|
|
2
|
+
|
|
3
|
+
python .github/scripts/field_tables.py
|
|
4
|
+
|
|
5
|
+
No token needed: this reads
|
|
6
|
+
https://docs.brightdata.com/scrapers-full.json, the list the Bright Data MCP
|
|
7
|
+
server also loads: every scraper's output fields, with type and description.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import pathlib
|
|
12
|
+
import re
|
|
13
|
+
import urllib.request
|
|
14
|
+
|
|
15
|
+
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
16
|
+
README = ROOT / "README.md"
|
|
17
|
+
START, END = "<!-- fields:start -->", "<!-- fields:end -->"
|
|
18
|
+
SOURCE = "https://docs.brightdata.com/scrapers-full.json"
|
|
19
|
+
|
|
20
|
+
#: The README's order. Each scraper's collect_by_url method lists its full record.
|
|
21
|
+
SCRAPERS = [
|
|
22
|
+
("gd_m7aof0k82r803d5bjm", "ChatGPT answer"),
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def table(fields):
|
|
27
|
+
rows = ["| field | type | description |", "| --- | --- | --- |"]
|
|
28
|
+
for field in fields:
|
|
29
|
+
desc = (field.get("description") or "").replace(" — ", ": ").replace("—", ":")
|
|
30
|
+
# no fences, no table breaks
|
|
31
|
+
desc = " ".join(desc.split()).replace("`", "'").replace("|", "/")
|
|
32
|
+
rows.append(f"| `{field['name']}` | {field.get('type', '')} | {desc} |")
|
|
33
|
+
return rows
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def main():
|
|
37
|
+
text = README.read_text(encoding="utf-8")
|
|
38
|
+
assert START in text and END in text, "README field-table markers are missing"
|
|
39
|
+
with urllib.request.urlopen(SOURCE, timeout=120) as response:
|
|
40
|
+
published = {s["id"]: s for s in json.load(response)}
|
|
41
|
+
sample = json.loads((ROOT / "examples" / "sample_output.json").read_text(encoding="utf-8"))
|
|
42
|
+
first = sample["prompts"][0]["answer"]
|
|
43
|
+
lines = [
|
|
44
|
+
START,
|
|
45
|
+
"Regenerated every week from the published schema,",
|
|
46
|
+
f"[scrapers-full.json]({SOURCE}), so they cannot go stale. A record",
|
|
47
|
+
"carries the fields that apply to it, plus `timestamp` and `input`; the",
|
|
48
|
+
f"sample file's first answer has {len(first)} keys.",
|
|
49
|
+
]
|
|
50
|
+
counts = []
|
|
51
|
+
for dataset, label in SCRAPERS:
|
|
52
|
+
fields = published[dataset]["scrapers"]["collect_by_url"]["output_fields"]
|
|
53
|
+
counts.append(len(fields))
|
|
54
|
+
lines += [
|
|
55
|
+
"",
|
|
56
|
+
"<details>",
|
|
57
|
+
f"<summary>All {len(fields)} {label} fields, <code>{dataset}</code></summary>",
|
|
58
|
+
"",
|
|
59
|
+
*table(fields),
|
|
60
|
+
"",
|
|
61
|
+
"</details>",
|
|
62
|
+
]
|
|
63
|
+
lines.append(END)
|
|
64
|
+
new = re.sub(re.escape(START) + r".*?" + re.escape(END), lambda _: "\n".join(lines), text,
|
|
65
|
+
flags=re.S)
|
|
66
|
+
README.write_text(new, encoding="utf-8")
|
|
67
|
+
state = "updated" if new != text else "already current"
|
|
68
|
+
print(f"\nSchemas: {len(SCRAPERS)} scrapers, {sum(counts)} fields. Tables {state}.")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
if __name__ == "__main__":
|
|
72
|
+
main()
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""The README's code blocks, as the live check runs them.
|
|
2
|
+
|
|
3
|
+
python readme_blocks.py list # JSON matrix for the block jobs (ONLY filters)
|
|
4
|
+
python readme_blocks.py run N # run block N with the real token, one retry
|
|
5
|
+
python readme_blocks.py cli-syntax # every README CLI line with a dummy key
|
|
6
|
+
|
|
7
|
+
One parser for all three, so the list and the runner can never disagree on
|
|
8
|
+
what block N is.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import pathlib
|
|
14
|
+
import re
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
|
|
19
|
+
README = pathlib.Path(__file__).resolve().parents[2] / "README.md"
|
|
20
|
+
SKIP = ("pip install", "export ", "npm i", "npm install",
|
|
21
|
+
"npx -p @brightdata/cli bdata login", "bdata login")
|
|
22
|
+
CLI = os.environ.get("BDATA_CLI", "") # the pinned CLI version; empty means whatever is installed
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def blocks():
|
|
26
|
+
readme = README.read_text(encoding="utf-8")
|
|
27
|
+
# never execute anything inside the generated field table: its text comes from
|
|
28
|
+
# the API, and a fence in a description must not become a command
|
|
29
|
+
readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
|
|
30
|
+
found, heading, seen = [], "README", {}
|
|
31
|
+
for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
|
|
32
|
+
if m.group(1):
|
|
33
|
+
heading = m.group(1).strip() # a block is named after its section, not its first line
|
|
34
|
+
continue
|
|
35
|
+
lang, code = m.group(2), m.group(3)
|
|
36
|
+
lines = [line for line in code.splitlines()
|
|
37
|
+
if line.strip() and not line.strip().startswith("#")]
|
|
38
|
+
if lang == "bash":
|
|
39
|
+
lines = [line for line in lines if not line.startswith(SKIP)]
|
|
40
|
+
if not lines:
|
|
41
|
+
continue
|
|
42
|
+
code = "\n".join(lines)
|
|
43
|
+
seen[heading] = seen.get(heading, 0) + 1
|
|
44
|
+
title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
|
|
45
|
+
found.append({"lang": lang, "title": title[:50], "code": code})
|
|
46
|
+
return found
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cli_lines(code, key):
|
|
50
|
+
"""The CLI cannot open a browser here, so the key goes in by flag, and an
|
|
51
|
+
unpinned npx is held to the version the job installed."""
|
|
52
|
+
out = []
|
|
53
|
+
for line in code.splitlines():
|
|
54
|
+
if CLI:
|
|
55
|
+
line = line.replace("npx -p @brightdata/cli ", f"npx -p @brightdata/cli@{CLI} ")
|
|
56
|
+
if "bdata pipelines" in line:
|
|
57
|
+
line += f" -k {key}"
|
|
58
|
+
out.append(line)
|
|
59
|
+
return "\n".join(out)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def list_blocks():
|
|
63
|
+
wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()}
|
|
64
|
+
out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]}
|
|
65
|
+
for i, b in enumerate(blocks()) if not wanted or i + 1 in wanted]
|
|
66
|
+
print("blocks=" + json.dumps(out))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run_block(n):
|
|
70
|
+
b = blocks()[n - 1]
|
|
71
|
+
if b["lang"] == "bash":
|
|
72
|
+
# only the shell ever sees the key's value
|
|
73
|
+
argv = ["bash", "-e", "-c", cli_lines(b["code"], '"$BRIGHTDATA_API_TOKEN"')]
|
|
74
|
+
else:
|
|
75
|
+
argv = [sys.executable, "-c", b["code"]]
|
|
76
|
+
secret = os.environ.get("BRIGHTDATA_API_TOKEN", "")
|
|
77
|
+
for _attempt in (1, 2):
|
|
78
|
+
started = time.time()
|
|
79
|
+
try:
|
|
80
|
+
run = subprocess.run(argv, capture_output=True, text=True, timeout=1080)
|
|
81
|
+
except subprocess.TimeoutExpired:
|
|
82
|
+
run = subprocess.CompletedProcess(
|
|
83
|
+
argv, 1, "", "block timeout: no result within 18 minutes")
|
|
84
|
+
secs = round(time.time() - started)
|
|
85
|
+
out, err = run.stdout.strip(), run.stderr.strip()
|
|
86
|
+
if secret: # summaries are not secret-masked the way logs are
|
|
87
|
+
out, err = out.replace(secret, "***"), err.replace(secret, "***")
|
|
88
|
+
ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err)
|
|
89
|
+
if ok or ((out or err) and "timeout" not in (out + err).lower()):
|
|
90
|
+
break
|
|
91
|
+
time.sleep(30) # the API timed out or answered with nothing at all; one retry
|
|
92
|
+
if ok:
|
|
93
|
+
shown = (out.splitlines() or [""])[:1]
|
|
94
|
+
else:
|
|
95
|
+
# the last few lines: a CLI error often puts the cause above its hint
|
|
96
|
+
shown = (err.splitlines() or out.splitlines() or ["no output"])[-6:]
|
|
97
|
+
body = "\n".join(" " + line[:160] for line in shown)
|
|
98
|
+
state = "ok" if ok else "FAILED"
|
|
99
|
+
print(f"\n`{b['title']}`: {state} in {secs}s (exit {run.returncode})\n\n{body}")
|
|
100
|
+
sys.exit(0 if ok else 1)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def cli_syntax():
|
|
104
|
+
"""Run every README CLI line with a key the API rejects. A line the CLI
|
|
105
|
+
still understands reaches the API and gets a 401; a line it no longer
|
|
106
|
+
understands fails before that. Free, and needs no secret."""
|
|
107
|
+
failed = 0
|
|
108
|
+
print("| block | CLI line | result |\n| --- | --- | --- |")
|
|
109
|
+
for i, b in enumerate(blocks(), 1):
|
|
110
|
+
if b["lang"] != "bash":
|
|
111
|
+
continue
|
|
112
|
+
for line in b["code"].splitlines():
|
|
113
|
+
if "bdata pipelines" not in line:
|
|
114
|
+
continue
|
|
115
|
+
run = subprocess.run(["bash", "-c", cli_lines(line, "invalid-key-for-syntax-check")],
|
|
116
|
+
capture_output=True, text=True, timeout=120)
|
|
117
|
+
text = run.stdout + run.stderr
|
|
118
|
+
ok = "401" in text or "Invalid credentials" in text
|
|
119
|
+
last = (run.stderr.strip().splitlines() or run.stdout.strip().splitlines()
|
|
120
|
+
or ["no output"])
|
|
121
|
+
why = "reached the API" if ok else " / ".join(x.strip() for x in last[-3:])[:200]
|
|
122
|
+
print(f"| {i} | `{line[:90]}` | {'ok' if ok else 'FAILED'}: {why.replace('|', '/')} |")
|
|
123
|
+
if not ok:
|
|
124
|
+
failed = 1
|
|
125
|
+
print(f"::error::README block {i}: the CLI rejects `{line}`", file=sys.stderr)
|
|
126
|
+
sys.exit(failed)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
if __name__ == "__main__":
|
|
130
|
+
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
131
|
+
if mode == "list":
|
|
132
|
+
list_blocks()
|
|
133
|
+
elif mode == "run":
|
|
134
|
+
run_block(int(sys.argv[2]))
|
|
135
|
+
elif mode == "cli-syntax":
|
|
136
|
+
cli_syntax()
|
|
137
|
+
else:
|
|
138
|
+
sys.exit(__doc__)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Runs by itself on every push and pull request.
|
|
2
|
+
# Never calls the ChatGPT Scraper API, so it needs no token and costs nothing.
|
|
3
|
+
name: Tests (automatic, no API calls, free)
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch: # a Run workflow button, to re-run the tests without a commit
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
concurrency: # a newer push to the same branch or PR replaces the run still going
|
|
14
|
+
group: ci-${{ github.event_name }}-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
tests:
|
|
19
|
+
name: Run the tests and the code style check
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
timeout-minutes: 10
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
24
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.12"
|
|
27
|
+
cache: pip
|
|
28
|
+
cache-dependency-path: pyproject.toml
|
|
29
|
+
- name: Install the package and its test tools
|
|
30
|
+
run: pip install -e ".[dev]"
|
|
31
|
+
- name: Check code style with ruff
|
|
32
|
+
run: ruff check .
|
|
33
|
+
- name: Run the tests
|
|
34
|
+
run: pytest
|
|
35
|
+
- name: README headings are well formed
|
|
36
|
+
# A section rewrite once produced "## The data## The data" and shipped;
|
|
37
|
+
# a heading with a second run of hashes mid-line is never intended.
|
|
38
|
+
run: |
|
|
39
|
+
if grep -nE '^#+ .*#{2,} ' README.md; then
|
|
40
|
+
echo "::error::malformed heading in README.md"; exit 1
|
|
41
|
+
fi
|
|
42
|
+
|
|
43
|
+
# These install steps shipped broken once, because they were only ever run
|
|
44
|
+
# from a working copy that already had the package. This job follows the
|
|
45
|
+
# README on an empty machine and stops where the tool asks for a token, which
|
|
46
|
+
# proves the clone, the install and the command without spending money.
|
|
47
|
+
coldstart:
|
|
48
|
+
name: Follow the README from scratch on an empty machine
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
timeout-minutes: 10
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
53
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
54
|
+
with:
|
|
55
|
+
python-version: "3.12"
|
|
56
|
+
- name: Copy the install steps out of the README and run them
|
|
57
|
+
run: |
|
|
58
|
+
set -o pipefail
|
|
59
|
+
python - <<'PY' > "$RUNNER_TEMP/quickstart.sh"
|
|
60
|
+
import os
|
|
61
|
+
import pathlib
|
|
62
|
+
import re
|
|
63
|
+
import sys
|
|
64
|
+
|
|
65
|
+
readme = pathlib.Path("README.md").read_text(encoding="utf-8")
|
|
66
|
+
|
|
67
|
+
def section(title):
|
|
68
|
+
return readme.split(f"## {title}", 1)[1].split("\n## ", 1)[0]
|
|
69
|
+
|
|
70
|
+
def block(text, lang):
|
|
71
|
+
return re.search(r"```" + lang + r"\n(.*?)```", text, re.S).group(1)
|
|
72
|
+
|
|
73
|
+
quick = section("How to scrape ChatGPT results with Python")
|
|
74
|
+
snippet = block(quick, "python")
|
|
75
|
+
command = block(section("Bulk ChatGPT prompt scraper CLI"), "bash")
|
|
76
|
+
for required, where in (("BRIGHTDATA_API_TOKEN", snippet),
|
|
77
|
+
("urllib.request", snippet),
|
|
78
|
+
("chatgpt-scraper", command)):
|
|
79
|
+
if required not in where:
|
|
80
|
+
sys.exit(f"the README no longer contains {required!r} where the cold start expects it")
|
|
81
|
+
|
|
82
|
+
steps = ["# --- Quickstart: the standard library path, nothing to install ---"]
|
|
83
|
+
pathlib.Path(os.environ["RUNNER_TEMP"], "quickstart_snippet.py").write_text(snippet, encoding="utf-8")
|
|
84
|
+
steps += [
|
|
85
|
+
"set +e",
|
|
86
|
+
"python quickstart_snippet.py > out.txt 2> err.txt",
|
|
87
|
+
"code=$?",
|
|
88
|
+
"set -e",
|
|
89
|
+
'test "$code" != "0" || { echo "the snippet ran without a token?"; cat out.txt; exit 1; }',
|
|
90
|
+
'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
|
|
91
|
+
'echo "Quickstart: runs on a bare Python, and stops at the token, as documented."',
|
|
92
|
+
"# --- Or run it as a command: the CLI path ---",
|
|
93
|
+
]
|
|
94
|
+
for line in command.splitlines():
|
|
95
|
+
if line.startswith("chatgpt-scraper"):
|
|
96
|
+
continue # run below, so the exit code can be checked
|
|
97
|
+
if line.startswith("pip install") and "git+" in line:
|
|
98
|
+
# The repository is public, so this installs the exact line a
|
|
99
|
+
# reader pastes. On a private fork the URL is unreachable, and
|
|
100
|
+
# the check falls back to the checkout on its own.
|
|
101
|
+
url = line.split("git+", 1)[1].strip() if "git+" in line else ""
|
|
102
|
+
line = "\n".join([
|
|
103
|
+
f'repo="{url}"',
|
|
104
|
+
'code=$(curl -s -o /dev/null -w "%{http_code}" "$repo" || echo 000)',
|
|
105
|
+
'if [ "$code" = "200" ]; then',
|
|
106
|
+
f' echo "$repo is public: installing the exact README line"',
|
|
107
|
+
f' {line}',
|
|
108
|
+
"else",
|
|
109
|
+
' echo "NOTE: $repo returns HTTP $code, so it is not public yet."',
|
|
110
|
+
' echo "Installing from this checkout instead."',
|
|
111
|
+
' pip install "$GITHUB_WORKSPACE"',
|
|
112
|
+
"fi",
|
|
113
|
+
])
|
|
114
|
+
steps.append(line)
|
|
115
|
+
steps += [
|
|
116
|
+
"set +e",
|
|
117
|
+
'chatgpt-scraper "What is the capital of Australia?" > out.txt 2> err.txt',
|
|
118
|
+
"code=$?",
|
|
119
|
+
"set -e",
|
|
120
|
+
'test "$code" = "2" || { echo "expected exit 2, got $code"; cat err.txt; exit 1; }',
|
|
121
|
+
'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
|
|
122
|
+
'echo "Success: a new user can follow both README paths from scratch."',
|
|
123
|
+
]
|
|
124
|
+
print("\n".join(steps))
|
|
125
|
+
PY
|
|
126
|
+
echo "--- these commands came out of README.md ---"
|
|
127
|
+
cat "$RUNNER_TEMP/quickstart.sh"
|
|
128
|
+
echo "--- running them on this empty machine ---"
|
|
129
|
+
cd "$RUNNER_TEMP" && bash -e "$RUNNER_TEMP/quickstart.sh"
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# The only workflow that calls the real API. Needs the BRIGHTDATA_API_TOKEN
|
|
2
|
+
# secret and is billed per answer.
|
|
3
|
+
#
|
|
4
|
+
# Three ways it runs:
|
|
5
|
+
# on every push to main: one answer to one prompt and the README's links,
|
|
6
|
+
# one record and about two minutes, so a broken commit shows up at once.
|
|
7
|
+
# A newer push cancels a run still going for an older commit.
|
|
8
|
+
# by hand, from the Actions tab, to ask whatever prompts you choose
|
|
9
|
+
# every Monday at 08:00 UTC: one answer, the async path, and the
|
|
10
|
+
# field tables, about two records
|
|
11
|
+
# in the same Monday run: every code block in the README,
|
|
12
|
+
# about 15 records and 10 minutes
|
|
13
|
+
#
|
|
14
|
+
# Every run also checks, for free and with no secret, that the newest Bright
|
|
15
|
+
# Data CLI still accepts every CLI line in the README. Readers run `npx -p
|
|
16
|
+
# @brightdata/cli`, which is always the newest release, while the jobs that hold
|
|
17
|
+
# the secret run the version pinned in BDATA_CLI below. When it warns about a
|
|
18
|
+
# new release, bump BDATA_CLI.
|
|
19
|
+
#
|
|
20
|
+
# A slow day at the API shows up as a red run. Rerun it before reading more
|
|
21
|
+
# into it.
|
|
22
|
+
#
|
|
23
|
+
# The Monday run ends by rewriting the "last verified" badge line at
|
|
24
|
+
# the top of README.md with today's date, and committing it if it changed. Push
|
|
25
|
+
# runs never do this, so your own push is never followed by a bot commit. Only
|
|
26
|
+
# the main repository commits; a copy of the repository just runs the checks.
|
|
27
|
+
name: Live check
|
|
28
|
+
|
|
29
|
+
run-name: "${{ github.event.schedule == '0 8 * * 1' && 'Weekly check: does every README block still work?' || github.event_name == 'schedule' && 'Daily check: is anything broken?' || github.event_name == 'push' && format('Push check: {0}', github.event.head_commit.message) || format('Ask {0}', inputs.prompts) }}"
|
|
30
|
+
|
|
31
|
+
on:
|
|
32
|
+
push:
|
|
33
|
+
branches: [main]
|
|
34
|
+
schedule:
|
|
35
|
+
- cron: "0 8 * * 1" # Monday: the whole README
|
|
36
|
+
workflow_dispatch:
|
|
37
|
+
inputs:
|
|
38
|
+
prompts:
|
|
39
|
+
description: Prompts, separated by commas
|
|
40
|
+
default: Best CRM software for small businesses, Best free photo editing app
|
|
41
|
+
validate:
|
|
42
|
+
description: Also run every README block, as the Monday run does
|
|
43
|
+
type: boolean
|
|
44
|
+
default: false
|
|
45
|
+
only:
|
|
46
|
+
description: Block numbers to run, comma separated, for a cheap check. Empty means all.
|
|
47
|
+
default: ""
|
|
48
|
+
|
|
49
|
+
permissions:
|
|
50
|
+
contents: write # the weekly check commits the regenerated field tables
|
|
51
|
+
|
|
52
|
+
env:
|
|
53
|
+
BDATA_CLI: "0.4.0" # pinned for the jobs that hold the API secret; bump after cli-syntax passes
|
|
54
|
+
|
|
55
|
+
concurrency:
|
|
56
|
+
group: live-${{ github.event_name }}-${{ github.ref }}
|
|
57
|
+
cancel-in-progress: ${{ github.event_name == 'push' }} # only the newest commit matters
|
|
58
|
+
|
|
59
|
+
jobs:
|
|
60
|
+
scrape:
|
|
61
|
+
name: Ask ChatGPT and attach the answers to this run
|
|
62
|
+
runs-on: ubuntu-latest
|
|
63
|
+
timeout-minutes: 45 # an answer took one to five minutes live; the README blocks have their own jobs
|
|
64
|
+
env:
|
|
65
|
+
BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
|
|
66
|
+
# On the weekly schedule there are no inputs, so fall back to one cheap prompt.
|
|
67
|
+
PROMPTS: ${{ inputs.prompts || 'Best CRM software for small businesses' }}
|
|
68
|
+
# the async path and the field tables run on every scheduled run and on request
|
|
69
|
+
CHECK: ${{ github.event_name == 'schedule' || inputs.validate }}
|
|
70
|
+
steps:
|
|
71
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
72
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
73
|
+
with:
|
|
74
|
+
python-version: "3.12"
|
|
75
|
+
cache: pip
|
|
76
|
+
cache-dependency-path: pyproject.toml
|
|
77
|
+
- name: Install the package
|
|
78
|
+
run: pip install -e .
|
|
79
|
+
|
|
80
|
+
- name: Do the README's links still resolve?
|
|
81
|
+
run: |
|
|
82
|
+
public=$(curl -s -o /dev/null -w '%{http_code}' https://github.com/brightdata/chatgpt-scraper)
|
|
83
|
+
failed=0
|
|
84
|
+
# links in prose only: fenced blocks hold code and scraped output, and the
|
|
85
|
+
# weekly run executes the code blocks anyway
|
|
86
|
+
for url in $(awk '/^```/{fenced=!fenced; next} !fenced' README.md AGENTS.md | grep -ohE 'https?://[^ )<>"`\\]+' | sed 's/[.,:]*$//' | sort -u); do
|
|
87
|
+
case "$url" in
|
|
88
|
+
*chatgpt.com/*) continue ;; # answers bot traffic with a challenge page
|
|
89
|
+
*zendesk.com/*) continue ;; # answers 403 to anything that is not a browser
|
|
90
|
+
*mcp.brightdata.com/*) continue ;; # carries a placeholder token; nothing to check without a real one
|
|
91
|
+
*api.brightdata.com/*) continue ;; # API endpoints answer 401 without a key; the README blocks call them
|
|
92
|
+
*github.com/brightdata/chatgpt-scraper*) [ "$public" = "200" ] || continue ;; # this repo and its Node twin, unreachable while private
|
|
93
|
+
esac
|
|
94
|
+
code=$(curl -s -o /dev/null -L --retry 3 --max-time 30 -A "Mozilla/5.0" -w '%{http_code}' "$url" || echo 000)
|
|
95
|
+
if [ "$code" = "200" ]; then echo "200 $url"; else echo "::error::$code $url"; failed=1; fi
|
|
96
|
+
done
|
|
97
|
+
[ "$failed" = 0 ] || exit 1
|
|
98
|
+
- name: Ask ChatGPT
|
|
99
|
+
run: |
|
|
100
|
+
test -n "$BRIGHTDATA_API_TOKEN" || {
|
|
101
|
+
echo "No API token. Add BRIGHTDATA_API_TOKEN under" >&2
|
|
102
|
+
echo "Settings, Secrets and variables, Actions, then start this again." >&2
|
|
103
|
+
exit 1
|
|
104
|
+
}
|
|
105
|
+
mapfile -t KW < <(python3 -c 'import os; [print(k.strip()) for k in os.environ["PROMPTS"].split(",") if k.strip()]')
|
|
106
|
+
# One retry, only on an API timeout. A slow day is not a broken build.
|
|
107
|
+
set +e
|
|
108
|
+
python -m chatgpt_scraper "${KW[@]}" 2>&1 | tee run.log
|
|
109
|
+
code=${PIPESTATUS[0]}
|
|
110
|
+
if [ "$code" != "0" ] && grep -qi "timeout" run.log; then
|
|
111
|
+
echo "::warning::The API timed out. Waiting 30s and trying once more."
|
|
112
|
+
sleep 30
|
|
113
|
+
python -m chatgpt_scraper "${KW[@]}" 2>&1 | tee run.log
|
|
114
|
+
code=${PIPESTATUS[0]}
|
|
115
|
+
fi
|
|
116
|
+
exit "$code"
|
|
117
|
+
|
|
118
|
+
- name: Show what was found
|
|
119
|
+
if: always()
|
|
120
|
+
run: |
|
|
121
|
+
set -o pipefail
|
|
122
|
+
test -f chatgpt.json || exit 0
|
|
123
|
+
python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
|
|
124
|
+
import json
|
|
125
|
+
|
|
126
|
+
doc = json.load(open("chatgpt.json"))
|
|
127
|
+
print(f"Run at {doc['generated_at']}, country {doc['country']}\n")
|
|
128
|
+
print("| prompt | characters | citations | model |")
|
|
129
|
+
print("| --- | --- | --- | --- |")
|
|
130
|
+
for entry in doc["prompts"]:
|
|
131
|
+
answer = entry["answer"] or {}
|
|
132
|
+
print(
|
|
133
|
+
f"| {entry['prompt']} | {len(answer.get('answer_text') or '')} "
|
|
134
|
+
f"| {len(answer.get('citations') or [])} | {answer.get('model')} |"
|
|
135
|
+
)
|
|
136
|
+
first = next((e["answer"] for e in doc["prompts"] if e["answer"]), None)
|
|
137
|
+
if first:
|
|
138
|
+
print("\n<details><summary>First record in full</summary>\n")
|
|
139
|
+
print("```json")
|
|
140
|
+
print(json.dumps(first, indent=2, ensure_ascii=False))
|
|
141
|
+
print("```\n")
|
|
142
|
+
print("</details>")
|
|
143
|
+
PY
|
|
144
|
+
|
|
145
|
+
# The two smoke checks. A live request is already covered by the step above.
|
|
146
|
+
- name: Does the async trigger, status, fetch path still work?
|
|
147
|
+
if: env.CHECK == 'true'
|
|
148
|
+
run: |
|
|
149
|
+
set -o pipefail
|
|
150
|
+
python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY"
|
|
151
|
+
import time
|
|
152
|
+
|
|
153
|
+
from chatgpt_scraper import CHATGPT, fetch, status, trigger
|
|
154
|
+
|
|
155
|
+
sid = trigger(CHATGPT, [{"url": "https://chatgpt.com/", "prompt": "What is the capital of Australia?"}])
|
|
156
|
+
for _ in range(90): # up to 15 minutes
|
|
157
|
+
state = status(sid)
|
|
158
|
+
if state in ("ready", "failed"):
|
|
159
|
+
break
|
|
160
|
+
time.sleep(10)
|
|
161
|
+
assert state == "ready", f"snapshot {sid} ended as {state}"
|
|
162
|
+
rows = fetch(sid)
|
|
163
|
+
assert rows and rows[0].get("answer_text"), f"fetch returned no answer: {rows[:1]}"
|
|
164
|
+
print(f"\nAsync path ok: trigger, status, fetch returned {len(rows)} record(s).")
|
|
165
|
+
PY
|
|
166
|
+
|
|
167
|
+
- name: Regenerate the README field tables from the live schema
|
|
168
|
+
if: always() && env.CHECK == 'true' && github.repository == 'brightdata/chatgpt-scraper' && github.ref == 'refs/heads/main'
|
|
169
|
+
run: |
|
|
170
|
+
python .github/scripts/field_tables.py | tee -a "$GITHUB_STEP_SUMMARY"
|
|
171
|
+
if git diff --quiet README.md; then
|
|
172
|
+
echo "Nothing to commit."
|
|
173
|
+
exit 0
|
|
174
|
+
fi
|
|
175
|
+
# Never commit a README the tests would reject.
|
|
176
|
+
pip install -q -e ".[dev]" && pytest -q
|
|
177
|
+
git config user.name "anil-bd"
|
|
178
|
+
git config user.email "210667061+anil-bd@users.noreply.github.com"
|
|
179
|
+
git add README.md
|
|
180
|
+
git commit -m "Regenerate the field tables from the live schema (automated weekly check)"
|
|
181
|
+
git pull --rebase origin main
|
|
182
|
+
git push
|
|
183
|
+
echo "Committed the regenerated tables."
|
|
184
|
+
|
|
185
|
+
- name: Attach chatgpt.json so you can download it
|
|
186
|
+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
187
|
+
if: always()
|
|
188
|
+
with:
|
|
189
|
+
name: chatgpt-json
|
|
190
|
+
path: chatgpt.json
|
|
191
|
+
if-no-files-found: warn
|
|
192
|
+
|
|
193
|
+
# Every code block in the README, each in its own job, all at once. Mondays,
|
|
194
|
+
# and on request. Wall clock is the slowest block instead of the sum of all.
|
|
195
|
+
plan:
|
|
196
|
+
name: List the README blocks
|
|
197
|
+
if: github.event.schedule == '0 8 * * 1' || inputs.validate == true
|
|
198
|
+
runs-on: ubuntu-latest
|
|
199
|
+
outputs:
|
|
200
|
+
blocks: ${{ steps.list.outputs.blocks }}
|
|
201
|
+
steps:
|
|
202
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
203
|
+
- id: list
|
|
204
|
+
env:
|
|
205
|
+
ONLY: ${{ inputs.only }}
|
|
206
|
+
run: python .github/scripts/readme_blocks.py list >> "$GITHUB_OUTPUT"
|
|
207
|
+
|
|
208
|
+
block:
|
|
209
|
+
name: "${{ matrix.n }}. ${{ matrix.title }}"
|
|
210
|
+
needs: plan
|
|
211
|
+
runs-on: ubuntu-latest
|
|
212
|
+
timeout-minutes: 45 # a snippet waits up to 15 minutes, and gets one retry; past this is stuck
|
|
213
|
+
strategy:
|
|
214
|
+
fail-fast: false
|
|
215
|
+
matrix:
|
|
216
|
+
include: ${{ fromJson(needs.plan.outputs.blocks) }}
|
|
217
|
+
env:
|
|
218
|
+
BRIGHTDATA_API_TOKEN: ${{ secrets.BRIGHTDATA_API_TOKEN }}
|
|
219
|
+
steps:
|
|
220
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
221
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
222
|
+
with:
|
|
223
|
+
python-version: "3.12"
|
|
224
|
+
cache: pip
|
|
225
|
+
cache-dependency-path: pyproject.toml
|
|
226
|
+
- name: Install the package and, for CLI blocks, the Bright Data CLI
|
|
227
|
+
env:
|
|
228
|
+
LANG_: ${{ matrix.lang }}
|
|
229
|
+
run: |
|
|
230
|
+
pip install -e .
|
|
231
|
+
if [ "$LANG_" = "bash" ]; then npm i -g "@brightdata/cli@$BDATA_CLI"; fi
|
|
232
|
+
- name: Run block ${{ matrix.n }} exactly as the README shows it
|
|
233
|
+
env:
|
|
234
|
+
N: ${{ matrix.n }}
|
|
235
|
+
run: |
|
|
236
|
+
set -o pipefail # without this, tee's exit code hides a failing block
|
|
237
|
+
python .github/scripts/readme_blocks.py run "$N" | tee -a "$GITHUB_STEP_SUMMARY"
|
|
238
|
+
|
|
239
|
+
cli-syntax:
|
|
240
|
+
name: Does the newest CLI still accept the README's commands?
|
|
241
|
+
runs-on: ubuntu-latest
|
|
242
|
+
timeout-minutes: 10
|
|
243
|
+
# no secret here on purpose: this job runs whatever npm calls latest
|
|
244
|
+
steps:
|
|
245
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
246
|
+
- name: Install the newest CLI, as a reader's npx would
|
|
247
|
+
run: |
|
|
248
|
+
npm i -g @brightdata/cli@latest
|
|
249
|
+
latest=$(bdata --version)
|
|
250
|
+
echo "Newest CLI: $latest. Pinned for the live jobs: $BDATA_CLI."
|
|
251
|
+
if [ "$latest" != "$BDATA_CLI" ]; then
|
|
252
|
+
echo "::warning::@brightdata/cli $latest is out; the live jobs still pin $BDATA_CLI. Bump BDATA_CLI in live.yml once this job passes."
|
|
253
|
+
fi
|
|
254
|
+
- name: Run each README CLI line with a key the API rejects
|
|
255
|
+
env:
|
|
256
|
+
BDATA_CLI: "" # use the newest, not the pin
|
|
257
|
+
run: |
|
|
258
|
+
set -o pipefail
|
|
259
|
+
python3 .github/scripts/readme_blocks.py cli-syntax | tee -a "$GITHUB_STEP_SUMMARY"
|
|
260
|
+
|
|
261
|
+
badge:
|
|
262
|
+
name: Put today's date on the README's "last verified" badge
|
|
263
|
+
needs: [scrape, plan, block, cli-syntax]
|
|
264
|
+
# scheduled and manual runs on main only, and only in the main repository: a
|
|
265
|
+
# push is never followed by a bot commit, a copy never diverges from the
|
|
266
|
+
# original, and a manual run on a branch never tries to push main
|
|
267
|
+
# `inputs.only` empty as well: one block passing is not the README verified.
|
|
268
|
+
if: always() && !cancelled() && github.event_name != 'push' && inputs.only == '' && github.repository == 'brightdata/chatgpt-scraper' && github.ref == 'refs/heads/main'
|
|
269
|
+
permissions:
|
|
270
|
+
contents: write # commits the badge line
|
|
271
|
+
actions: read # counts this run's job results
|
|
272
|
+
runs-on: ubuntu-latest
|
|
273
|
+
timeout-minutes: 5
|
|
274
|
+
steps:
|
|
275
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
276
|
+
- name: Rewrite one badge line, commit only if it changed
|
|
277
|
+
env:
|
|
278
|
+
GH_TOKEN: ${{ github.token }}
|
|
279
|
+
run: |
|
|
280
|
+
# Two states: red only when every check in this run failed. One passing
|
|
281
|
+
# check keeps the badge green; the run itself still shows which failed.
|
|
282
|
+
read -r passed failed < <(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" \
|
|
283
|
+
--jq '[.jobs[] | select((.name | startswith("Put today") or startswith("Open or update") or . == "List the README blocks") | not)] as $j | "\([$j[] | select(.conclusion == "success")] | length) \([$j[] | select(.conclusion == "failure")] | length)"') || true
|
|
284
|
+
echo "checks passed: ${passed:-0}, failed: ${failed:-0}"
|
|
285
|
+
if [ "${passed:-0}" -gt 0 ]; then
|
|
286
|
+
label="last verified"; color="brightgreen"
|
|
287
|
+
else
|
|
288
|
+
label="last check failed"; color="red"
|
|
289
|
+
fi
|
|
290
|
+
today=$(date -u +'%-d %b %Y')
|
|
291
|
+
line="[](https://github.com/brightdata/chatgpt-scraper/actions/workflows/live.yml) <!-- verified: rewritten by the weekly run -->"
|
|
292
|
+
python3 - "$line" <<'PY'
|
|
293
|
+
import pathlib, re, sys
|
|
294
|
+
p = pathlib.Path("README.md"); t = p.read_text(encoding="utf-8")
|
|
295
|
+
new, n = re.subn(r"^\[!\[last [^\]]*\]\(https://img\.shields\.io/badge/[^)]*\)\]\([^)]*\) <!-- verified:[^\n]*$", sys.argv[1], t, count=1, flags=re.M)
|
|
296
|
+
if n != 1:
|
|
297
|
+
sys.exit("the badge line is missing from README.md")
|
|
298
|
+
p.write_text(new, encoding="utf-8")
|
|
299
|
+
PY
|
|
300
|
+
if git diff --quiet; then echo "badge already shows $today"; exit 0; fi
|
|
301
|
+
git config user.name "anil-bd"
|
|
302
|
+
git config user.email "210667061+anil-bd@users.noreply.github.com"
|
|
303
|
+
git commit -qam "${label^} $today (automated weekly check)"
|
|
304
|
+
git pull -q --rebase origin main
|
|
305
|
+
git push -q origin main
|
|
306
|
+
echo "badge now shows: $label, $today"
|
|
307
|
+
|
|
308
|
+
# A red scheduled run used to show only as a red badge. Open one issue and keep
|
|
309
|
+
# adding to it, so a failure reaches the people watching the repository.
|
|
310
|
+
alert:
|
|
311
|
+
name: Open or update the live-check issue
|
|
312
|
+
needs: [scrape, block, cli-syntax]
|
|
313
|
+
if: failure() && github.event_name == 'schedule' && github.repository == 'brightdata/chatgpt-scraper'
|
|
314
|
+
runs-on: ubuntu-latest
|
|
315
|
+
timeout-minutes: 5
|
|
316
|
+
permissions:
|
|
317
|
+
issues: write
|
|
318
|
+
steps:
|
|
319
|
+
- env:
|
|
320
|
+
GH_TOKEN: ${{ github.token }}
|
|
321
|
+
GH_REPO: ${{ github.repository }}
|
|
322
|
+
RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
323
|
+
RESULTS: "scrape ${{ needs.scrape.result }}, README blocks ${{ needs.block.result }}, CLI syntax ${{ needs.cli-syntax.result }}"
|
|
324
|
+
run: |
|
|
325
|
+
title="Live check is failing"
|
|
326
|
+
body="$(date -u +'%-d %b %Y'): $RESULTS. $RUN"
|
|
327
|
+
n=$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number')
|
|
328
|
+
if [ -n "$n" ]; then
|
|
329
|
+
gh issue comment "$n" --body "$body"
|
|
330
|
+
else
|
|
331
|
+
gh issue create --title "$title" --body "$body"
|
|
332
|
+
fi
|