brightdata-chatgpt-scraper 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brightdata_chatgpt_scraper-0.1.0.dist-info/METADATA +520 -0
- brightdata_chatgpt_scraper-0.1.0.dist-info/RECORD +8 -0
- brightdata_chatgpt_scraper-0.1.0.dist-info/WHEEL +4 -0
- brightdata_chatgpt_scraper-0.1.0.dist-info/entry_points.txt +2 -0
- brightdata_chatgpt_scraper-0.1.0.dist-info/licenses/LICENSE +202 -0
- chatgpt_scraper/__init__.py +34 -0
- chatgpt_scraper/__main__.py +97 -0
- chatgpt_scraper/scrape.py +258 -0
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: brightdata-chatgpt-scraper
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: ChatGPT scraper in Python: ChatGPT answers, citations and sources for any prompt as JSON, by country, with web search and follow-up prompts, on the Bright Data ChatGPT Scraper API. No OpenAI API key, no browser, no dependencies. 5,000 free credits a month.
|
|
5
|
+
Project-URL: Homepage, https://github.com/brightdata/chatgpt-scraper
|
|
6
|
+
Project-URL: Documentation, https://docs.brightdata.com/products/scrapers/chatgpt/introduction
|
|
7
|
+
Project-URL: Source, https://github.com/brightdata/chatgpt-scraper
|
|
8
|
+
Project-URL: Issues, https://github.com/brightdata/chatgpt-scraper/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/brightdata/chatgpt-scraper/blob/main/CHANGELOG.md
|
|
10
|
+
Author: Bright Data
|
|
11
|
+
License: Apache-2.0
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: ai-search,bright-data,chatgpt,chatgpt-api,chatgpt-scraper,geo,llm,scraper,web-scraping
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
24
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# ChatGPT scraper for Python
|
|
28
|
+
|
|
29
|
+
Scrape ChatGPT answers, citations and sources to JSON with Python. Track brand visibility in ChatGPT. No OpenAI API key.
|
|
30
|
+
Built on the [Bright Data ChatGPT Scraper API](https://brightdata.com/products/web-scraper/chatgpt?utm_source=github&utm_medium=referral&utm_content=chatgpt-scraper). New accounts get 5,000 free credits a month.
|
|
31
|
+
|
|
32
|
+
[](https://github.com/brightdata/chatgpt-scraper/actions/workflows/live.yml)
|
|
33
|
+
[](https://github.com/brightdata/chatgpt-scraper/actions/workflows/live.yml) <!-- verified: rewritten by the weekly run -->
|
|
34
|
+
|
|
35
|
+
[Quickstart](#how-to-scrape-chatgpt-results-with-python) · [No OpenAI key](#scrape-chatgpt-without-an-openai-api-key) · [CLI](#bulk-chatgpt-prompt-scraper-cli) · [Citations](#track-chatgpt-citations-and-brand-visibility-for-a-prompt) · [Endpoints](#scrape-chatgpt-answers-citations-and-sources) · [Data](#chatgpt-answer-data-fields-json) · [Errors](#chatgpt-scraper-errors) · [MCP and Claude Code](#chatgpt-mcp-server-and-claude-code-skill) · [Free credits](#is-this-chatgpt-scraper-free) · [Docs](https://docs.brightdata.com/products/scrapers/chatgpt/introduction) · [Support](#support)
|
|
36
|
+
|
|
37
|
+
ChatGPT answers as JSON, in Python: send a prompt, get back the answer ChatGPT
|
|
38
|
+
shows in its own interface, as text, Markdown and HTML, with the sites it
|
|
39
|
+
cited, by country, with web search and follow-up prompts. Built for ChatGPT
|
|
40
|
+
visibility tracking and GEO: which brands ChatGPT mentions and recommends, and
|
|
41
|
+
which sources it cites. No OpenAI API key, no browser, no dependencies. Built on the
|
|
42
|
+
[Bright Data ChatGPT Scraper API](https://brightdata.com/products/web-scraper/chatgpt?utm_source=github&utm_medium=referral&utm_content=chatgpt-scraper).
|
|
43
|
+
|
|
44
|
+
Also here: a one-command CLI for a list of prompts, and the
|
|
45
|
+
[Bright Data CLI](#chatgpt-mcp-server-and-claude-code-skill), which needs no Python at all.
|
|
46
|
+
|
|
47
|
+
## How to scrape ChatGPT results with Python
|
|
48
|
+
|
|
49
|
+
Python 3.10 or newer, and nothing to install: this is the standard library.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
export BRIGHTDATA_API_TOKEN=YOUR_API_KEY
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Get a token from the
|
|
56
|
+
[Bright Data control panel](https://brightdata.com/cp/setting/users). No
|
|
57
|
+
account yet? [Create one](https://brightdata.com/cp/start); new accounts get
|
|
58
|
+
[5,000 free credits a month](https://docs.brightdata.com/general/account/billing-and-pricing/free-tier).
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import json
|
|
62
|
+
import os
|
|
63
|
+
import time
|
|
64
|
+
import urllib.request
|
|
65
|
+
|
|
66
|
+
API = "https://api.brightdata.com/datasets/v3"
|
|
67
|
+
HEADERS = {
|
|
68
|
+
"Authorization": f"Bearer {os.environ['BRIGHTDATA_API_TOKEN']}",
|
|
69
|
+
"Content-Type": "application/json",
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def call(path, body=None):
|
|
74
|
+
data = None if body is None else json.dumps(body).encode()
|
|
75
|
+
request = urllib.request.Request(API + path, data=data, headers=HEADERS)
|
|
76
|
+
with urllib.request.urlopen(request) as response:
|
|
77
|
+
return json.load(response)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
prompt = {
|
|
81
|
+
"url": "https://chatgpt.com/",
|
|
82
|
+
"prompt": "Best CRM software for small businesses",
|
|
83
|
+
"country": "US",
|
|
84
|
+
"web_search": True,
|
|
85
|
+
}
|
|
86
|
+
job = call("/trigger?dataset_id=gd_m7aof0k82r803d5bjm&include_errors=true", [prompt])
|
|
87
|
+
while call(f"/progress/{job['snapshot_id']}")["status"] not in ("ready", "failed"):
|
|
88
|
+
time.sleep(10)
|
|
89
|
+
answer = call(f"/snapshot/{job['snapshot_id']}?format=json")[0]
|
|
90
|
+
if "error" in answer:
|
|
91
|
+
raise SystemExit(answer["error"])
|
|
92
|
+
print(answer["answer_text"][:120])
|
|
93
|
+
for citation in (answer["citations"] or [])[:3]:
|
|
94
|
+
print("cited:", citation["url"])
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
The best CRM software for a small business depends on whether you primarily need to manage customers, track sales, gener
|
|
99
|
+
cited: https://www.hubspot.com/products/crm?utm_source=chatgpt.com
|
|
100
|
+
cited: https://www.zoho.com/crm/?utm_source=chatgpt.com
|
|
101
|
+
cited: https://www.pipedrive.com/?utm_source=chatgpt.com
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Expect one to five minutes: the API asks ChatGPT and the loop waits for the
|
|
105
|
+
answer. One run with web search and sources took eleven. Billed
|
|
106
|
+
[per answer](https://brightdata.com/pricing/web-scraper).
|
|
107
|
+
|
|
108
|
+
Three things in that snippet are not optional.
|
|
109
|
+
|
|
110
|
+
`url` must be `https://chatgpt.com/`. The scraper requires it on every input;
|
|
111
|
+
the prompt goes in `prompt`.
|
|
112
|
+
|
|
113
|
+
`web_search: true` asks ChatGPT to search the web, which is what usually
|
|
114
|
+
brings citations. Without it, `citations` can come back `null` rather than an
|
|
115
|
+
empty list, so the loop guards for that.
|
|
116
|
+
|
|
117
|
+
`include_errors=true` makes the API report a failed prompt as a row with an
|
|
118
|
+
`error` key. The API default is off.
|
|
119
|
+
|
|
120
|
+
The Bright Data [Python SDK](https://github.com/brightdata/sdk-python) wraps this scraper, but it does not send `require_sources`, so it cannot ask ChatGPT to cite. Hence the
|
|
121
|
+
plain HTTP, here and in this package.
|
|
122
|
+
|
|
123
|
+
## Scrape ChatGPT without an OpenAI API key
|
|
124
|
+
|
|
125
|
+
The ChatGPT Scraper API needs no OpenAI account, no OpenAI API key and no
|
|
126
|
+
ChatGPT login. The only credential is a Bright Data API key, set as
|
|
127
|
+
`BRIGHTDATA_API_TOKEN`. This package and its command also find the key stored
|
|
128
|
+
by `bdata login`.
|
|
129
|
+
|
|
130
|
+
It returns what a person sees in ChatGPT itself, not what the OpenAI API
|
|
131
|
+
returns: the answer with its citations, links, shopping and map blocks, as
|
|
132
|
+
ChatGPT renders them for that country.
|
|
133
|
+
|
|
134
|
+
## Bulk ChatGPT prompt scraper CLI
|
|
135
|
+
|
|
136
|
+
The command in this repo runs the quickstart for several prompts at once and
|
|
137
|
+
writes one JSON file.
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
pip install git+https://github.com/brightdata/chatgpt-scraper
|
|
141
|
+
chatgpt-scraper "Best CRM software for small businesses" "Best project management tool for startups" --web-search
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
Asking ChatGPT 2 prompt(s) from US, with web search
|
|
146
|
+
Usually one to five minutes each. Billed per answer, 5,000 free credits a month.
|
|
147
|
+
asking 'Best CRM software for small businesses'...
|
|
148
|
+
got 'Best CRM software for small businesses': 3,416 characters, 11 citations
|
|
149
|
+
asking 'Best project management tool for startups'...
|
|
150
|
+
got 'Best project management tool for startups': 2,903 characters, 0 citations
|
|
151
|
+
|
|
152
|
+
Saved 2 answers as JSON to chatgpt.json
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
In a terminal the `asking` lines are replaced by a spinner and a running
|
|
156
|
+
clock, updating in place, so you can see it is working and how long it has
|
|
157
|
+
been going.
|
|
158
|
+
|
|
159
|
+
```
|
|
160
|
+
--country CC two-letter country ChatGPT answers from, default US
|
|
161
|
+
--web-search ask ChatGPT to search the web before answering
|
|
162
|
+
--out PATH output file, default chatgpt.json
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
`python -m chatgpt_scraper` works too. The command finds the token in
|
|
166
|
+
`BRIGHTDATA_API_TOKEN`, or, if you ran `npx -p @brightdata/cli bdata login`
|
|
167
|
+
once, in the stored login, for you and for any coding agent in that terminal.
|
|
168
|
+
|
|
169
|
+
Import it instead of running it, for `ok` and `error` per prompt instead of
|
|
170
|
+
raw rows. `scrape` never raises for one bad prompt; check `ok` before reading
|
|
171
|
+
`answer`:
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
from chatgpt_scraper import scrape
|
|
175
|
+
|
|
176
|
+
for outcome in scrape(["Best running shoes for beginners", "Best free photo editing app"]):
|
|
177
|
+
if outcome.ok:
|
|
178
|
+
print(f"{outcome.prompt}: {outcome.answer['answer_text'][:60]}")
|
|
179
|
+
else:
|
|
180
|
+
print(f"{outcome.prompt} failed: {outcome.error}")
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
Best running shoes for beginners: If you're just getting into running, I'd recommend shoes tha
|
|
185
|
+
Best free photo editing app: Best overall free app Snapseed Snapseed Free on iPhone and A
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## Scrape ChatGPT answers, citations and sources
|
|
189
|
+
|
|
190
|
+
There is one ChatGPT scraper, documented in the
|
|
191
|
+
[ChatGPT Scraper API docs](https://docs.brightdata.com/products/scrapers/chatgpt/introduction).
|
|
192
|
+
Every snippet below is complete: paste it as is after the `pip install`
|
|
193
|
+
above. Every one of them runs in Actions each Monday, a smaller check runs
|
|
194
|
+
every other day, and the badge at the top is the latest result.
|
|
195
|
+
|
|
196
|
+
| you want | how |
|
|
197
|
+
| --- | --- |
|
|
198
|
+
| ChatGPT's answer to prompts | `ask(["prompt one", "prompt two"], "US")` |
|
|
199
|
+
| the sites ChatGPT cites | `ask(prompt, web_search=True, require_sources=True)` |
|
|
200
|
+
| a follow-up in the same chat | `ask(prompt, additional_prompt="...")` |
|
|
201
|
+
| the answer from another country | `ask(prompt, "DE")` |
|
|
202
|
+
| the answer as Markdown | `answer_text_markdown` on any answer |
|
|
203
|
+
|
|
204
|
+
`ask` takes one prompt or a list. A list is one job, not one per prompt.
|
|
205
|
+
Each answer is one record, and a prompt the API could not answer comes back
|
|
206
|
+
as a row with an `error` key. Any other input the API accepts goes in as the
|
|
207
|
+
API names it, such as `index`.
|
|
208
|
+
|
|
209
|
+
### Track ChatGPT citations and brand visibility for a prompt
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
from urllib.parse import urlparse
|
|
213
|
+
|
|
214
|
+
from chatgpt_scraper import ask
|
|
215
|
+
|
|
216
|
+
answer = ask("Best project management software for remote teams", web_search=True, require_sources=True)[0]
|
|
217
|
+
domains = [urlparse(c["url"]).netloc for c in answer["citations"]]
|
|
218
|
+
print(len(domains), "citations from", len(set(domains)), "sites:")
|
|
219
|
+
for domain in sorted(set(domains))[:5]:
|
|
220
|
+
print(" ", domain, domains.count(domain))
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
```
|
|
224
|
+
10 citations from 9 sites:
|
|
225
|
+
asana.com 2
|
|
226
|
+
basecamp.com 1
|
|
227
|
+
clickup.com 1
|
|
228
|
+
monday.com 1
|
|
229
|
+
resources.rework.com 1
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
`require_sources` asks ChatGPT to cite, and `web_search` makes it look.
|
|
233
|
+
`citations` holds each cited `url` with its `title`; `links_attached` holds the
|
|
234
|
+
links inside the answer text. Asking the same prompt every day is how GEO
|
|
235
|
+
tools track share of citations over time.
|
|
236
|
+
|
|
237
|
+
### Ask ChatGPT a follow-up prompt
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
from chatgpt_scraper import ask
|
|
241
|
+
|
|
242
|
+
answer = ask(
|
|
243
|
+
"Name three popular sourdough bread recipes",
|
|
244
|
+
additional_prompt="Which of them is easiest for a beginner? One sentence.",
|
|
245
|
+
)[0]
|
|
246
|
+
print("first: ", answer["answer_text"][:80])
|
|
247
|
+
print("second:", answer["additional_answer_text"][:80])
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
```
|
|
251
|
+
first: Here are three popular sourdough bread recipes: 1. Classic Artisan Sourdough Bre
|
|
252
|
+
second: Sourdough sandwich bread is the easiest for beginners because it's soft, forgivi
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
The follow-up runs in the same chat, so it can refer to the first answer. Its
|
|
256
|
+
answer lands in `additional_answer_text`, with its own
|
|
257
|
+
`citations_additional_prompt`.
|
|
258
|
+
|
|
259
|
+
### Compare ChatGPT answers by country
|
|
260
|
+
|
|
261
|
+
```python
|
|
262
|
+
from chatgpt_scraper import ask
|
|
263
|
+
|
|
264
|
+
for country in ("US", "DE"):
|
|
265
|
+
answer = ask("Which supermarket chain has the cheapest groceries?", country, web_search=True)[0]
|
|
266
|
+
print(country, "|", answer["answer_text"][:90])
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
```
|
|
270
|
+
US | ALDI is generally the cheapest traditional supermarket chain in the U.S. , according to 20
|
|
271
|
+
DE | In Germany, Aldi, Lidl, Netto and Penny are generally the cheapest supermarket chains. The
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
`country` is the two-letter country ChatGPT answers from. The US answer named
|
|
275
|
+
ALDI; the German one named Aldi, Lidl, Netto and Penny. `geolocation` is also accepted, but on
|
|
276
|
+
9 Nov 2026 coordinates for Midtown Manhattan got an answer about Houston that
|
|
277
|
+
began "Since your precise location isn't available", so this package does not
|
|
278
|
+
document it.
|
|
279
|
+
|
|
280
|
+
### Get ChatGPT's answer as Markdown
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
from pathlib import Path
|
|
284
|
+
|
|
285
|
+
from chatgpt_scraper import ask
|
|
286
|
+
|
|
287
|
+
answer = ask("Explain what a REST API is, with one short code example")[0]
|
|
288
|
+
markdown = answer["answer_text_markdown"]
|
|
289
|
+
Path("answer.md").write_text(markdown, encoding="utf-8")
|
|
290
|
+
lines = markdown.splitlines()
|
|
291
|
+
headings = [line for line, under in zip(lines, lines[1:]) if under and set(under) == {"-"}]
|
|
292
|
+
print(len(markdown), "characters saved to answer.md; headings:", headings)
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
```
|
|
296
|
+
1854 characters saved to answer.md; headings: ['How a REST API works', 'Short code example', 'In simple terms']
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
`answer_text_markdown` is converted from the rendered page: headings are
|
|
300
|
+
underlined with dashes rather than written with `#`, tables are pipe tables,
|
|
301
|
+
and code is indented four spaces under a line naming the language, not
|
|
302
|
+
fenced. `answer_text` is the same answer as plain text, and `answer_html` the
|
|
303
|
+
full page, about 1.4 MB.
|
|
304
|
+
|
|
305
|
+
### Trigger now, fetch later
|
|
306
|
+
|
|
307
|
+
For a long list of prompts, do not block a process for an hour. Trigger, keep
|
|
308
|
+
the snapshot id, fetch when ready. Snapshots stay downloadable for 30 days.
|
|
309
|
+
|
|
310
|
+
```python
|
|
311
|
+
import time
|
|
312
|
+
|
|
313
|
+
from chatgpt_scraper import CHATGPT, fetch, status, trigger
|
|
314
|
+
|
|
315
|
+
item = {"url": "https://chatgpt.com/", "prompt": "What is the capital of Australia?", "country": "US"}
|
|
316
|
+
snapshot = trigger(CHATGPT, [item])
|
|
317
|
+
print("snapshot:", snapshot)
|
|
318
|
+
while (state := status(snapshot)) not in ("ready", "failed"):
|
|
319
|
+
time.sleep(10)
|
|
320
|
+
print("status:", state)
|
|
321
|
+
print("fetched:", fetch(snapshot)[0]["answer_text"][:60])
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
```
|
|
325
|
+
snapshot: sd_mw9bv3ophde91g5k7
|
|
326
|
+
status: ready
|
|
327
|
+
fetched: The capital of Australia is Canberra . 🇦🇺 It is located in t
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
The input fields the live API accepted on 2026-11-09, for dataset
|
|
331
|
+
`gd_m7aof0k82r803d5bjm`:
|
|
332
|
+
|
|
333
|
+
| input | required | what it does |
|
|
334
|
+
| --- | --- | --- |
|
|
335
|
+
| `url` | yes | always `https://chatgpt.com/` |
|
|
336
|
+
| `prompt` | yes | the question |
|
|
337
|
+
| `country` | no | two-letter country ChatGPT answers from |
|
|
338
|
+
| `web_search` | no | `true` asks ChatGPT to search the web |
|
|
339
|
+
| `require_sources` | no | `true` asks ChatGPT to cite sources |
|
|
340
|
+
| `additional_prompt` | no | a follow-up in the same chat |
|
|
341
|
+
| `index` | no | your own number, returned on the record |
|
|
342
|
+
| `geolocation` | no | `"lat,long"`; not honoured in our tests |
|
|
343
|
+
|
|
344
|
+
`bdata pipelines info gd_m7aof0k82r803d5bjm collect_by_url` prints the input
|
|
345
|
+
fields the API validates.
|
|
346
|
+
|
|
347
|
+
## ChatGPT answer data fields (JSON)
|
|
348
|
+
|
|
349
|
+
The fields most people want from an answer:
|
|
350
|
+
|
|
351
|
+
```
|
|
352
|
+
prompt answer_text answer_text_markdown citations links_attached country web_search_triggered
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
`web_search_triggered` does not follow the `web_search` input: on
|
|
356
|
+
9 Nov 2026 it read `false` for an answer with ten citations and `true` for a
|
|
357
|
+
prompt sent without `web_search`. `model` was `null` on every record. The code
|
|
358
|
+
hardcodes no field list. Whatever the API returns lands in each record, and
|
|
359
|
+
in the command's file.
|
|
360
|
+
|
|
361
|
+
<!-- fields:start -->
|
|
362
|
+
Regenerated every week from the published schema,
|
|
363
|
+
[scrapers-full.json](https://docs.brightdata.com/scrapers-full.json), so they cannot go stale. A record
|
|
364
|
+
carries the fields that apply to it, plus `timestamp` and `input`; the
|
|
365
|
+
sample file's first answer has 33 keys.
|
|
366
|
+
|
|
367
|
+
<details>
|
|
368
|
+
<summary>All 32 ChatGPT answer fields, <code>gd_m7aof0k82r803d5bjm</code></summary>
|
|
369
|
+
|
|
370
|
+
| field | type | description |
|
|
371
|
+
| --- | --- | --- |
|
|
372
|
+
| `url` | url | ChatGPT page URL |
|
|
373
|
+
| `prompt` | text | The input prompt |
|
|
374
|
+
| `answer_html` | text | The answer of ChatGPT search |
|
|
375
|
+
| `answer_text` | text | Answer text in plain text |
|
|
376
|
+
| `links_attached` | array | Hyperlinks included in the response text, along with their position and linked words. |
|
|
377
|
+
| `citations` | array | Sources directly referenced in the answer, often linked to specific facts or claims. |
|
|
378
|
+
| `country` | country | Country from which to perform the search |
|
|
379
|
+
| `is_map` | boolean | Indication whether a map package result was generated |
|
|
380
|
+
| `references` | array | Links grouped under the References section |
|
|
381
|
+
| `shopping` | array | Product recommendation cards from ChatGPT's Shopping panel |
|
|
382
|
+
| `shopping_visible` | boolean | Whether the Shopping panel appeared for this answer |
|
|
383
|
+
| `index` | number | The index column lets you assign a unique ID to each crawl request for easy tracking. |
|
|
384
|
+
| `answer_text_markdown` | html2markdown | Answer text in markdown format. |
|
|
385
|
+
| `web_search_triggered` | boolean | Was a web search triggered by ChatGPT when returning the answer |
|
|
386
|
+
| `additional_prompt` | text | The additional_prompt is a follow-up input sent after receiving the first answer, aiming to clarify, expand, or refine the initial response. The system processes this prompt and returns the result as additional_answer_text. |
|
|
387
|
+
| `additional_answer_text` | text | The additional_answer_text is the system's response to the additional_prompt, which follows the initial interaction. It provides additional information, clarification, or a refined answer based on the user's follow-up input. |
|
|
388
|
+
| `map` | array | Map package result |
|
|
389
|
+
| `search_sources` | array | A list of external web pages ChatGPT used to gather information, usually shown at the bottom of the response. |
|
|
390
|
+
| `search_sources_more` | array | A list of external web pages ChatGPT used to gather information, usually shown at the bottom of the response (from "More" partition). |
|
|
391
|
+
| `response_raw` | text | Raw data of ChatGPT's response |
|
|
392
|
+
| `response_raw_additional_prompt` | text | Raw data of ChatGPT's response for additional prompt. |
|
|
393
|
+
| `answer_section_html` | text | The answer's section html |
|
|
394
|
+
| `model` | text | Model used for answer |
|
|
395
|
+
| `web_search_query` | array | Model-generated web search queries (exact strings, in execution order) |
|
|
396
|
+
| `prompt_sent_at` | date | exact prompt timestamp |
|
|
397
|
+
| `answer_text_raw` | text | Original answer_text as extracted, without trimming or whitespace normalization (may include leading/trailing spaces and newline characters). |
|
|
398
|
+
| `business_locations` | array | Please consider adding new fields, attaching example of JSON we can get for each business location mentioned in answer. Array of business location entities returned when the answer includes local business results. Each entry contains the business ID, coordinates, name, category, address, contact info, ratings, hours, images, and open/closed status. |
|
|
399
|
+
| `ads` | array | Array of advertisement's card data. |
|
|
400
|
+
| `additional_answer_text_markdown` | html2markdown | additional_answer_text_markdown in markdown format. |
|
|
401
|
+
| `questionnaire` | array | Questionnaire |
|
|
402
|
+
| `shopping_additional_prompt` | array | Product recommendation cards from ChatGPT's Shopping panel from results of additional prompt |
|
|
403
|
+
| `citations_additional_prompt` | array | Sources directly referenced in the answer of additional prompt, often linked to specific facts or claims. |
|
|
404
|
+
|
|
405
|
+
</details>
|
|
406
|
+
<!-- fields:end -->
|
|
407
|
+
|
|
408
|
+
<details>
|
|
409
|
+
<summary>The start of a real output file, from <code>chatgpt-scraper "Best CRM software for small businesses"</code></summary>
|
|
410
|
+
|
|
411
|
+
Each line is cut at 160 characters here; `answer_html` alone is a 1.4 MB line.
|
|
412
|
+
|
|
413
|
+
```json
|
|
414
|
+
{
|
|
415
|
+
"generated_at": "2026-11-09T14:00:41.695758+00:00",
|
|
416
|
+
"country": "US",
|
|
417
|
+
"prompts": [
|
|
418
|
+
{
|
|
419
|
+
"prompt": "Best CRM software for small businesses",
|
|
420
|
+
"answer": {
|
|
421
|
+
"url": "https://chatgpt.com/?q=Best%20CRM%20software%20for%20small%20businesses",
|
|
422
|
+
"prompt": "Best CRM software for small businesses",
|
|
423
|
+
"answer_html": "<html class=\"x108lcm5 xzx5cyk xojncy5 xy3g5xu x1nktip0 xwbwmuu x1it3fuk xh8yej3 xuu8av0 xeuugli xgs08xk x1javzw2 xfqjyog x1rob05q xvmah...
|
|
424
|
+
"answer_text": "The best CRM software for a small business depends on whether you need to manage leads, track sales, automate follow-ups, manage custome...
|
|
425
|
+
"links_attached": [
|
|
426
|
+
{
|
|
427
|
+
"url": "https://www.hubspot.com/products/crm?utm_source=chatgpt.com",
|
|
428
|
+
"text": "HubSpot website",
|
|
429
|
+
"position": 1
|
|
430
|
+
},
|
|
431
|
+
{
|
|
432
|
+
"url": "https://www.zoho.com/crm/?utm_source=chatgpt.com",
|
|
433
|
+
"text": "Zoho website",
|
|
434
|
+
...
|
|
435
|
+
```
|
|
436
|
+
|
|
437
|
+
The whole file, one answer with every field, is
|
|
438
|
+
[examples/sample_output.json](https://github.com/brightdata/chatgpt-scraper/blob/main/examples/sample_output.json).
|
|
439
|
+
|
|
440
|
+
</details>
|
|
441
|
+
|
|
442
|
+
## ChatGPT scraper errors
|
|
443
|
+
|
|
444
|
+
| you see | what it means |
|
|
445
|
+
| --- | --- |
|
|
446
|
+
| `API token required but not found.` | Exit 2, before any request. Set the token, or run `bdata login`. |
|
|
447
|
+
| `failed 'prompt': ...` | Exit 1. The API returned an error row for that prompt instead of an answer. The message is the API's. |
|
|
448
|
+
| `failed 'prompt': timeout: snapshot ... not ready after 900s` | Exit 1. Run it again, or fetch the snapshot later with `fetch()`. |
|
|
449
|
+
| `failed 'prompt': HTTP 401: ...` | The token is set but wrong. |
|
|
450
|
+
|
|
451
|
+
Any failure exits 1, so a run is safe to gate a script on.
|
|
452
|
+
|
|
453
|
+
From the functions, the same conditions look like this:
|
|
454
|
+
|
|
455
|
+
| you see | what it means |
|
|
456
|
+
| --- | --- |
|
|
457
|
+
| `TokenMissing` | No `BRIGHTDATA_API_TOKEN` and no `bdata login`. |
|
|
458
|
+
| `ApiError: HTTP 400: ...validation_error...` | An input the API does not accept. The message names the field. |
|
|
459
|
+
| a row with an `error` key | The API's answer for one prompt. The other rows are fine. |
|
|
460
|
+
| `TimeoutError` | The job outlived the 900-second wait. Pass `timeout=` to `run`, or use `trigger` and fetch later. |
|
|
461
|
+
|
|
462
|
+
## ChatGPT MCP server and Claude Code skill
|
|
463
|
+
|
|
464
|
+
No Python, nothing installed. Paste both lines; the first opens a browser
|
|
465
|
+
once, or use `bdata login --device` over SSH and in CI:
|
|
466
|
+
|
|
467
|
+
```bash
|
|
468
|
+
npx -p @brightdata/cli bdata login
|
|
469
|
+
npx -p @brightdata/cli bdata pipelines run gd_m7aof0k82r803d5bjm collect_by_url --url "https://chatgpt.com/" --prompt "Best CRM software for small businesses" --country US --pretty
|
|
470
|
+
```
|
|
471
|
+
|
|
472
|
+
1 record, 19 fields, 75 seconds on 9 Nov 2026. The CLI leaves out fields that
|
|
473
|
+
are null. The record is about 3 MB of JSON, mostly `answer_html`.
|
|
474
|
+
|
|
475
|
+
`npx skills add brightdata/skills` teaches Claude Code, Cursor and Codex these
|
|
476
|
+
commands and the docs, so plain language works afterwards. Full guide:
|
|
477
|
+
[Bright Data for your coding agent](https://docs.brightdata.com/quickstart-coding-agent).
|
|
478
|
+
|
|
479
|
+
No terminal, for a hosted assistant? The
|
|
480
|
+
[Bright Data MCP server](https://github.com/brightdata/brightdata-mcp#which-tool-to-use)
|
|
481
|
+
runs any pre-built scraper, ChatGPT included, through `search_scrapers` and
|
|
482
|
+
`run_scraper` in its `scrapers` group:
|
|
483
|
+
|
|
484
|
+
https://mcp.brightdata.com/mcp?token=YOUR_API_TOKEN&groups=scrapers
|
|
485
|
+
|
|
486
|
+
An agent can also open the account itself, no signup form:
|
|
487
|
+
[agent registration](https://brightdata.com/auth.md). Everything else Bright
|
|
488
|
+
Data connects to, from LangChain to Zapier and n8n:
|
|
489
|
+
[integrations](https://docs.brightdata.com/integrations/introduction).
|
|
490
|
+
|
|
491
|
+
## Is this ChatGPT scraper free?
|
|
492
|
+
|
|
493
|
+
New Bright Data accounts get
|
|
494
|
+
[5,000 free credits a month](https://docs.brightdata.com/general/account/billing-and-pricing/free-tier).
|
|
495
|
+
The ChatGPT scraper bills per record, one per answer.
|
|
496
|
+
[Pricing](https://brightdata.com/pricing/web-scraper) lists the rates.
|
|
497
|
+
|
|
498
|
+
## Support
|
|
499
|
+
|
|
500
|
+
Bugs in this repo:
|
|
501
|
+
[open an issue](https://github.com/brightdata/chatgpt-scraper/issues), and
|
|
502
|
+
[CONTRIBUTING.md](https://github.com/brightdata/chatgpt-scraper/blob/main/CONTRIBUTING.md) says what to put in it.
|
|
503
|
+
Anything about the API, your account or your credits:
|
|
504
|
+
[Bright Data support](https://brightdata.zendesk.com/hc/en-us/requests/new).
|
|
505
|
+
|
|
506
|
+
## License
|
|
507
|
+
|
|
508
|
+
[Apache 2.0](https://github.com/brightdata/chatgpt-scraper/blob/main/LICENSE).
|
|
509
|
+
|
|
510
|
+
## ChatGPT scraper for Node.js
|
|
511
|
+
|
|
512
|
+
The same ChatGPT scraper is available for Node.js: [brightdata/chatgpt-scraper-node](https://github.com/brightdata/chatgpt-scraper-node).
|
|
513
|
+
|
|
514
|
+
## More open source scrapers from Bright Data
|
|
515
|
+
|
|
516
|
+
Other Python scrapers built the same way, with live-verified examples:
|
|
517
|
+
|
|
518
|
+
- [URL to Markdown (Fetch API) for Python](https://github.com/brightdata/fetch-api-scraper)
|
|
519
|
+
- [YouTube scraper for Python](https://github.com/brightdata/youtube-scraper)
|
|
520
|
+
- [LinkedIn scraper for Python](https://github.com/brightdata/linkedin-scraper)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
chatgpt_scraper/__init__.py,sha256=TvQg_18qzxhjmEJ0Pyx7ElMYX7nkmSpmpX62W8m1JT0,460
|
|
2
|
+
chatgpt_scraper/__main__.py,sha256=zMvz-TOj-wD0lXiX6Swvs8d_bLMTdpgKWwsBnxL2dBo,3365
|
|
3
|
+
chatgpt_scraper/scrape.py,sha256=Ft_H6D2PaW56IgQhkMhlBbZSv3gFRM4Qv4gcvgpJzSs,9117
|
|
4
|
+
brightdata_chatgpt_scraper-0.1.0.dist-info/METADATA,sha256=t8hxEvqx6cYcYYQ-Is0TXtUcD_pTNGjRXd58G5JeraA,23967
|
|
5
|
+
brightdata_chatgpt_scraper-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
6
|
+
brightdata_chatgpt_scraper-0.1.0.dist-info/entry_points.txt,sha256=ejojuapx1JZxTG4LxOX8B2ewKdDRkeRJMnw8vlXLNkQ,66
|
|
7
|
+
brightdata_chatgpt_scraper-0.1.0.dist-info/licenses/LICENSE,sha256=yVuuHRzgI17MzTVgt3LsHvuX80innw--CmNPDCzO_iw,11358
|
|
8
|
+
brightdata_chatgpt_scraper-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
|
202
|
+
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""A ChatGPT scraper on Bright Data's Scraper API."""
|
|
2
|
+
|
|
3
|
+
from .scrape import (
|
|
4
|
+
CHATGPT,
|
|
5
|
+
ApiError,
|
|
6
|
+
Outcome,
|
|
7
|
+
TokenMissing,
|
|
8
|
+
ask,
|
|
9
|
+
ask_one,
|
|
10
|
+
fetch,
|
|
11
|
+
run,
|
|
12
|
+
scrape,
|
|
13
|
+
status,
|
|
14
|
+
token,
|
|
15
|
+
trigger,
|
|
16
|
+
write,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"CHATGPT",
|
|
21
|
+
"ApiError",
|
|
22
|
+
"Outcome",
|
|
23
|
+
"TokenMissing",
|
|
24
|
+
"ask",
|
|
25
|
+
"ask_one",
|
|
26
|
+
"fetch",
|
|
27
|
+
"run",
|
|
28
|
+
"scrape",
|
|
29
|
+
"status",
|
|
30
|
+
"token",
|
|
31
|
+
"trigger",
|
|
32
|
+
"write",
|
|
33
|
+
]
|
|
34
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""chatgpt-scraper "best crm for startups", or python -m chatgpt_scraper "best crm for startups" """
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import re
|
|
7
|
+
import sys
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
|
|
12
|
+
from .scrape import TokenMissing, ask_one, token, write
|
|
13
|
+
|
|
14
|
+
FRAMES = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def country_code(value: str) -> str:
|
|
18
|
+
if not re.fullmatch(r"[A-Za-z]{2}", value):
|
|
19
|
+
raise argparse.ArgumentTypeError(f"{value!r} is not a two-letter country code, like US")
|
|
20
|
+
return value.upper()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@contextmanager
|
|
24
|
+
def waiting(label: str):
|
|
25
|
+
"""A spinner and a running clock, so minutes of waiting look alive.
|
|
26
|
+
|
|
27
|
+
Outside a terminal it prints one plain line instead: a log wants a line,
|
|
28
|
+
not an animation.
|
|
29
|
+
"""
|
|
30
|
+
if not sys.stdout.isatty():
|
|
31
|
+
print(f"asking {label}...", flush=True)
|
|
32
|
+
yield
|
|
33
|
+
return
|
|
34
|
+
done = threading.Event()
|
|
35
|
+
started = time.monotonic()
|
|
36
|
+
|
|
37
|
+
def spin() -> None:
|
|
38
|
+
frame = 0
|
|
39
|
+
while not done.wait(0.1):
|
|
40
|
+
secs = int(time.monotonic() - started)
|
|
41
|
+
clock = f"{secs // 3600}:{secs // 60 % 60:02d}:{secs % 60:02d}"
|
|
42
|
+
sys.stdout.write(f"\r\033[2K{FRAMES[frame % len(FRAMES)]} {label} {clock}")
|
|
43
|
+
sys.stdout.flush()
|
|
44
|
+
frame += 1
|
|
45
|
+
|
|
46
|
+
thread = threading.Thread(target=spin, daemon=True)
|
|
47
|
+
thread.start()
|
|
48
|
+
try:
|
|
49
|
+
yield
|
|
50
|
+
finally:
|
|
51
|
+
done.set()
|
|
52
|
+
thread.join()
|
|
53
|
+
sys.stdout.write("\r\033[2K") # leave the line clean for the result
|
|
54
|
+
sys.stdout.flush()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def main(argv: list[str] | None = None) -> int:
|
|
58
|
+
parser = argparse.ArgumentParser(
|
|
59
|
+
prog="chatgpt-scraper",
|
|
60
|
+
description="Ask ChatGPT one or more prompts and save its answers with their citations.",
|
|
61
|
+
)
|
|
62
|
+
parser.add_argument("prompts", nargs="+", help="what you would type into ChatGPT, quoted")
|
|
63
|
+
parser.add_argument("--country", type=country_code, default="US",
|
|
64
|
+
help="two-letter country ChatGPT answers from, default %(default)s")
|
|
65
|
+
parser.add_argument("--web-search", action="store_true",
|
|
66
|
+
help="ask ChatGPT to search the web before answering")
|
|
67
|
+
parser.add_argument("--out", default="chatgpt.json", help="output file, default %(default)s")
|
|
68
|
+
args = parser.parse_args(argv)
|
|
69
|
+
|
|
70
|
+
print(
|
|
71
|
+
f"Asking ChatGPT {len(args.prompts)} prompt(s) from {args.country}"
|
|
72
|
+
f"{', with web search' if args.web_search else ''}\n"
|
|
73
|
+
"Usually one to five minutes each. Billed per answer, 5,000 free credits a month.",
|
|
74
|
+
flush=True, # piped, an unflushed header would print after the error below
|
|
75
|
+
)
|
|
76
|
+
try:
|
|
77
|
+
token() # fail before the first wait, not after it
|
|
78
|
+
except TokenMissing as exc:
|
|
79
|
+
print(exc, file=sys.stderr)
|
|
80
|
+
return 2
|
|
81
|
+
|
|
82
|
+
outcomes = []
|
|
83
|
+
for prompt in args.prompts:
|
|
84
|
+
label = prompt if len(prompt) <= 50 else prompt[:47] + "..."
|
|
85
|
+
with waiting(repr(label)):
|
|
86
|
+
outcome = ask_one(prompt, args.country, True if args.web_search else None)
|
|
87
|
+
print(outcome.line(), flush=True)
|
|
88
|
+
outcomes.append(outcome)
|
|
89
|
+
|
|
90
|
+
path = write(outcomes, args.out, args.country)
|
|
91
|
+
answered = sum(1 for o in outcomes if o.ok)
|
|
92
|
+
print(f"\nSaved {answered} answers as JSON to {path}")
|
|
93
|
+
return 0 if all(o.ok for o in outcomes) else 1
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
if __name__ == "__main__":
|
|
97
|
+
sys.exit(main())
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""ChatGPT answers, citations and sources on the Bright Data Scraper API.
|
|
2
|
+
|
|
3
|
+
prompt -> trigger -> poll progress -> fetch snapshot -> ChatGPT's answer
|
|
4
|
+
|
|
5
|
+
Standard library only. The Python SDK wraps this scraper but sends no
|
|
6
|
+
`require_sources`, so this module talks to the API over HTTP and passes every
|
|
7
|
+
input the scraper accepts. Every call is an asynchronous job: trigger it, poll
|
|
8
|
+
until the snapshot is ready, download it. On 2026-11-09 an answer took 65 to
|
|
9
|
+
282 seconds.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
import urllib.error
|
|
19
|
+
import urllib.parse
|
|
20
|
+
import urllib.request
|
|
21
|
+
from collections.abc import Iterable
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from datetime import datetime, timezone
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
API = "https://api.brightdata.com/datasets/v3"
|
|
28
|
+
|
|
29
|
+
#: The one ChatGPT scraper: a prompt in, ChatGPT's answer out, with its citations.
|
|
30
|
+
CHATGPT = "gd_m7aof0k82r803d5bjm"
|
|
31
|
+
|
|
32
|
+
#: Every prompt is sent to this URL. The scraper requires it as an input.
|
|
33
|
+
CHATGPT_URL = "https://chatgpt.com/"
|
|
34
|
+
|
|
35
|
+
#: The slowest answer on 2026-11-09 took 282 seconds; the wait outlasts that comfortably.
|
|
36
|
+
TIMEOUT = 900
|
|
37
|
+
POLL_SECONDS = 10
|
|
38
|
+
|
|
39
|
+
NO_TOKEN = (
|
|
40
|
+
"API token required but not found.\n"
|
|
41
|
+
" export BRIGHTDATA_API_TOKEN=YOUR_API_KEY token: https://brightdata.com/cp/setting/users\n"
|
|
42
|
+
" or run once: npx -p @brightdata/cli bdata login"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ApiError(Exception):
|
|
47
|
+
"""The API refused a request, or a job ended without a snapshot."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TokenMissing(ApiError):
|
|
51
|
+
"""No token in the environment and no Bright Data CLI login."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _cli_credentials() -> Path:
|
|
55
|
+
"""Where `bdata login` stores the key, the same path the Python SDK reads."""
|
|
56
|
+
if sys.platform == "win32":
|
|
57
|
+
base = Path(os.environ.get("APPDATA", str(Path.home() / "AppData" / "Roaming")))
|
|
58
|
+
elif sys.platform == "darwin":
|
|
59
|
+
base = Path.home() / "Library" / "Application Support"
|
|
60
|
+
else:
|
|
61
|
+
base = Path.home() / ".config"
|
|
62
|
+
return base / "brightdata-cli" / "credentials.json"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def token() -> str:
|
|
66
|
+
"""BRIGHTDATA_API_TOKEN, else the Bright Data CLI login. Never a key in code."""
|
|
67
|
+
found = os.environ.get("BRIGHTDATA_API_TOKEN", "").strip()
|
|
68
|
+
if found:
|
|
69
|
+
return found
|
|
70
|
+
try:
|
|
71
|
+
found = json.loads(_cli_credentials().read_text(encoding="utf-8")).get("api_key", "")
|
|
72
|
+
except (OSError, ValueError, AttributeError):
|
|
73
|
+
found = ""
|
|
74
|
+
if not found:
|
|
75
|
+
raise TokenMissing(NO_TOKEN)
|
|
76
|
+
return found
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _call(path: str, body: Any = None) -> Any:
|
|
80
|
+
data = None if body is None else json.dumps(body).encode()
|
|
81
|
+
request = urllib.request.Request(
|
|
82
|
+
API + path,
|
|
83
|
+
data=data,
|
|
84
|
+
headers={"Authorization": f"Bearer {token()}", "Content-Type": "application/json"},
|
|
85
|
+
)
|
|
86
|
+
try:
|
|
87
|
+
with urllib.request.urlopen(request, timeout=120) as response:
|
|
88
|
+
return json.loads(response.read() or b"null")
|
|
89
|
+
except urllib.error.HTTPError as exc:
|
|
90
|
+
detail = exc.read().decode(errors="replace").strip()
|
|
91
|
+
raise ApiError(f"HTTP {exc.code}: {detail[:300]}") from None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def trigger(
|
|
95
|
+
dataset_id: str,
|
|
96
|
+
inputs: list[dict[str, Any]],
|
|
97
|
+
*,
|
|
98
|
+
discover_by: str | None = None,
|
|
99
|
+
limit_per_input: int | None = None,
|
|
100
|
+
) -> str:
|
|
101
|
+
"""Start a job and return its snapshot id. Error rows are always asked for."""
|
|
102
|
+
query: dict[str, Any] = {"dataset_id": dataset_id, "include_errors": "true"}
|
|
103
|
+
if discover_by:
|
|
104
|
+
query.update(type="discover_new", discover_by=discover_by)
|
|
105
|
+
if limit_per_input is not None:
|
|
106
|
+
query["limit_per_input"] = limit_per_input
|
|
107
|
+
return _call("/trigger?" + urllib.parse.urlencode(query), inputs)["snapshot_id"]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def status(snapshot_id: str) -> str:
|
|
111
|
+
"""running, ready or failed."""
|
|
112
|
+
return _call(f"/progress/{snapshot_id}")["status"]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def fetch(snapshot_id: str) -> list[dict[str, Any]]:
|
|
116
|
+
"""The rows of a ready snapshot. Snapshots stay downloadable for 30 days."""
|
|
117
|
+
data = _call(f"/snapshot/{snapshot_id}?format=json")
|
|
118
|
+
return [row for row in (data if isinstance(data, list) else [data]) if isinstance(row, dict)]
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _poll(snapshot_id: str) -> str:
|
|
122
|
+
"""status(), except that a dropped connection is not the end of a paid job.
|
|
123
|
+
|
|
124
|
+
A poll is safe to repeat. A Node run of this same loop lost a job to one
|
|
125
|
+
"fetch failed" on 2026-10-24, five minutes in. An HTTP error still raises.
|
|
126
|
+
"""
|
|
127
|
+
try:
|
|
128
|
+
return status(snapshot_id)
|
|
129
|
+
except (urllib.error.URLError, OSError):
|
|
130
|
+
return "running"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def run(
|
|
134
|
+
dataset_id: str,
|
|
135
|
+
inputs: list[dict[str, Any]],
|
|
136
|
+
*,
|
|
137
|
+
discover_by: str | None = None,
|
|
138
|
+
limit_per_input: int | None = None,
|
|
139
|
+
timeout: float = TIMEOUT,
|
|
140
|
+
) -> list[dict[str, Any]]:
|
|
141
|
+
"""Trigger, wait, fetch. Rows the API could not collect carry an `error` key."""
|
|
142
|
+
snapshot_id = trigger(
|
|
143
|
+
dataset_id, inputs, discover_by=discover_by, limit_per_input=limit_per_input
|
|
144
|
+
)
|
|
145
|
+
deadline = time.monotonic() + timeout
|
|
146
|
+
while (state := _poll(snapshot_id)) not in ("ready", "failed"):
|
|
147
|
+
if time.monotonic() > deadline:
|
|
148
|
+
raise TimeoutError(f"timeout: snapshot {snapshot_id} not ready after {timeout:.0f}s")
|
|
149
|
+
time.sleep(POLL_SECONDS)
|
|
150
|
+
if state == "failed":
|
|
151
|
+
raise ApiError(f"snapshot {snapshot_id} failed")
|
|
152
|
+
return fetch(snapshot_id)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _many(value: str | Iterable[str]) -> list[str]:
|
|
156
|
+
return [value] if isinstance(value, str) else list(value)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _set(**fields: Any) -> dict[str, Any]:
|
|
160
|
+
"""Only the fields that were actually set."""
|
|
161
|
+
return {name: value for name, value in fields.items() if value is not None}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def ask(
|
|
165
|
+
prompts: str | Iterable[str],
|
|
166
|
+
country: str = "US",
|
|
167
|
+
*,
|
|
168
|
+
web_search: bool | None = None,
|
|
169
|
+
require_sources: bool | None = None,
|
|
170
|
+
additional_prompt: str | None = None,
|
|
171
|
+
**options: Any,
|
|
172
|
+
) -> list[dict[str, Any]]:
|
|
173
|
+
"""ChatGPT's answer to each prompt, one record per prompt, one job for the list.
|
|
174
|
+
|
|
175
|
+
`web_search` asks ChatGPT to search the web, `require_sources` asks for
|
|
176
|
+
cited sources, and `additional_prompt` is a follow-up asked in the same
|
|
177
|
+
chat. `options` go into each input as the API names them, such as `index`.
|
|
178
|
+
"""
|
|
179
|
+
extra = _set(
|
|
180
|
+
country=country.upper() if country else None,
|
|
181
|
+
web_search=web_search,
|
|
182
|
+
require_sources=require_sources,
|
|
183
|
+
additional_prompt=additional_prompt,
|
|
184
|
+
**options,
|
|
185
|
+
)
|
|
186
|
+
items = [{"url": CHATGPT_URL, "prompt": prompt, **extra} for prompt in _many(prompts)]
|
|
187
|
+
return run(CHATGPT, items)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def split(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[str]]:
|
|
191
|
+
"""Records, and the error messages of the rows that are not records."""
|
|
192
|
+
records = [row for row in rows if not row.get("error")]
|
|
193
|
+
errors = [str(row["error"]) for row in rows if row.get("error")]
|
|
194
|
+
return records, errors
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass
|
|
198
|
+
class Outcome:
|
|
199
|
+
"""What happened to one prompt."""
|
|
200
|
+
|
|
201
|
+
prompt: str
|
|
202
|
+
answer: dict[str, Any] | None = None
|
|
203
|
+
error: str | None = None
|
|
204
|
+
|
|
205
|
+
@property
|
|
206
|
+
def ok(self) -> bool:
|
|
207
|
+
return self.error is None
|
|
208
|
+
|
|
209
|
+
def line(self) -> str:
|
|
210
|
+
"""One line a reader can understand without having read the source."""
|
|
211
|
+
label = self.prompt if len(self.prompt) <= 50 else self.prompt[:47] + "..."
|
|
212
|
+
if not self.ok:
|
|
213
|
+
return f"failed {label!r}: {self.error}"
|
|
214
|
+
text = self.answer.get("answer_text") or ""
|
|
215
|
+
cited = len(self.answer.get("citations") or [])
|
|
216
|
+
return f"got {label!r}: {len(text):,} characters, {cited} citations"
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def ask_one(prompt: str, country: str = "US", web_search: bool | None = None) -> Outcome:
|
|
220
|
+
"""Ask one prompt. Never raises for a bad prompt: a failure becomes an Outcome.
|
|
221
|
+
|
|
222
|
+
A missing token is the exception, because it fails every prompt the same way.
|
|
223
|
+
"""
|
|
224
|
+
outcome = Outcome(prompt=prompt)
|
|
225
|
+
try:
|
|
226
|
+
found, errors = split(ask(prompt, country, web_search=web_search))
|
|
227
|
+
except TokenMissing:
|
|
228
|
+
raise
|
|
229
|
+
except Exception as exc: # one bad prompt must not end the run
|
|
230
|
+
outcome.error = str(exc) if isinstance(exc, (ApiError, TimeoutError)) else (
|
|
231
|
+
f"{type(exc).__name__}: {exc}"
|
|
232
|
+
)
|
|
233
|
+
return outcome
|
|
234
|
+
if found:
|
|
235
|
+
outcome.answer = found[0]
|
|
236
|
+
else:
|
|
237
|
+
outcome.error = "; ".join(dict.fromkeys(errors)) or "no answer and no error row"
|
|
238
|
+
return outcome
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def scrape(
|
|
242
|
+
prompts: Iterable[str], country: str = "US", web_search: bool | None = None
|
|
243
|
+
) -> list[Outcome]:
|
|
244
|
+
"""Ask every prompt and return one Outcome each, in order."""
|
|
245
|
+
return [ask_one(prompt, country, web_search) for prompt in prompts]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def write(outcomes: list[Outcome], path: str | Path, country: str = "US") -> Path:
|
|
249
|
+
"""Write one JSON file: when it ran, and ChatGPT's answer per prompt."""
|
|
250
|
+
document = {
|
|
251
|
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
252
|
+
"country": country.upper(),
|
|
253
|
+
"prompts": [{"prompt": o.prompt, "answer": o.answer} for o in outcomes],
|
|
254
|
+
}
|
|
255
|
+
target = Path(path)
|
|
256
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
257
|
+
target.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
258
|
+
return target
|