brightdata-facebook-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brightdata_facebook_scraper-0.1.0/.env.example +3 -0
- brightdata_facebook_scraper-0.1.0/.github/scripts/field_tables.py +83 -0
- brightdata_facebook_scraper-0.1.0/.github/scripts/readme_blocks.py +138 -0
- brightdata_facebook_scraper-0.1.0/.github/workflows/ci.yml +129 -0
- brightdata_facebook_scraper-0.1.0/.github/workflows/live.yml +341 -0
- brightdata_facebook_scraper-0.1.0/.github/workflows/publish.yml +38 -0
- brightdata_facebook_scraper-0.1.0/.gitignore +7 -0
- brightdata_facebook_scraper-0.1.0/AGENTS.md +75 -0
- brightdata_facebook_scraper-0.1.0/CHANGELOG.md +8 -0
- brightdata_facebook_scraper-0.1.0/CONTRIBUTING.md +27 -0
- brightdata_facebook_scraper-0.1.0/LICENSE +202 -0
- brightdata_facebook_scraper-0.1.0/PKG-INFO +997 -0
- brightdata_facebook_scraper-0.1.0/README.md +971 -0
- brightdata_facebook_scraper-0.1.0/examples/sample_output.json +176 -0
- brightdata_facebook_scraper-0.1.0/pyproject.toml +55 -0
- brightdata_facebook_scraper-0.1.0/src/facebook_scraper/__init__.py +82 -0
- brightdata_facebook_scraper-0.1.0/src/facebook_scraper/__main__.py +97 -0
- brightdata_facebook_scraper-0.1.0/src/facebook_scraper/scrape.py +385 -0
- brightdata_facebook_scraper-0.1.0/tests/test_scrape.py +343 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Rewrite the README's field tables from the published scraper schemas.
|
|
2
|
+
|
|
3
|
+
python .github/scripts/field_tables.py
|
|
4
|
+
|
|
5
|
+
The Python SDK reads no Facebook dataset schema, so this reads
|
|
6
|
+
https://docs.brightdata.com/scrapers-full.json, the list the Bright Data MCP
|
|
7
|
+
server also loads: every scraper's output fields, with type and description.
|
|
8
|
+
No token needed.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import pathlib
|
|
13
|
+
import re
|
|
14
|
+
import urllib.request
|
|
15
|
+
|
|
16
|
+
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
17
|
+
README = ROOT / "README.md"
|
|
18
|
+
START, END = "<!-- fields:start -->", "<!-- fields:end -->"
|
|
19
|
+
SOURCE = "https://docs.brightdata.com/scrapers-full.json"
|
|
20
|
+
|
|
21
|
+
#: The README's order. Each scraper's collect_by_url method lists its full record.
|
|
22
|
+
SCRAPERS = [
|
|
23
|
+
("gd_lkaxegm826bjpoo9m5", "page post"),
|
|
24
|
+
("gd_lz11l67o2cb3r0lkj3", "group post"),
|
|
25
|
+
("gd_lyclm1571iy3mv57zw", "post by URL"),
|
|
26
|
+
("gd_lkay758p1eanlolqw8", "comment"),
|
|
27
|
+
("gd_lyclm3ey2q6rww027t", "reel"),
|
|
28
|
+
("gd_mf124a0511bauquyow", "page"),
|
|
29
|
+
("gd_mf0urb782734ik94dz", "profile"),
|
|
30
|
+
("gd_m0dtqpiu1mbcyc2g86", "review"),
|
|
31
|
+
("gd_lvt9iwuh6fbcwmx1a", "Marketplace listing"),
|
|
32
|
+
("gd_m14sd0to1jz48ppm51", "event"),
|
|
33
|
+
("gd_mtveskb62h7q65ftgz", "Ad Library ad"),
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def table(fields):
|
|
38
|
+
rows = ["| field | type | description |", "| --- | --- | --- |"]
|
|
39
|
+
for field in fields:
|
|
40
|
+
desc = (field.get("description") or "").replace(" — ", ": ").replace("—", ":")
|
|
41
|
+
# no fences, no table breaks
|
|
42
|
+
desc = " ".join(desc.split()).replace("`", "'").replace("|", "/")
|
|
43
|
+
rows.append(f"| `{field['name']}` | {field.get('type', '')} | {desc} |")
|
|
44
|
+
return rows
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def main():
|
|
48
|
+
text = README.read_text(encoding="utf-8")
|
|
49
|
+
assert START in text and END in text, "README field-table markers are missing"
|
|
50
|
+
with urllib.request.urlopen(SOURCE, timeout=120) as response:
|
|
51
|
+
published = {s["id"]: s for s in json.load(response)}
|
|
52
|
+
sample = json.loads((ROOT / "examples" / "sample_output.json").read_text(encoding="utf-8"))
|
|
53
|
+
first = sample["pages"][0]["posts"][0]
|
|
54
|
+
lines = [
|
|
55
|
+
START,
|
|
56
|
+
"Regenerated every week from the published schema,",
|
|
57
|
+
f"[scrapers-full.json]({SOURCE}), so they cannot go stale. A record",
|
|
58
|
+
"carries the fields that apply to it, plus `timestamp` and `input`; the",
|
|
59
|
+
f"sample file's first post has {len(first)} keys.",
|
|
60
|
+
]
|
|
61
|
+
counts = []
|
|
62
|
+
for dataset, label in SCRAPERS:
|
|
63
|
+
fields = published[dataset]["scrapers"]["collect_by_url"]["output_fields"]
|
|
64
|
+
counts.append(len(fields))
|
|
65
|
+
lines += [
|
|
66
|
+
"",
|
|
67
|
+
"<details>",
|
|
68
|
+
f"<summary>All {len(fields)} {label} fields, <code>{dataset}</code></summary>",
|
|
69
|
+
"",
|
|
70
|
+
*table(fields),
|
|
71
|
+
"",
|
|
72
|
+
"</details>",
|
|
73
|
+
]
|
|
74
|
+
lines.append(END)
|
|
75
|
+
new = re.sub(re.escape(START) + r".*?" + re.escape(END), lambda _: "\n".join(lines), text,
|
|
76
|
+
flags=re.S)
|
|
77
|
+
README.write_text(new, encoding="utf-8")
|
|
78
|
+
state = "updated" if new != text else "already current"
|
|
79
|
+
print(f"\nSchemas: {len(SCRAPERS)} scrapers, {sum(counts)} fields. Tables {state}.")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
main()
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""The README's code blocks, as the live check runs them.
|
|
2
|
+
|
|
3
|
+
python readme_blocks.py list # JSON matrix for the block jobs (ONLY filters)
|
|
4
|
+
python readme_blocks.py run N # run block N with the real token, one retry
|
|
5
|
+
python readme_blocks.py cli-syntax # every README CLI line with a dummy key
|
|
6
|
+
|
|
7
|
+
One parser for all three, so the list and the runner can never disagree on
|
|
8
|
+
what block N is.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import pathlib
|
|
14
|
+
import re
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
|
|
19
|
+
README = pathlib.Path(__file__).resolve().parents[2] / "README.md"
|
|
20
|
+
SKIP = ("pip install", "export ", "npm i", "npm install",
|
|
21
|
+
"npx -p @brightdata/cli bdata login", "bdata login")
|
|
22
|
+
CLI = os.environ.get("BDATA_CLI", "") # the pinned CLI version; empty means whatever is installed
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def blocks():
|
|
26
|
+
readme = README.read_text(encoding="utf-8")
|
|
27
|
+
# never execute anything inside the generated field table: its text comes from
|
|
28
|
+
# the API, and a fence in a description must not become a command
|
|
29
|
+
readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
|
|
30
|
+
found, heading, seen = [], "README", {}
|
|
31
|
+
for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
|
|
32
|
+
if m.group(1):
|
|
33
|
+
heading = m.group(1).strip() # a block is named after its section, not its first line
|
|
34
|
+
continue
|
|
35
|
+
lang, code = m.group(2), m.group(3)
|
|
36
|
+
lines = [line for line in code.splitlines()
|
|
37
|
+
if line.strip() and not line.strip().startswith("#")]
|
|
38
|
+
if lang == "bash":
|
|
39
|
+
lines = [line for line in lines if not line.startswith(SKIP)]
|
|
40
|
+
if not lines:
|
|
41
|
+
continue
|
|
42
|
+
code = "\n".join(lines)
|
|
43
|
+
seen[heading] = seen.get(heading, 0) + 1
|
|
44
|
+
title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
|
|
45
|
+
found.append({"lang": lang, "title": title[:50], "code": code})
|
|
46
|
+
return found
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cli_lines(code, key):
|
|
50
|
+
"""The CLI cannot open a browser here, so the key goes in by flag, and an
|
|
51
|
+
unpinned npx is held to the version the job installed."""
|
|
52
|
+
out = []
|
|
53
|
+
for line in code.splitlines():
|
|
54
|
+
if CLI:
|
|
55
|
+
line = line.replace("npx -p @brightdata/cli ", f"npx -p @brightdata/cli@{CLI} ")
|
|
56
|
+
if "bdata pipelines" in line:
|
|
57
|
+
line += f" -k {key}"
|
|
58
|
+
out.append(line)
|
|
59
|
+
return "\n".join(out)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def list_blocks():
|
|
63
|
+
wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()}
|
|
64
|
+
out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]}
|
|
65
|
+
for i, b in enumerate(blocks()) if not wanted or i + 1 in wanted]
|
|
66
|
+
print("blocks=" + json.dumps(out))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def run_block(n):
|
|
70
|
+
b = blocks()[n - 1]
|
|
71
|
+
if b["lang"] == "bash":
|
|
72
|
+
# only the shell ever sees the key's value
|
|
73
|
+
argv = ["bash", "-e", "-c", cli_lines(b["code"], '"$BRIGHTDATA_API_TOKEN"')]
|
|
74
|
+
else:
|
|
75
|
+
argv = [sys.executable, "-c", b["code"]]
|
|
76
|
+
secret = os.environ.get("BRIGHTDATA_API_TOKEN", "")
|
|
77
|
+
for _attempt in (1, 2):
|
|
78
|
+
started = time.time()
|
|
79
|
+
try:
|
|
80
|
+
run = subprocess.run(argv, capture_output=True, text=True, timeout=1080)
|
|
81
|
+
except subprocess.TimeoutExpired:
|
|
82
|
+
run = subprocess.CompletedProcess(
|
|
83
|
+
argv, 1, "", "block timeout: no result within 18 minutes")
|
|
84
|
+
secs = round(time.time() - started)
|
|
85
|
+
out, err = run.stdout.strip(), run.stderr.strip()
|
|
86
|
+
if secret: # summaries are not secret-masked the way logs are
|
|
87
|
+
out, err = out.replace(secret, "***"), err.replace(secret, "***")
|
|
88
|
+
ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err)
|
|
89
|
+
if ok or ((out or err) and "timeout" not in (out + err).lower()):
|
|
90
|
+
break
|
|
91
|
+
time.sleep(30) # the API timed out or answered with nothing at all; one retry
|
|
92
|
+
if ok:
|
|
93
|
+
shown = (out.splitlines() or [""])[:1]
|
|
94
|
+
else:
|
|
95
|
+
# the last few lines: a CLI error often puts the cause above its hint
|
|
96
|
+
shown = (err.splitlines() or out.splitlines() or ["no output"])[-6:]
|
|
97
|
+
body = "\n".join(" " + line[:160] for line in shown)
|
|
98
|
+
state = "ok" if ok else "FAILED"
|
|
99
|
+
print(f"\n`{b['title']}`: {state} in {secs}s (exit {run.returncode})\n\n{body}")
|
|
100
|
+
sys.exit(0 if ok else 1)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def cli_syntax():
|
|
104
|
+
"""Run every README CLI line with a key the API rejects. A line the CLI
|
|
105
|
+
still understands reaches the API and gets a 401; a line it no longer
|
|
106
|
+
understands fails before that. Free, and needs no secret."""
|
|
107
|
+
failed = 0
|
|
108
|
+
print("| block | CLI line | result |\n| --- | --- | --- |")
|
|
109
|
+
for i, b in enumerate(blocks(), 1):
|
|
110
|
+
if b["lang"] != "bash":
|
|
111
|
+
continue
|
|
112
|
+
for line in b["code"].splitlines():
|
|
113
|
+
if "bdata pipelines" not in line:
|
|
114
|
+
continue
|
|
115
|
+
run = subprocess.run(["bash", "-c", cli_lines(line, "invalid-key-for-syntax-check")],
|
|
116
|
+
capture_output=True, text=True, timeout=120)
|
|
117
|
+
text = run.stdout + run.stderr
|
|
118
|
+
ok = "401" in text or "Invalid credentials" in text
|
|
119
|
+
last = (run.stderr.strip().splitlines() or run.stdout.strip().splitlines()
|
|
120
|
+
or ["no output"])
|
|
121
|
+
why = "reached the API" if ok else " / ".join(x.strip() for x in last[-3:])[:200]
|
|
122
|
+
print(f"| {i} | `{line[:90]}` | {'ok' if ok else 'FAILED'}: {why.replace('|', '/')} |")
|
|
123
|
+
if not ok:
|
|
124
|
+
failed = 1
|
|
125
|
+
print(f"::error::README block {i}: the CLI rejects `{line}`", file=sys.stderr)
|
|
126
|
+
sys.exit(failed)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
if __name__ == "__main__":
|
|
130
|
+
mode = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
131
|
+
if mode == "list":
|
|
132
|
+
list_blocks()
|
|
133
|
+
elif mode == "run":
|
|
134
|
+
run_block(int(sys.argv[2]))
|
|
135
|
+
elif mode == "cli-syntax":
|
|
136
|
+
cli_syntax()
|
|
137
|
+
else:
|
|
138
|
+
sys.exit(__doc__)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Runs by itself on every push and pull request.
|
|
2
|
+
# Never calls the Facebook Scraper API, so it needs no token and costs nothing.
|
|
3
|
+
name: Tests (automatic, no API calls, free)
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch: # a Run workflow button, to re-run the tests without a commit
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
concurrency: # a newer push to the same branch or PR replaces the run still going
|
|
14
|
+
group: ci-${{ github.event_name }}-${{ github.ref }}
|
|
15
|
+
cancel-in-progress: true
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
tests:
|
|
19
|
+
name: Run the tests and the code style check
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
timeout-minutes: 10
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
24
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.12"
|
|
27
|
+
cache: pip
|
|
28
|
+
cache-dependency-path: pyproject.toml
|
|
29
|
+
- name: Install the package and its test tools
|
|
30
|
+
run: pip install -e ".[dev]"
|
|
31
|
+
- name: Check code style with ruff
|
|
32
|
+
run: ruff check .
|
|
33
|
+
- name: Run the tests
|
|
34
|
+
run: pytest
|
|
35
|
+
- name: README headings are well formed
|
|
36
|
+
# A section rewrite once produced "## The data## The data" and shipped;
|
|
37
|
+
# a heading with a second run of hashes mid-line is never intended.
|
|
38
|
+
run: |
|
|
39
|
+
if grep -nE '^#+ .*#{2,} ' README.md; then
|
|
40
|
+
echo "::error::malformed heading in README.md"; exit 1
|
|
41
|
+
fi
|
|
42
|
+
|
|
43
|
+
# These install steps shipped broken once, because they were only ever run
|
|
44
|
+
# from a working copy that already had the package. This job follows the
|
|
45
|
+
# README on an empty machine and stops where the tool asks for a token, which
|
|
46
|
+
# proves the clone, the install and the command without spending money.
|
|
47
|
+
coldstart:
|
|
48
|
+
name: Follow the README from scratch on an empty machine
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
timeout-minutes: 10
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
53
|
+
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
|
|
54
|
+
with:
|
|
55
|
+
python-version: "3.12"
|
|
56
|
+
- name: Copy the install steps out of the README and run them
|
|
57
|
+
run: |
|
|
58
|
+
set -o pipefail
|
|
59
|
+
python - <<'PY' > "$RUNNER_TEMP/quickstart.sh"
|
|
60
|
+
import os
|
|
61
|
+
import pathlib
|
|
62
|
+
import re
|
|
63
|
+
import sys
|
|
64
|
+
|
|
65
|
+
readme = pathlib.Path("README.md").read_text(encoding="utf-8")
|
|
66
|
+
|
|
67
|
+
def section(title):
|
|
68
|
+
return readme.split(f"## {title}", 1)[1].split("\n## ", 1)[0]
|
|
69
|
+
|
|
70
|
+
def block(text, lang):
|
|
71
|
+
return re.search(r"```" + lang + r"\n(.*?)```", text, re.S).group(1)
|
|
72
|
+
|
|
73
|
+
quick = section("How to scrape Facebook posts with Python")
|
|
74
|
+
snippet = block(quick, "python")
|
|
75
|
+
command = block(section("Bulk Facebook page posts CLI"), "bash")
|
|
76
|
+
for required, where in (("BRIGHTDATA_API_TOKEN", snippet),
|
|
77
|
+
("urllib.request", snippet),
|
|
78
|
+
("facebook-scraper", command)):
|
|
79
|
+
if required not in where:
|
|
80
|
+
sys.exit(f"the README no longer contains {required!r} where the cold start expects it")
|
|
81
|
+
|
|
82
|
+
steps = ["# --- Quickstart: the standard library path, nothing to install ---"]
|
|
83
|
+
pathlib.Path(os.environ["RUNNER_TEMP"], "quickstart_snippet.py").write_text(snippet, encoding="utf-8")
|
|
84
|
+
steps += [
|
|
85
|
+
"set +e",
|
|
86
|
+
"python quickstart_snippet.py > out.txt 2> err.txt",
|
|
87
|
+
"code=$?",
|
|
88
|
+
"set -e",
|
|
89
|
+
'test "$code" != "0" || { echo "the snippet ran without a token?"; cat out.txt; exit 1; }',
|
|
90
|
+
'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
|
|
91
|
+
'echo "Quickstart: runs on a bare Python, and stops at the token, as documented."',
|
|
92
|
+
"# --- Or run it as a command: the CLI path ---",
|
|
93
|
+
]
|
|
94
|
+
for line in command.splitlines():
|
|
95
|
+
if line.startswith("facebook-scraper"):
|
|
96
|
+
continue # run below, so the exit code can be checked
|
|
97
|
+
if line.startswith("pip install") and "git+" in line:
|
|
98
|
+
# The repository is public, so this installs the exact line a
|
|
99
|
+
# reader pastes. On a private fork the URL is unreachable, and
|
|
100
|
+
# the check falls back to the checkout on its own.
|
|
101
|
+
url = line.split("git+", 1)[1].strip() if "git+" in line else ""
|
|
102
|
+
line = "\n".join([
|
|
103
|
+
f'repo="{url}"',
|
|
104
|
+
'code=$(curl -s -o /dev/null -w "%{http_code}" "$repo" || echo 000)',
|
|
105
|
+
'if [ "$code" = "200" ]; then',
|
|
106
|
+
f' echo "$repo is public: installing the exact README line"',
|
|
107
|
+
f' {line}',
|
|
108
|
+
"else",
|
|
109
|
+
' echo "NOTE: $repo returns HTTP $code, so it is not public yet."',
|
|
110
|
+
' echo "Installing from this checkout instead."',
|
|
111
|
+
' pip install "$GITHUB_WORKSPACE"',
|
|
112
|
+
"fi",
|
|
113
|
+
])
|
|
114
|
+
steps.append(line)
|
|
115
|
+
steps += [
|
|
116
|
+
"set +e",
|
|
117
|
+
'facebook-scraper Meta --limit 1 > out.txt 2> err.txt',
|
|
118
|
+
"code=$?",
|
|
119
|
+
"set -e",
|
|
120
|
+
'test "$code" = "2" || { echo "expected exit 2, got $code"; cat err.txt; exit 1; }',
|
|
121
|
+
'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
|
|
122
|
+
'echo "Success: a new user can follow both README paths from scratch."',
|
|
123
|
+
]
|
|
124
|
+
print("\n".join(steps))
|
|
125
|
+
PY
|
|
126
|
+
echo "--- these commands came out of README.md ---"
|
|
127
|
+
cat "$RUNNER_TEMP/quickstart.sh"
|
|
128
|
+
echo "--- running them on this empty machine ---"
|
|
129
|
+
cd "$RUNNER_TEMP" && bash -e "$RUNNER_TEMP/quickstart.sh"
|