brightdata-facebook-scraper 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ # Copy to .env, then load it into the shell: set -a; . ./.env; set +a
2
+ # Token: https://brightdata.com/cp/setting/users
3
+ BRIGHTDATA_API_TOKEN=
@@ -0,0 +1,83 @@
1
+ """Rewrite the README's field tables from the published scraper schemas.
2
+
3
+ python .github/scripts/field_tables.py
4
+
5
+ The Python SDK reads no Facebook dataset schema, so this reads
6
+ https://docs.brightdata.com/scrapers-full.json, the list the Bright Data MCP
7
+ server also loads: every scraper's output fields, with type and description.
8
+ No token needed.
9
+ """
10
+
11
+ import json
12
+ import pathlib
13
+ import re
14
+ import urllib.request
15
+
16
+ ROOT = pathlib.Path(__file__).resolve().parents[2]
17
+ README = ROOT / "README.md"
18
+ START, END = "<!-- fields:start -->", "<!-- fields:end -->"
19
+ SOURCE = "https://docs.brightdata.com/scrapers-full.json"
20
+
21
+ #: The README's order. Each scraper's collect_by_url method lists its full record.
22
+ SCRAPERS = [
23
+ ("gd_lkaxegm826bjpoo9m5", "page post"),
24
+ ("gd_lz11l67o2cb3r0lkj3", "group post"),
25
+ ("gd_lyclm1571iy3mv57zw", "post by URL"),
26
+ ("gd_lkay758p1eanlolqw8", "comment"),
27
+ ("gd_lyclm3ey2q6rww027t", "reel"),
28
+ ("gd_mf124a0511bauquyow", "page"),
29
+ ("gd_mf0urb782734ik94dz", "profile"),
30
+ ("gd_m0dtqpiu1mbcyc2g86", "review"),
31
+ ("gd_lvt9iwuh6fbcwmx1a", "Marketplace listing"),
32
+ ("gd_m14sd0to1jz48ppm51", "event"),
33
+ ("gd_mtveskb62h7q65ftgz", "Ad Library ad"),
34
+ ]
35
+
36
+
37
+ def table(fields):
38
+ rows = ["| field | type | description |", "| --- | --- | --- |"]
39
+ for field in fields:
40
+ desc = (field.get("description") or "").replace(" — ", ": ").replace("—", ":")
41
+ # no fences, no table breaks
42
+ desc = " ".join(desc.split()).replace("`", "'").replace("|", "/")
43
+ rows.append(f"| `{field['name']}` | {field.get('type', '')} | {desc} |")
44
+ return rows
45
+
46
+
47
+ def main():
48
+ text = README.read_text(encoding="utf-8")
49
+ assert START in text and END in text, "README field-table markers are missing"
50
+ with urllib.request.urlopen(SOURCE, timeout=120) as response:
51
+ published = {s["id"]: s for s in json.load(response)}
52
+ sample = json.loads((ROOT / "examples" / "sample_output.json").read_text(encoding="utf-8"))
53
+ first = sample["pages"][0]["posts"][0]
54
+ lines = [
55
+ START,
56
+ "Regenerated every week from the published schema,",
57
+ f"[scrapers-full.json]({SOURCE}), so they cannot go stale. A record",
58
+ "carries the fields that apply to it, plus `timestamp` and `input`; the",
59
+ f"sample file's first post has {len(first)} keys.",
60
+ ]
61
+ counts = []
62
+ for dataset, label in SCRAPERS:
63
+ fields = published[dataset]["scrapers"]["collect_by_url"]["output_fields"]
64
+ counts.append(len(fields))
65
+ lines += [
66
+ "",
67
+ "<details>",
68
+ f"<summary>All {len(fields)} {label} fields, <code>{dataset}</code></summary>",
69
+ "",
70
+ *table(fields),
71
+ "",
72
+ "</details>",
73
+ ]
74
+ lines.append(END)
75
+ new = re.sub(re.escape(START) + r".*?" + re.escape(END), lambda _: "\n".join(lines), text,
76
+ flags=re.S)
77
+ README.write_text(new, encoding="utf-8")
78
+ state = "updated" if new != text else "already current"
79
+ print(f"\nSchemas: {len(SCRAPERS)} scrapers, {sum(counts)} fields. Tables {state}.")
80
+
81
+
82
+ if __name__ == "__main__":
83
+ main()
@@ -0,0 +1,138 @@
1
+ """The README's code blocks, as the live check runs them.
2
+
3
+ python readme_blocks.py list # JSON matrix for the block jobs (ONLY filters)
4
+ python readme_blocks.py run N # run block N with the real token, one retry
5
+ python readme_blocks.py cli-syntax # every README CLI line with a dummy key
6
+
7
+ One parser for all three, so the list and the runner can never disagree on
8
+ what block N is.
9
+ """
10
+
11
+ import json
12
+ import os
13
+ import pathlib
14
+ import re
15
+ import subprocess
16
+ import sys
17
+ import time
18
+
19
+ README = pathlib.Path(__file__).resolve().parents[2] / "README.md"
20
+ SKIP = ("pip install", "export ", "npm i", "npm install",
21
+ "npx -p @brightdata/cli bdata login", "bdata login")
22
+ CLI = os.environ.get("BDATA_CLI", "") # the pinned CLI version; empty means whatever is installed
23
+
24
+
25
+ def blocks():
26
+ readme = README.read_text(encoding="utf-8")
27
+ # never execute anything inside the generated field table: its text comes from
28
+ # the API, and a fence in a description must not become a command
29
+ readme = re.sub(r"<!-- fields:start -->.*?<!-- fields:end -->", "", readme, flags=re.S)
30
+ found, heading, seen = [], "README", {}
31
+ for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S):
32
+ if m.group(1):
33
+ heading = m.group(1).strip() # a block is named after its section, not its first line
34
+ continue
35
+ lang, code = m.group(2), m.group(3)
36
+ lines = [line for line in code.splitlines()
37
+ if line.strip() and not line.strip().startswith("#")]
38
+ if lang == "bash":
39
+ lines = [line for line in lines if not line.startswith(SKIP)]
40
+ if not lines:
41
+ continue
42
+ code = "\n".join(lines)
43
+ seen[heading] = seen.get(heading, 0) + 1
44
+ title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})"
45
+ found.append({"lang": lang, "title": title[:50], "code": code})
46
+ return found
47
+
48
+
49
+ def cli_lines(code, key):
50
+ """The CLI cannot open a browser here, so the key goes in by flag, and an
51
+ unpinned npx is held to the version the job installed."""
52
+ out = []
53
+ for line in code.splitlines():
54
+ if CLI:
55
+ line = line.replace("npx -p @brightdata/cli ", f"npx -p @brightdata/cli@{CLI} ")
56
+ if "bdata pipelines" in line:
57
+ line += f" -k {key}"
58
+ out.append(line)
59
+ return "\n".join(out)
60
+
61
+
62
+ def list_blocks():
63
+ wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()}
64
+ out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]}
65
+ for i, b in enumerate(blocks()) if not wanted or i + 1 in wanted]
66
+ print("blocks=" + json.dumps(out))
67
+
68
+
69
+ def run_block(n):
70
+ b = blocks()[n - 1]
71
+ if b["lang"] == "bash":
72
+ # only the shell ever sees the key's value
73
+ argv = ["bash", "-e", "-c", cli_lines(b["code"], '"$BRIGHTDATA_API_TOKEN"')]
74
+ else:
75
+ argv = [sys.executable, "-c", b["code"]]
76
+ secret = os.environ.get("BRIGHTDATA_API_TOKEN", "")
77
+ for _attempt in (1, 2):
78
+ started = time.time()
79
+ try:
80
+ run = subprocess.run(argv, capture_output=True, text=True, timeout=1080)
81
+ except subprocess.TimeoutExpired:
82
+ run = subprocess.CompletedProcess(
83
+ argv, 1, "", "block timeout: no result within 18 minutes")
84
+ secs = round(time.time() - started)
85
+ out, err = run.stdout.strip(), run.stderr.strip()
86
+ if secret: # summaries are not secret-masked the way logs are
87
+ out, err = out.replace(secret, "***"), err.replace(secret, "***")
88
+ ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err)
89
+ if ok or ((out or err) and "timeout" not in (out + err).lower()):
90
+ break
91
+ time.sleep(30) # the API timed out or answered with nothing at all; one retry
92
+ if ok:
93
+ shown = (out.splitlines() or [""])[:1]
94
+ else:
95
+ # the last few lines: a CLI error often puts the cause above its hint
96
+ shown = (err.splitlines() or out.splitlines() or ["no output"])[-6:]
97
+ body = "\n".join(" " + line[:160] for line in shown)
98
+ state = "ok" if ok else "FAILED"
99
+ print(f"\n`{b['title']}`: {state} in {secs}s (exit {run.returncode})\n\n{body}")
100
+ sys.exit(0 if ok else 1)
101
+
102
+
103
+ def cli_syntax():
104
+ """Run every README CLI line with a key the API rejects. A line the CLI
105
+ still understands reaches the API and gets a 401; a line it no longer
106
+ understands fails before that. Free, and needs no secret."""
107
+ failed = 0
108
+ print("| block | CLI line | result |\n| --- | --- | --- |")
109
+ for i, b in enumerate(blocks(), 1):
110
+ if b["lang"] != "bash":
111
+ continue
112
+ for line in b["code"].splitlines():
113
+ if "bdata pipelines" not in line:
114
+ continue
115
+ run = subprocess.run(["bash", "-c", cli_lines(line, "invalid-key-for-syntax-check")],
116
+ capture_output=True, text=True, timeout=120)
117
+ text = run.stdout + run.stderr
118
+ ok = "401" in text or "Invalid credentials" in text
119
+ last = (run.stderr.strip().splitlines() or run.stdout.strip().splitlines()
120
+ or ["no output"])
121
+ why = "reached the API" if ok else " / ".join(x.strip() for x in last[-3:])[:200]
122
+ print(f"| {i} | `{line[:90]}` | {'ok' if ok else 'FAILED'}: {why.replace('|', '/')} |")
123
+ if not ok:
124
+ failed = 1
125
+ print(f"::error::README block {i}: the CLI rejects `{line}`", file=sys.stderr)
126
+ sys.exit(failed)
127
+
128
+
129
+ if __name__ == "__main__":
130
+ mode = sys.argv[1] if len(sys.argv) > 1 else ""
131
+ if mode == "list":
132
+ list_blocks()
133
+ elif mode == "run":
134
+ run_block(int(sys.argv[2]))
135
+ elif mode == "cli-syntax":
136
+ cli_syntax()
137
+ else:
138
+ sys.exit(__doc__)
@@ -0,0 +1,129 @@
1
+ # Runs by itself on every push and pull request.
2
+ # Never calls the Facebook Scraper API, so it needs no token and costs nothing.
3
+ name: Tests (automatic, no API calls, free)
4
+
5
+ on:
6
+ push:
7
+ pull_request:
8
+ workflow_dispatch: # a Run workflow button, to re-run the tests without a commit
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ concurrency: # a newer push to the same branch or PR replaces the run still going
14
+ group: ci-${{ github.event_name }}-${{ github.ref }}
15
+ cancel-in-progress: true
16
+
17
+ jobs:
18
+ tests:
19
+ name: Run the tests and the code style check
20
+ runs-on: ubuntu-latest
21
+ timeout-minutes: 10
22
+ steps:
23
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
24
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
25
+ with:
26
+ python-version: "3.12"
27
+ cache: pip
28
+ cache-dependency-path: pyproject.toml
29
+ - name: Install the package and its test tools
30
+ run: pip install -e ".[dev]"
31
+ - name: Check code style with ruff
32
+ run: ruff check .
33
+ - name: Run the tests
34
+ run: pytest
35
+ - name: README headings are well formed
36
+ # A section rewrite once produced "## The data## The data" and shipped;
37
+ # a heading with a second run of hashes mid-line is never intended.
38
+ run: |
39
+ if grep -nE '^#+ .*#{2,} ' README.md; then
40
+ echo "::error::malformed heading in README.md"; exit 1
41
+ fi
42
+
43
+ # These install steps shipped broken once, because they were only ever run
44
+ # from a working copy that already had the package. This job follows the
45
+ # README on an empty machine and stops where the tool asks for a token, which
46
+ # proves the clone, the install and the command without spending money.
47
+ coldstart:
48
+ name: Follow the README from scratch on an empty machine
49
+ runs-on: ubuntu-latest
50
+ timeout-minutes: 10
51
+ steps:
52
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
53
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
54
+ with:
55
+ python-version: "3.12"
56
+ - name: Copy the install steps out of the README and run them
57
+ run: |
58
+ set -o pipefail
59
+ python - <<'PY' > "$RUNNER_TEMP/quickstart.sh"
60
+ import os
61
+ import pathlib
62
+ import re
63
+ import sys
64
+
65
+ readme = pathlib.Path("README.md").read_text(encoding="utf-8")
66
+
67
+ def section(title):
68
+ return readme.split(f"## {title}", 1)[1].split("\n## ", 1)[0]
69
+
70
+ def block(text, lang):
71
+ return re.search(r"```" + lang + r"\n(.*?)```", text, re.S).group(1)
72
+
73
+ quick = section("How to scrape Facebook posts with Python")
74
+ snippet = block(quick, "python")
75
+ command = block(section("Bulk Facebook page posts CLI"), "bash")
76
+ for required, where in (("BRIGHTDATA_API_TOKEN", snippet),
77
+ ("urllib.request", snippet),
78
+ ("facebook-scraper", command)):
79
+ if required not in where:
80
+ sys.exit(f"the README no longer contains {required!r} where the cold start expects it")
81
+
82
+ steps = ["# --- Quickstart: the standard library path, nothing to install ---"]
83
+ pathlib.Path(os.environ["RUNNER_TEMP"], "quickstart_snippet.py").write_text(snippet, encoding="utf-8")
84
+ steps += [
85
+ "set +e",
86
+ "python quickstart_snippet.py > out.txt 2> err.txt",
87
+ "code=$?",
88
+ "set -e",
89
+ 'test "$code" != "0" || { echo "the snippet ran without a token?"; cat out.txt; exit 1; }',
90
+ 'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
91
+ 'echo "Quickstart: runs on a bare Python, and stops at the token, as documented."',
92
+ "# --- Or run it as a command: the CLI path ---",
93
+ ]
94
+ for line in command.splitlines():
95
+ if line.startswith("facebook-scraper"):
96
+ continue # run below, so the exit code can be checked
97
+ if line.startswith("pip install") and "git+" in line:
98
+ # The repository is public, so this installs the exact line a
99
+ # reader pastes. On a private fork the URL is unreachable, and
100
+ # the check falls back to the checkout on its own.
101
+ url = line.split("git+", 1)[1].strip() if "git+" in line else ""
102
+ line = "\n".join([
103
+ f'repo="{url}"',
104
+ 'code=$(curl -s -o /dev/null -w "%{http_code}" "$repo" || echo 000)',
105
+ 'if [ "$code" = "200" ]; then',
106
+ f' echo "$repo is public: installing the exact README line"',
107
+ f' {line}',
108
+ "else",
109
+ ' echo "NOTE: $repo returns HTTP $code, so it is not public yet."',
110
+ ' echo "Installing from this checkout instead."',
111
+ ' pip install "$GITHUB_WORKSPACE"',
112
+ "fi",
113
+ ])
114
+ steps.append(line)
115
+ steps += [
116
+ "set +e",
117
+ 'facebook-scraper Meta --limit 1 > out.txt 2> err.txt',
118
+ "code=$?",
119
+ "set -e",
120
+ 'test "$code" = "2" || { echo "expected exit 2, got $code"; cat err.txt; exit 1; }',
121
+ 'grep -qi token err.txt || { echo "no token message"; cat err.txt; exit 1; }',
122
+ 'echo "Success: a new user can follow both README paths from scratch."',
123
+ ]
124
+ print("\n".join(steps))
125
+ PY
126
+ echo "--- these commands came out of README.md ---"
127
+ cat "$RUNNER_TEMP/quickstart.sh"
128
+ echo "--- running them on this empty machine ---"
129
+ cd "$RUNNER_TEMP" && bash -e "$RUNNER_TEMP/quickstart.sh"