scraper-for-facebook 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scraper_for_facebook-0.1.0/.github/workflows/ci.yml +83 -0
- scraper_for_facebook-0.1.0/.github/workflows/publish.yml +49 -0
- scraper_for_facebook-0.1.0/.gitignore +36 -0
- scraper_for_facebook-0.1.0/.pre-commit-config.yaml +16 -0
- scraper_for_facebook-0.1.0/CHANGELOG.md +20 -0
- scraper_for_facebook-0.1.0/CLAUDE.md +65 -0
- scraper_for_facebook-0.1.0/DISCLAIMER.md +38 -0
- scraper_for_facebook-0.1.0/LICENSE +21 -0
- scraper_for_facebook-0.1.0/PKG-INFO +164 -0
- scraper_for_facebook-0.1.0/README.md +133 -0
- scraper_for_facebook-0.1.0/pyproject.toml +59 -0
- scraper_for_facebook-0.1.0/requirements-dev.lock +46 -0
- scraper_for_facebook-0.1.0/scripts/check_fixtures_pii.py +96 -0
- scraper_for_facebook-0.1.0/scripts/check_tag_version.py +42 -0
- scraper_for_facebook-0.1.0/scripts/record_fixture.py +63 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/__init__.py +174 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/cli.py +308 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/config.py +59 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/errors.py +49 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/model.py +237 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/parse.py +333 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/profiles.py +170 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/redact.py +122 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/retrieve.py +179 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/scroll.py +244 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/session.py +165 -0
- scraper_for_facebook-0.1.0/src/scraper_for_facebook/truncation.py +70 -0
- scraper_for_facebook-0.1.0/tests/conftest.py +15 -0
- scraper_for_facebook-0.1.0/tests/fixtures/basic_status_post.ndjson +1 -0
- scraper_for_facebook-0.1.0/tests/fixtures/comment_preview_and_universal_marker.ndjson +1 -0
- scraper_for_facebook-0.1.0/tests/fixtures/media_and_links.ndjson +1 -0
- scraper_for_facebook-0.1.0/tests/fixtures/pinned_decoy_and_missing_date.ndjson +2 -0
- scraper_for_facebook-0.1.0/tests/fixtures/shared_post.ndjson +1 -0
- scraper_for_facebook-0.1.0/tests/fixtures/split_defer_merge.ndjson +2 -0
- scraper_for_facebook-0.1.0/tests/fixtures/truncated_post.ndjson +1 -0
- scraper_for_facebook-0.1.0/tests/test_identifier.py +69 -0
- scraper_for_facebook-0.1.0/tests/test_model.py +120 -0
- scraper_for_facebook-0.1.0/tests/test_parse.py +168 -0
- scraper_for_facebook-0.1.0/tests/test_redact.py +74 -0
- scraper_for_facebook-0.1.0/tests/test_retrieve.py +111 -0
- scraper_for_facebook-0.1.0/tests/test_truncation.py +69 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
lint-and-test:
|
|
13
|
+
# macOS is the first-class, tested target (v1); the ubuntu leg exists
|
|
14
|
+
# because the fixture/parse/CLI layer is browser-agnostic, and running it
|
|
15
|
+
# on Linux too is a cheap way to make that cross-platform claim real
|
|
16
|
+
# (plan §14). Neither leg installs an actual browser binary.
|
|
17
|
+
strategy:
|
|
18
|
+
fail-fast: false
|
|
19
|
+
matrix:
|
|
20
|
+
os: [macos-latest, ubuntu-latest]
|
|
21
|
+
runs-on: ${{ matrix.os }}
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: "3.12"
|
|
28
|
+
|
|
29
|
+
- name: Install pinned dev dependencies
|
|
30
|
+
run: python -m pip install -r requirements-dev.lock
|
|
31
|
+
|
|
32
|
+
- name: Install package (dependencies already pinned above)
|
|
33
|
+
run: python -m pip install -e . --no-deps
|
|
34
|
+
|
|
35
|
+
- name: Ruff lint
|
|
36
|
+
run: ruff check .
|
|
37
|
+
|
|
38
|
+
- name: Ruff format check
|
|
39
|
+
run: ruff format --check .
|
|
40
|
+
|
|
41
|
+
- name: Fixture PII/secret scan
|
|
42
|
+
run: python scripts/check_fixtures_pii.py
|
|
43
|
+
|
|
44
|
+
- name: Run tests
|
|
45
|
+
run: pytest
|
|
46
|
+
|
|
47
|
+
build-and-smoke:
|
|
48
|
+
# A wheel-install smoke test, WITH scrapling (a real dependency) but
|
|
49
|
+
# WITHOUT a browser binary — that's a separate, explicit `scrape-fb setup`
|
|
50
|
+
# step this job never runs. This catches a wrong entry-point path, a
|
|
51
|
+
# missing runtime dep, or an import that only fails once scrapling is
|
|
52
|
+
# actually present (plan §13) — none of which the fixture tests touch.
|
|
53
|
+
strategy:
|
|
54
|
+
fail-fast: false
|
|
55
|
+
matrix:
|
|
56
|
+
os: [macos-latest, ubuntu-latest]
|
|
57
|
+
runs-on: ${{ matrix.os }}
|
|
58
|
+
steps:
|
|
59
|
+
- uses: actions/checkout@v4
|
|
60
|
+
|
|
61
|
+
- uses: actions/setup-python@v5
|
|
62
|
+
with:
|
|
63
|
+
python-version: "3.12"
|
|
64
|
+
|
|
65
|
+
- name: Install build tool
|
|
66
|
+
run: python -m pip install build
|
|
67
|
+
|
|
68
|
+
- name: Build wheel
|
|
69
|
+
run: python -m build --wheel
|
|
70
|
+
|
|
71
|
+
- name: Install built wheel into a clean venv
|
|
72
|
+
# POSIX venv layout (bin/, not Scripts\) and the bash default shell
|
|
73
|
+
# both assume macos-latest/ubuntu-latest only — adding windows-latest
|
|
74
|
+
# to the matrix above would need this step rewritten, not just the
|
|
75
|
+
# matrix line (Windows unsupported in v1; see README/pyproject).
|
|
76
|
+
run: |
|
|
77
|
+
python -m venv smoke-venv
|
|
78
|
+
smoke-venv/bin/pip install dist/*.whl
|
|
79
|
+
|
|
80
|
+
- name: Smoke test the entry point
|
|
81
|
+
run: |
|
|
82
|
+
smoke-venv/bin/scrape-fb --version
|
|
83
|
+
smoke-venv/bin/scrape-fb --help
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
|
|
14
|
+
- uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.12"
|
|
17
|
+
|
|
18
|
+
- name: Verify tag matches pyproject.toml version
|
|
19
|
+
run: python3 scripts/check_tag_version.py "${{ github.ref_name }}"
|
|
20
|
+
|
|
21
|
+
- name: Install build tool
|
|
22
|
+
run: python -m pip install build
|
|
23
|
+
|
|
24
|
+
- name: Build sdist and wheel
|
|
25
|
+
run: python -m build
|
|
26
|
+
|
|
27
|
+
- uses: actions/upload-artifact@v4
|
|
28
|
+
with:
|
|
29
|
+
name: dist
|
|
30
|
+
path: dist/
|
|
31
|
+
|
|
32
|
+
publish:
|
|
33
|
+
needs: build
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
permissions:
|
|
36
|
+
# Required to mint the OIDC token for PyPI Trusted Publishing — there is
|
|
37
|
+
# no stored API token anywhere in this repo (plan §15).
|
|
38
|
+
id-token: write
|
|
39
|
+
contents: read
|
|
40
|
+
steps:
|
|
41
|
+
- uses: actions/download-artifact@v4
|
|
42
|
+
with:
|
|
43
|
+
name: dist
|
|
44
|
+
path: dist/
|
|
45
|
+
|
|
46
|
+
# Pinned to the commit for release v1.14.0 — a floating tag on an
|
|
47
|
+
# action that runs with publish credentials is a hijack vector
|
|
48
|
+
# (plan §15). Bump deliberately, not automatically.
|
|
49
|
+
- uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
|
|
9
|
+
# Dev environment
|
|
10
|
+
.venv/
|
|
11
|
+
.ruff_cache/
|
|
12
|
+
.pytest_cache/
|
|
13
|
+
|
|
14
|
+
# Raw captures and runtime outputs are a PII trap — ignore by an unambiguous,
|
|
15
|
+
# distinct name/location so a safe fixture and an unsafe raw capture can never
|
|
16
|
+
# collide under one glob (see plan §4).
|
|
17
|
+
scratch/
|
|
18
|
+
*.raw.ndjson
|
|
19
|
+
profiles/
|
|
20
|
+
*.json
|
|
21
|
+
*.ndjson
|
|
22
|
+
|
|
23
|
+
# Un-ignore the committed, synthetic-by-construction fixtures — enumerated
|
|
24
|
+
# explicitly, NOT a wildcard. A wildcard un-ignore (`!tests/fixtures/*.ndjson`)
|
|
25
|
+
# would re-include ANY new .ndjson dropped into this directory, not just
|
|
26
|
+
# these known-synthetic ones — defeating the whole point of the blanket
|
|
27
|
+
# ignore above. Adding a new fixture means adding its name here too.
|
|
28
|
+
!tests/fixtures/basic_status_post.ndjson
|
|
29
|
+
!tests/fixtures/comment_preview_and_universal_marker.ndjson
|
|
30
|
+
!tests/fixtures/media_and_links.ndjson
|
|
31
|
+
!tests/fixtures/pinned_decoy_and_missing_date.ndjson
|
|
32
|
+
!tests/fixtures/shared_post.ndjson
|
|
33
|
+
!tests/fixtures/split_defer_merge.ndjson
|
|
34
|
+
!tests/fixtures/truncated_post.ndjson
|
|
35
|
+
|
|
36
|
+
.DS_Store
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.8.6
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff
|
|
6
|
+
args: [--fix]
|
|
7
|
+
- id: ruff-format
|
|
8
|
+
|
|
9
|
+
- repo: local
|
|
10
|
+
hooks:
|
|
11
|
+
- id: fixture-pii-scan
|
|
12
|
+
name: scan committed fixtures for PII/secrets
|
|
13
|
+
entry: python3 scripts/check_fixtures_pii.py
|
|
14
|
+
language: system
|
|
15
|
+
files: ^tests/fixtures/.*\.ndjson$
|
|
16
|
+
pass_filenames: false
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2026-07-05
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- Initial release: `scrape-fb login` / `status` / `setup` / `doctor` / `fetch`.
|
|
14
|
+
- Logged-in Facebook timeline scraping via GraphQL XHR observation (no token replay).
|
|
15
|
+
- `--limit`, `--since`/`--until` retrieval with stop-reason reporting.
|
|
16
|
+
- JSON and NDJSON output formats.
|
|
17
|
+
- Python API: `FacebookScraper`, `Post`, `Media`, `LinkAttachment`.
|
|
18
|
+
|
|
19
|
+
[Unreleased]: https://github.com/tjdwls101010/Scraper-for-Facebook/compare/v0.1.0...HEAD
|
|
20
|
+
[0.1.0]: https://github.com/tjdwls101010/Scraper-for-Facebook/releases/tag/v0.1.0
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
|
|
4
|
+
|
|
5
|
+
**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
|
|
6
|
+
|
|
7
|
+
## 1. Think Before Coding
|
|
8
|
+
|
|
9
|
+
**Don't assume. Don't hide confusion. Surface tradeoffs.**
|
|
10
|
+
|
|
11
|
+
Before implementing:
|
|
12
|
+
- State your assumptions explicitly. If uncertain, ask.
|
|
13
|
+
- If multiple interpretations exist, present them - don't pick silently.
|
|
14
|
+
- If a simpler approach exists, say so. Push back when warranted.
|
|
15
|
+
- If something is unclear, stop. Name what's confusing. Ask.
|
|
16
|
+
|
|
17
|
+
## 2. Simplicity First
|
|
18
|
+
|
|
19
|
+
**Minimum code that solves the problem. Nothing speculative.**
|
|
20
|
+
|
|
21
|
+
- No features beyond what was asked.
|
|
22
|
+
- No abstractions for single-use code.
|
|
23
|
+
- No "flexibility" or "configurability" that wasn't requested.
|
|
24
|
+
- No error handling for impossible scenarios.
|
|
25
|
+
- If you write 200 lines and it could be 50, rewrite it.
|
|
26
|
+
|
|
27
|
+
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
|
|
28
|
+
|
|
29
|
+
## 3. Surgical Changes
|
|
30
|
+
|
|
31
|
+
**Touch only what you must. Clean up only your own mess.**
|
|
32
|
+
|
|
33
|
+
When editing existing code:
|
|
34
|
+
- Don't "improve" adjacent code, comments, or formatting.
|
|
35
|
+
- Don't refactor things that aren't broken.
|
|
36
|
+
- Match existing style, even if you'd do it differently.
|
|
37
|
+
- If you notice unrelated dead code, mention it - don't delete it.
|
|
38
|
+
|
|
39
|
+
When your changes create orphans:
|
|
40
|
+
- Remove imports/variables/functions that YOUR changes made unused.
|
|
41
|
+
- Don't remove pre-existing dead code unless asked.
|
|
42
|
+
|
|
43
|
+
The test: Every changed line should trace directly to the user's request.
|
|
44
|
+
|
|
45
|
+
## 4. Goal-Driven Execution
|
|
46
|
+
|
|
47
|
+
**Define success criteria. Loop until verified.**
|
|
48
|
+
|
|
49
|
+
Transform tasks into verifiable goals:
|
|
50
|
+
- "Add validation" → "Write tests for invalid inputs, then make them pass"
|
|
51
|
+
- "Fix the bug" → "Write a test that reproduces it, then make it pass"
|
|
52
|
+
- "Refactor X" → "Ensure tests pass before and after"
|
|
53
|
+
|
|
54
|
+
For multi-step tasks, state a brief plan:
|
|
55
|
+
```
|
|
56
|
+
1. [Step] → verify: [check]
|
|
57
|
+
2. [Step] → verify: [check]
|
|
58
|
+
3. [Step] → verify: [check]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Disclaimer — read before you use this
|
|
2
|
+
|
|
3
|
+
This is not legal advice. It is a plain-language summary of risks you take on by using this tool. If any of this matters to your situation, talk to a lawyer.
|
|
4
|
+
|
|
5
|
+
## 1. This violates Facebook's Terms of Service
|
|
6
|
+
|
|
7
|
+
Automating any Meta account to collect data — including logging in with a real browser and reading what it loads — is against Facebook's Terms of Service. Meta enforces this via account bans (temporary or permanent), checkpoints/"login approval" challenges, cease-and-desist letters, and in some cases litigation. Using this tool is entirely at your own risk. Use a **dedicated or throwaway account**, not your primary one, and keep volume low (§9 of the design doc; the CLI's scroll-pause floor and per-run scope exist because of this).
|
|
8
|
+
|
|
9
|
+
## 2. Publishing this tool exposes its maintainer, not just its users
|
|
10
|
+
|
|
11
|
+
This package is named `scraper-for-facebook`, published under a real GitHub identity, and distributed via PyPI Trusted Publishing, which binds each release to a named GitHub repository and account. That is a deliberate, informed choice by the maintainer — but it means the maintainer is identifiable in a way an anonymous or unpublished tool would not be. Meta has previously pursued named authors and operators of public Facebook/Instagram scrapers, historically skewed toward commercial mass-scraping and data-broker operations rather than small personal-scale tools — but the exposure exists regardless of scale, and this is recorded here so the choice stays informed.
|
|
12
|
+
|
|
13
|
+
## 3. You may become a "data controller" for other people's data
|
|
14
|
+
|
|
15
|
+
Posts you scrape belong to other people — authors, commenters, anyone tagged or mentioned. Collecting identifiable personal data about other people can make *you* a data controller under GDPR, CCPA, or similar law, with real obligations: a lawful basis for processing, honoring data-subject access/deletion requests, and limiting retention. "I did this for personal use" is not automatically a lawful basis. Minimize what you keep, and delete captured output once you're done with it. The MIT license on this code says nothing about, and does not excuse, privacy-law obligations around the *data* you collect with it.
|
|
16
|
+
|
|
17
|
+
## 4. Output files are not scrubbed — treat them as sensitive
|
|
18
|
+
|
|
19
|
+
Captured posts contain third-party names, message text, and signed media URLs. This tool:
|
|
20
|
+
- never writes output to a location you'd casually commit to git (default `--output` is outside any repo; see README),
|
|
21
|
+
- never redacts the *output* files themselves (only diagnostic/verbose logs go through redaction — see below),
|
|
22
|
+
- relies on you to delete output you no longer need.
|
|
23
|
+
|
|
24
|
+
Don't commit scraped output to a public (or even private) git repository, and don't share it beyond what you'd be comfortable being responsible for under §3.
|
|
25
|
+
|
|
26
|
+
## 5. Diagnostics are redacted — but redaction is not a certification
|
|
27
|
+
|
|
28
|
+
`-v`/`--verbose` output, error dumps, and anything printed to your terminal are passed through a single redaction path that strips signed media URLs, token-shaped fields, and truncates message text. This reduces accidental leakage into terminal scrollback, bug reports, or screenshots — it is **not** a guarantee that every sensitive value is caught, and it does not apply to the actual `--output` file, which is the full, unredacted capture by design (that's the point of the tool). `--raw --no-redact` disables this path entirely for debugging; only use it locally.
|
|
29
|
+
|
|
30
|
+
## 6. Your login profile is a live, unencrypted session credential
|
|
31
|
+
|
|
32
|
+
`scrape-fb login` persists your Facebook session (cookies, local storage) to a directory on disk, permissioned `0700`. Anyone who can read that directory has authenticated access to your Facebook account — no password or 2FA required, because the session already satisfied both. This is **less** protected than your regular browser's keychain-encrypted cookie store. Concretely:
|
|
33
|
+
- **Do not** back this directory up to Time Machine, sync it via iCloud/Dropbox, or commit it anywhere.
|
|
34
|
+
- If the machine or disk is lost or compromised, **revoke the session immediately** by logging out of that session from facebook.com (Settings → Security → Where You're Logged In), not just by deleting the local directory.
|
|
35
|
+
|
|
36
|
+
## 7. No warranty
|
|
37
|
+
|
|
38
|
+
This software is provided "as is" under the MIT License, without warranty of any kind. See [LICENSE](LICENSE). Facebook's internal API can and does change without notice; this tool may stop working, or silently return incomplete data, at any time (see the design doc's durability section).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 tjdwls101010
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scraper-for-facebook
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scrape your own logged-in Facebook timeline by observing the GraphQL responses your own browser session makes. No token replay, no credential injection.
|
|
5
|
+
Project-URL: Homepage, https://github.com/tjdwls101010/Scraper-for-Facebook
|
|
6
|
+
Project-URL: Issues, https://github.com/tjdwls101010/Scraper-for-Facebook/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/tjdwls101010/Scraper-for-Facebook/blob/main/CHANGELOG.md
|
|
8
|
+
Author: tjdwls101010
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: facebook,graphql,scraper,scraping,scrapling
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: platformdirs>=4.0
|
|
24
|
+
Requires-Dist: scrapling[fetchers]<0.5,>=0.4.9
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
27
|
+
Requires-Dist: pre-commit>=3.8; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: ruff>=0.8; extra == 'dev'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# scraper-for-facebook
|
|
33
|
+
|
|
34
|
+
Scrape posts from a **logged-in personal Facebook timeline** by observing the GraphQL responses your own browser session makes — no token replay, no credential injection. You log in once by hand; the browser generates its own `fb_dtsg`/`doc_id`/`lsd`, and this tool just reads what comes back.
|
|
35
|
+
|
|
36
|
+
> **Read [DISCLAIMER.md](DISCLAIMER.md) before using this.** Automating a Facebook account violates its Terms of Service, publishing this tool exposes its maintainer, and scraping other people's posts can make *you* a data controller over their personal data. Use a dedicated/throwaway account, not your primary one.
|
|
37
|
+
|
|
38
|
+
**This is not the first tool that does this.** [`facebook-graphql-scraper`](https://pypi.org/project/facebook-graphql-scraper/) captures GraphQL responses via Selenium + `selenium-wire` with credential-based login. This project's difference is incremental, not categorical: it reuses a **persisted browser-login profile** instead of injecting a username/password, and builds on [scrapling](https://github.com/D4Vinci/Scrapling)'s modern, actively-maintained fetch stack (Playwright-driven Chromium) instead of the largely-unmaintained `selenium-wire`.
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
This package depends on `scrapling[fetchers]`, which pins exact Playwright/patchright versions. Installing it into a shared environment alongside other Playwright-based tools can fail to resolve, or silently break one of them. **Always install this tool in an isolated environment:**
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
uv tool install scraper-for-facebook
|
|
46
|
+
# or
|
|
47
|
+
pipx install scraper-for-facebook
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Do **not** `pip install scraper-for-facebook` into a general-purpose virtualenv you share with other projects.
|
|
51
|
+
|
|
52
|
+
After installing, provision the browser (into its own isolated cache — this never touches a browser install any other tool manages):
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
scrape-fb setup
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
**Platform:** macOS is the tested, first-class target (v1). Linux likely works for the fetch/parse/CLI layer but is untested against a live Facebook session. Windows is unsupported.
|
|
59
|
+
|
|
60
|
+
## Quick start
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# 1. One-time interactive login — opens a real browser window, you log in by hand.
|
|
64
|
+
scrape-fb login
|
|
65
|
+
|
|
66
|
+
# 2. Verify the session (and that the browser + capture pipeline actually work).
|
|
67
|
+
scrape-fb doctor
|
|
68
|
+
|
|
69
|
+
# 3. Fetch the last 30 posts from a profile you're logged in and able to view.
|
|
70
|
+
scrape-fb fetch https://www.facebook.com/some.profile --limit 30
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Output defaults to a JSON file under this tool's own data directory (never your current directory or stdout — see `--output` below), because captured posts contain other people's personal data (§4 of the disclaimer) that shouldn't casually end up in a git-tracked path.
|
|
74
|
+
|
|
75
|
+
## CLI reference
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
scrape-fb --version
|
|
79
|
+
scrape-fb login [--profile NAME] [--profile-dir PATH]
|
|
80
|
+
scrape-fb status [--profile NAME] [--profile-dir PATH] [--json]
|
|
81
|
+
scrape-fb setup
|
|
82
|
+
scrape-fb doctor [--profile NAME] [--profile-dir PATH]
|
|
83
|
+
scrape-fb fetch <profile_url_or_username>
|
|
84
|
+
--profile NAME persisted login profile (default: "default")
|
|
85
|
+
--limit N max posts
|
|
86
|
+
--since YYYY-MM-DD lower date bound (inclusive), best-effort (see "Limitations" below)
|
|
87
|
+
--until YYYY-MM-DD upper date bound (inclusive)
|
|
88
|
+
--format json|ndjson default: json
|
|
89
|
+
--output PATH default: a non-repo path under this tool's data directory
|
|
90
|
+
--scroll-pause MIN,MAX seconds between scrolls; MIN is clamped to >= 0.5 (see "Guardrails")
|
|
91
|
+
--max-scrolls N scroll budget (default 40)
|
|
92
|
+
--profile-dir PATH override where the login profile is stored
|
|
93
|
+
--headed show the browser (debugging)
|
|
94
|
+
--raw include the raw captured story node per post (debug; contains PII).
|
|
95
|
+
Redacted (scrubbed) by default — combine with --no-redact for the
|
|
96
|
+
truly raw, unscrubbed node (prints an on-screen PII warning).
|
|
97
|
+
--no-redact disable redaction of --raw output (only has an effect with --raw)
|
|
98
|
+
-v / --verbose extra diagnostics (redaction-scrubbed by default)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### Exit codes
|
|
102
|
+
|
|
103
|
+
| Code | Meaning |
|
|
104
|
+
|---|---|
|
|
105
|
+
| 0 | Success — limit met, requested date window fully reached, or feed genuinely exhausted |
|
|
106
|
+
| 1 | Other/unexpected error |
|
|
107
|
+
| 2 | Login required or session expired — run `scrape-fb login` |
|
|
108
|
+
| 3 | Account checkpoint (Meta flagged the session) — log in again in a real browser |
|
|
109
|
+
| 4 | Zero posts returned — possibly parser drift against a Facebook response-shape change |
|
|
110
|
+
| 5 | Profile unavailable (memorialized, blocked, restricted, or nonexistent) |
|
|
111
|
+
| 7 | Partial: `--since` was requested but not confirmed reached within `--max-scrolls` |
|
|
112
|
+
|
|
113
|
+
A one-line summary on stderr always states the post count, observed date range, and *why* the run stopped — so a partial `--since` run is never mistaken for a complete one.
|
|
114
|
+
|
|
115
|
+
## Guardrails
|
|
116
|
+
|
|
117
|
+
- The scroll-pause floor (`--scroll-pause`) is clamped to **≥ 0.5s and cannot be set to 0** — this is the one non-bypassable limit in this tool, and it exists both to reduce your account's checkpoint/ban risk and to keep this from being usable as a mass-scraping tool.
|
|
118
|
+
- One target profile per invocation; no batch/multi-profile mode; no built-in scheduler or daemon loop.
|
|
119
|
+
- Deeper `--since` runs scroll more, and more scrolling raises checkpoint risk. If you value the account, prefer shallow/recent fetches and `--headed` runs over deep history.
|
|
120
|
+
|
|
121
|
+
## Limitations (v1)
|
|
122
|
+
|
|
123
|
+
- Facebook only — no Instagram, no Threads (see roadmap).
|
|
124
|
+
- Personal-profile timeline posts only — no groups, pages, or photo albums.
|
|
125
|
+
- `--since` is **best-effort**: because this tool observes pagination rather than driving it, Facebook can stall further pagination before your requested date is reached. Exit code `7` and the stderr summary tell you when that happened.
|
|
126
|
+
- Media is captured as URLs only (no file download) — and those URLs are signed, expire, and are scoped to your viewing session; treat them as sensitive.
|
|
127
|
+
- No guaranteed deep-history reach; no incremental `--since-last` state (yet).
|
|
128
|
+
|
|
129
|
+
## Python API
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from scraper_for_facebook import FacebookScraper, Post, Media, LinkAttachment
|
|
133
|
+
from scraper_for_facebook.errors import (
|
|
134
|
+
LoginRequiredError, SessionExpiredError, ChallengeError,
|
|
135
|
+
ProfileUnavailableError, SessionClosedError,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# One-time interactive login (opens a headed browser; you log in by hand).
|
|
139
|
+
FacebookScraper(profile="default").login()
|
|
140
|
+
|
|
141
|
+
with FacebookScraper(profile="default") as fb: # headless reuse
|
|
142
|
+
posts: list[Post] = fb.fetch_profile(
|
|
143
|
+
"https://www.facebook.com/some.profile", limit=30, since="2026-01-01",
|
|
144
|
+
)
|
|
145
|
+
for post in fb.iter_profile("https://www.facebook.com/some.profile", limit=30):
|
|
146
|
+
... # must be consumed inside the `with` block
|
|
147
|
+
|
|
148
|
+
FacebookScraper(profile="default").status() # -> Status.LOGGED_IN | EXPIRED | CHECKPOINT
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Contributing
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
uv venv && source .venv/bin/activate
|
|
155
|
+
uv pip install -e ".[dev]"
|
|
156
|
+
pre-commit install
|
|
157
|
+
pytest
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Unit tests run against synthetic, PII-free fixtures (`tests/fixtures/`) — never against real captures. Live integration tests (`tests/live/`) are opt-in (`SFB_LIVE_TESTS=1`) and never run in CI. See the design doc in this repo's history for the full architecture and the reasoning behind each guardrail.
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
MIT — see [LICENSE](LICENSE). The license covers the code; it does not cover what you do with the data you collect (see [DISCLAIMER.md](DISCLAIMER.md)).
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# scraper-for-facebook
|
|
2
|
+
|
|
3
|
+
Scrape posts from a **logged-in personal Facebook timeline** by observing the GraphQL responses your own browser session makes — no token replay, no credential injection. You log in once by hand; the browser generates its own `fb_dtsg`/`doc_id`/`lsd`, and this tool just reads what comes back.
|
|
4
|
+
|
|
5
|
+
> **Read [DISCLAIMER.md](DISCLAIMER.md) before using this.** Automating a Facebook account violates its Terms of Service, publishing this tool exposes its maintainer, and scraping other people's posts can make *you* a data controller over their personal data. Use a dedicated/throwaway account, not your primary one.
|
|
6
|
+
|
|
7
|
+
**This is not the first tool that does this.** [`facebook-graphql-scraper`](https://pypi.org/project/facebook-graphql-scraper/) captures GraphQL responses via Selenium + `selenium-wire` with credential-based login. This project's difference is incremental, not categorical: it reuses a **persisted browser-login profile** instead of injecting a username/password, and builds on [scrapling](https://github.com/D4Vinci/Scrapling)'s modern, actively-maintained fetch stack (Playwright-driven Chromium) instead of the largely-unmaintained `selenium-wire`.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
This package depends on `scrapling[fetchers]`, which pins exact Playwright/patchright versions. Installing it into a shared environment alongside other Playwright-based tools can fail to resolve, or silently break one of them. **Always install this tool in an isolated environment:**
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
uv tool install scraper-for-facebook
|
|
15
|
+
# or
|
|
16
|
+
pipx install scraper-for-facebook
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Do **not** `pip install scraper-for-facebook` into a general-purpose virtualenv you share with other projects.
|
|
20
|
+
|
|
21
|
+
After installing, provision the browser (into its own isolated cache — this never touches a browser install any other tool manages):
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
scrape-fb setup
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
**Platform:** macOS is the tested, first-class target (v1). Linux likely works for the fetch/parse/CLI layer but is untested against a live Facebook session. Windows is unsupported.
|
|
28
|
+
|
|
29
|
+
## Quick start
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# 1. One-time interactive login — opens a real browser window, you log in by hand.
|
|
33
|
+
scrape-fb login
|
|
34
|
+
|
|
35
|
+
# 2. Verify the session (and that the browser + capture pipeline actually work).
|
|
36
|
+
scrape-fb doctor
|
|
37
|
+
|
|
38
|
+
# 3. Fetch the last 30 posts from a profile you're logged in and able to view.
|
|
39
|
+
scrape-fb fetch https://www.facebook.com/some.profile --limit 30
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Output defaults to a JSON file under this tool's own data directory (never your current directory or stdout — see `--output` below), because captured posts contain other people's personal data (§4 of the disclaimer) that shouldn't casually end up in a git-tracked path.
|
|
43
|
+
|
|
44
|
+
## CLI reference
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
scrape-fb --version
|
|
48
|
+
scrape-fb login [--profile NAME] [--profile-dir PATH]
|
|
49
|
+
scrape-fb status [--profile NAME] [--profile-dir PATH] [--json]
|
|
50
|
+
scrape-fb setup
|
|
51
|
+
scrape-fb doctor [--profile NAME] [--profile-dir PATH]
|
|
52
|
+
scrape-fb fetch <profile_url_or_username>
|
|
53
|
+
--profile NAME persisted login profile (default: "default")
|
|
54
|
+
--limit N max posts
|
|
55
|
+
--since YYYY-MM-DD lower date bound (inclusive), best-effort (see "Limitations" below)
|
|
56
|
+
--until YYYY-MM-DD upper date bound (inclusive)
|
|
57
|
+
--format json|ndjson default: json
|
|
58
|
+
--output PATH default: a non-repo path under this tool's data directory
|
|
59
|
+
--scroll-pause MIN,MAX seconds between scrolls; MIN is clamped to >= 0.5 (see "Guardrails")
|
|
60
|
+
--max-scrolls N scroll budget (default 40)
|
|
61
|
+
--profile-dir PATH override where the login profile is stored
|
|
62
|
+
--headed show the browser (debugging)
|
|
63
|
+
--raw include the raw captured story node per post (debug; contains PII).
|
|
64
|
+
Redacted (scrubbed) by default — combine with --no-redact for the
|
|
65
|
+
truly raw, unscrubbed node (prints an on-screen PII warning).
|
|
66
|
+
--no-redact disable redaction of --raw output (only has an effect with --raw)
|
|
67
|
+
-v / --verbose extra diagnostics (redaction-scrubbed by default)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Exit codes
|
|
71
|
+
|
|
72
|
+
| Code | Meaning |
|
|
73
|
+
|---|---|
|
|
74
|
+
| 0 | Success — limit met, requested date window fully reached, or feed genuinely exhausted |
|
|
75
|
+
| 1 | Other/unexpected error |
|
|
76
|
+
| 2 | Login required or session expired — run `scrape-fb login` |
|
|
77
|
+
| 3 | Account checkpoint (Meta flagged the session) — log in again in a real browser |
|
|
78
|
+
| 4 | Zero posts returned — possibly parser drift against a Facebook response-shape change |
|
|
79
|
+
| 5 | Profile unavailable (memorialized, blocked, restricted, or nonexistent) |
|
|
80
|
+
| 7 | Partial: `--since` was requested but not confirmed reached within `--max-scrolls` |
|
|
81
|
+
|
|
82
|
+
A one-line summary on stderr always states the post count, observed date range, and *why* the run stopped — so a partial `--since` run is never mistaken for a complete one.
|
|
83
|
+
|
|
84
|
+
## Guardrails
|
|
85
|
+
|
|
86
|
+
- The scroll-pause floor (`--scroll-pause`) is clamped to **≥ 0.5s and cannot be set to 0** — this is the one non-bypassable limit in this tool, and it exists both to reduce your account's checkpoint/ban risk and to keep this from being usable as a mass-scraping tool.
|
|
87
|
+
- One target profile per invocation; no batch/multi-profile mode; no built-in scheduler or daemon loop.
|
|
88
|
+
- Deeper `--since` runs scroll more, and more scrolling raises checkpoint risk. If you value the account, prefer shallow/recent fetches and `--headed` runs over deep history.
|
|
89
|
+
|
|
90
|
+
## Limitations (v1)
|
|
91
|
+
|
|
92
|
+
- Facebook only — no Instagram, no Threads (see roadmap).
|
|
93
|
+
- Personal-profile timeline posts only — no groups, pages, or photo albums.
|
|
94
|
+
- `--since` is **best-effort**: because this tool observes pagination rather than driving it, Facebook can stall further pagination before your requested date is reached. Exit code `7` and the stderr summary tell you when that happened.
|
|
95
|
+
- Media is captured as URLs only (no file download) — and those URLs are signed, expire, and are scoped to your viewing session; treat them as sensitive.
|
|
96
|
+
- No guaranteed deep-history reach; no incremental `--since-last` state (yet).
|
|
97
|
+
|
|
98
|
+
## Python API
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from scraper_for_facebook import FacebookScraper, Post, Media, LinkAttachment
|
|
102
|
+
from scraper_for_facebook.errors import (
|
|
103
|
+
LoginRequiredError, SessionExpiredError, ChallengeError,
|
|
104
|
+
ProfileUnavailableError, SessionClosedError,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
# One-time interactive login (opens a headed browser; you log in by hand).
|
|
108
|
+
FacebookScraper(profile="default").login()
|
|
109
|
+
|
|
110
|
+
with FacebookScraper(profile="default") as fb: # headless reuse
|
|
111
|
+
posts: list[Post] = fb.fetch_profile(
|
|
112
|
+
"https://www.facebook.com/some.profile", limit=30, since="2026-01-01",
|
|
113
|
+
)
|
|
114
|
+
for post in fb.iter_profile("https://www.facebook.com/some.profile", limit=30):
|
|
115
|
+
... # must be consumed inside the `with` block
|
|
116
|
+
|
|
117
|
+
FacebookScraper(profile="default").status() # -> Status.LOGGED_IN | EXPIRED | CHECKPOINT
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Contributing
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
uv venv && source .venv/bin/activate
|
|
124
|
+
uv pip install -e ".[dev]"
|
|
125
|
+
pre-commit install
|
|
126
|
+
pytest
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Unit tests run against synthetic, PII-free fixtures (`tests/fixtures/`) — never against real captures. Live integration tests (`tests/live/`) are opt-in (`SFB_LIVE_TESTS=1`) and never run in CI. See the design doc in this repo's history for the full architecture and the reasoning behind each guardrail.
|
|
130
|
+
|
|
131
|
+
## License
|
|
132
|
+
|
|
133
|
+
MIT — see [LICENSE](LICENSE). The license covers the code; it does not cover what you do with the data you collect (see [DISCLAIMER.md](DISCLAIMER.md)).
|