ForageFacebook 1.0.0__tar.gz

foragefacebook-1.0.0/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [jwmoss]

foragefacebook-1.0.0/.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+    labels:
+      - "dependencies"
+      - "ci"

foragefacebook-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+on:
+  push:
+    branches: [master, main]
+  pull_request:
+    branches: [master, main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium
+
+      - name: Run tests
+        run: uv run pytest tests/ -v --tb=short
+
+  type-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Run type checker
+        run: uv run ty check src/forage/
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install ruff
+        run: uv tool install ruff
+
+      - name: Run ruff check
+        run: ruff check src/forage/ tests/
+
+      - name: Run ruff format check
+        run: ruff format --check src/forage/ tests/

foragefacebook-1.0.0/.github/workflows/publish.yml

@@ -0,0 +1,79 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      test_pypi:
+        description: 'Publish to Test PyPI instead'
+        required: false
+        default: 'false'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Build package
+        run: uv build
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-test-pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_pypi == 'true'
+
+    environment:
+      name: test-pypi
+      url: https://test.pypi.org/project/forage/
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to Test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  publish-pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+
+    environment:
+      name: pypi
+      url: https://pypi.org/project/forage/
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

foragefacebook-1.0.0/.gitignore

@@ -0,0 +1,62 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Type checking
+.ty_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+*.json
+!package.json
+
+# Session data (contains auth cookies - NEVER commit!)
+session/
+*.session
+~/.config/forage/
+
+# Output files (may contain scraped data)
+*.db
+output/
+
+# Environment variables (may contain secrets)
+.env
+.env.local
+.env.*.local

foragefacebook-1.0.0/AGENTS.md

@@ -0,0 +1,104 @@
+# Agent Context for Forage
+
+This document provides context for AI agents working on this codebase.
+
+## Project Overview
+
+**Forage** is a Python CLI tool for scraping private Facebook groups. It uses Playwright for browser automation with saved session cookies for authentication.
+
+## Key Architecture Decisions
+
+### Browser Automation (Not API)
+Facebook doesn't provide a public API for group data. We use Playwright to automate a real browser, which:
+- Looks like legitimate user activity
+- Works with private groups
+- Handles JavaScript-heavy modern Facebook UI
+
+### Session-Based Auth
+Users log in manually once via `forage login`, which saves browser cookies to `~/.config/forage/session/`. This avoids storing credentials and handles 2FA.
+
+### Anti-Detection
+- Random delays between actions (not fixed timing)
+- Random viewport sizes from common resolutions
+- Real browser fingerprint (Playwright uses actual Chromium)
+
+## File Structure
+
+```
+src/forage/
+├── cli.py       # Entry point, Click commands
+├── auth.py      # Login flow, session persistence
+├── scraper.py   # Main scraping orchestration
+├── parser.py    # HTML parsing (posts, comments)
+└── models.py    # Pydantic models for data
+```
+
+## Common Tasks
+
+### Adding a New CLI Flag
+1. Add to `ScrapeOptions` dataclass in `scraper.py`
+2. Add Click option in `cli.py` scrape command
+3. Pass to `ScrapeOptions` when constructing
+
+### Fixing Broken Selectors
+Facebook frequently changes their HTML. When scraping breaks:
+1. Run with `--no-headless -v` to see the browser
+2. Use browser DevTools to inspect current HTML
+3. Update selectors in `parser.py` (look for `query_selector`)
+
+### Adding New Data Fields
+1. Add field to model in `models.py`
+2. Extract in appropriate `parse_*` function in `parser.py`
+3. JSON output updates automatically (Pydantic)
+
+## Known Fragile Areas
+
+1. **Post selectors** (`[data-pagelet^="FeedUnit"]`) - Facebook changes these
+2. **Timestamp parsing** - Many formats, relative times ("2h", "Yesterday")
+3. **Comment expansion** - "View more comments" button selectors change
+4. **Reaction counts** - Multiple ways reactions appear in HTML
+
+## Testing
+
+```bash
+# Quick test (3 posts, no comments)
+uv run forage -v scrape GROUP_SLUG --limit 3 --skip-comments
+
+# Debug mode (watch browser)
+uv run forage -v scrape GROUP_SLUG --limit 1 --no-headless
+
+# Type check
+uv run ty check src/
+```
+
+## Dependencies
+
+- **click**: CLI framework
+- **playwright**: Browser automation
+- **pydantic**: Data validation and JSON serialization
+- **rich**: Terminal output formatting
+
+## Anti-Detection Strategy
+
+The scraper uses several techniques to appear human:
+
+1. **Random delays**: `human_delay()` adds variance to wait times
+2. **Viewport randomization**: Picks from common screen sizes
+3. **Real browser**: Playwright runs actual Chromium, not headless-only
+4. **Session persistence**: Uses real logged-in session, not API tokens
+
+## Rate Limiting
+
+Default delay is 2 seconds between actions. For large scrapes:
+- Use `--delay 5.0` or higher
+- Consider running in batches with breaks
+- Facebook may temporarily block if too aggressive
+
+## Session Management
+
+Sessions are stored in `~/.config/forage/session/storage_state.json`. This includes:
+- Cookies
+- Local storage
+- Session storage
+
+Sessions typically expire after ~30 days or if Facebook detects unusual activity.

foragefacebook-1.0.0/CHANGELOG.md

@@ -0,0 +1,38 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [1.0.0] - 2025-01-08
+
+### Added
+
+- Initial stable release
+- `forage login` command for interactive Facebook authentication
+- `forage scrape` command for scraping posts, comments, and reactions
+- JSON output format (default)
+- SQLite export format (`-f sqlite`)
+- CSV export format (`-f csv`)
+- Date range filtering (`--days`, `--since`, `--until`)
+- Comment filtering (`--min-reactions`, `--top-comments`, `--skip-comments`)
+- Rate limiting with configurable delay (`--delay`)
+- Retry logic with exponential backoff for network errors
+- Anti-detection features (random delays, viewport rotation)
+- Stdin support for group input (`echo "group" | forage scrape -`)
+- GitHub Actions CI with tests, type checking, and linting
+- PyPI publishing workflow
+- Comprehensive test suite (96 tests)
+- Documentation (README, CONTRIBUTING, SECURITY, AGENTS, CLAUDE)
+
+### Security
+
+- Session data stored securely in `~/.config/forage/session/`
+- Sensitive files excluded via `.gitignore`
+- Security guidelines in SECURITY.md
+
+[Unreleased]: https://github.com/jwmoss/forage/compare/v1.0.0...HEAD
+[1.0.0]: https://github.com/jwmoss/forage/releases/tag/v1.0.0

foragefacebook-1.0.0/CLAUDE.md

@@ -0,0 +1,126 @@
+# Claude Context for Forage
+
+This file provides context for Claude (or other AI assistants) when working on this project.
+
+## Project Purpose
+
+Forage is a personal tool for scraping private Facebook groups to analyze community discussions. The owner uses it to:
+- Understand what's popular in their local community
+- Identify trends and common topics
+- Gather data for potential app ideas
+
+## Technology Choices
+
+| Choice | Why |
+|--------|-----|
+| Python | Fast iteration, good scraping ecosystem |
+| Playwright | Best browser automation, handles modern SPAs |
+| Click | Clean CLI framework |
+| Pydantic | Type-safe data models, automatic JSON |
+| uv | Fast Python package management |
+| ty | Astral's type checker (fast, modern) |
+
+## Development Workflow
+
+```bash
+# Always use uv, not python directly
+uv sync                           # Install deps
+uv run forage --help              # Run CLI
+uv run ty check src/              # Type check
+uv run pytest                     # Tests
+```
+
+## Code Style
+
+- Type hints on all functions
+- Docstrings for public functions
+- Use `from __future__ import annotations` for forward refs
+- Prefer explicit over implicit
+- Handle exceptions gracefully (scraping is brittle)
+
+## Key Implementation Details
+
+### The Scraping Flow
+
+1. `cli.py:scrape()` - Entry point, builds options
+2. `scraper.py:scrape_group()` - Main loop
+3. For each post in feed:
+   - `parser.py:parse_modern_post()` - Extract post data
+   - If comments enabled: `scraper.py:scrape_post_comments()` or `scrape_comments_from_post_page()`
+4. Apply filters (min_reactions, top_comments)
+5. Return `ScrapeResult` (serializes to JSON)
+
+### Authentication Flow
+
+1. `cli.py:login()` - Opens browser
+2. `auth.py:login()` - Waits for user to log in
+3. Saves `storage_state.json` (cookies + storage)
+4. Future runs load this state into browser context
+
+### HTML Parsing Strategy
+
+Facebook's React UI has deeply nested, frequently-changing HTML. Strategy:
+1. Find stable containers (`[data-pagelet^="FeedUnit"]`)
+2. Extract text content
+3. Use heuristics to identify author, content, timestamps
+4. Fall back gracefully when selectors fail
+
+## Common Issues & Solutions
+
+### "Session expired" errors
+- Session cookies expire (~30 days)
+- Run `forage login` again
+- CLI auto-prompts for re-login
+
+### Scraper finds 0 posts
+- Facebook changed their HTML structure
+- Run with `--no-headless -v` to debug
+- Check console for selector matches
+- Update selectors in `parser.py`
+
+### Rate limiting / blocked
+- Increase `--delay` (try 5-10 seconds)
+- Take breaks between large scrapes
+- May need to wait 24h if blocked
+
+### Comments not loading
+- Facebook loads comments lazily
+- The scraper tries to click "View more comments"
+- May need to navigate to post page
+
+## Future Improvements
+
+- [ ] Async scraping for performance
+- [ ] Better timestamp parsing
+- [ ] Individual reaction types (like/love/haha)
+- [ ] Media URL extraction
+- [ ] Export to CSV/SQLite
+- [ ] Retry logic with exponential backoff
+
+## Testing Tips
+
+When making changes:
+1. Test with a small limit first: `--limit 3`
+2. Use `--skip-comments` for faster iteration
+3. Use `--no-headless -v` to see what's happening
+4. Check the JSON output structure is correct
+
+## Important Files
+
+| File | Purpose |
+|------|---------|
+| `cli.py` | All CLI commands and options |
+| `scraper.py` | Scraping logic, browser control |
+| `parser.py` | HTML parsing, data extraction |
+| `models.py` | Pydantic models (Post, Comment, etc.) |
+| `auth.py` | Session management |
+
+## Fragile Code Warnings
+
+These areas break frequently due to Facebook changes:
+
+1. **`parser.py:parse_modern_post()`** - Post extraction heuristics
+2. **`scraper.py` selectors** - `[data-pagelet^="FeedUnit"]` etc.
+3. **Timestamp parsing** - Many edge cases
+
+When something breaks, check these first.

foragefacebook-1.0.0/CONTRIBUTING.md

@@ -0,0 +1,81 @@
+# Contributing to Forage
+
+Thank you for your interest in contributing to Forage!
+
+## Development Setup
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/jwmoss/forage.git
+   cd forage
+   ```
+
+2. Install dependencies using uv:
+   ```bash
+   uv sync --all-extras
+   ```
+
+3. Install Playwright browsers:
+   ```bash
+   uv run playwright install chromium
+   ```
+
+4. Run tests:
+   ```bash
+   uv run pytest tests/ -v
+   ```
+
+5. Run type checker:
+   ```bash
+   uv run ty check src/forage/
+   ```
+
+## Code Style
+
+- Use [ruff](https://docs.astral.sh/ruff/) for linting and formatting
+- Follow PEP 8 style guidelines
+- Add type hints to all function signatures
+- Write docstrings for public functions
+
+## Commit Messages
+
+We use [Conventional Commits](https://www.conventionalcommits.org/):
+
+- `feat:` new feature
+- `fix:` bug fix
+- `docs:` documentation changes
+- `test:` adding or updating tests
+- `refactor:` code refactoring
+- `chore:` maintenance tasks
+
+Example: `feat: add SQLite export option`
+
+## Pull Request Process
+
+1. Fork the repository
+2. Create a feature branch (`git checkout -b feat/my-feature`)
+3. Make your changes
+4. Ensure tests pass (`uv run pytest tests/`)
+5. Ensure type checking passes (`uv run ty check src/forage/`)
+6. Commit with a descriptive message
+7. Push and open a pull request
+
+## Testing
+
+- Write tests for new functionality
+- Use pytest fixtures for common test data
+- Mock external dependencies (Playwright, network)
+- Aim for good coverage of edge cases
+
+## Reporting Issues
+
+When reporting bugs, please include:
+- Python version (`python --version`)
+- Operating system
+- Steps to reproduce
+- Expected vs actual behavior
+- Any error messages
+
+## Questions?
+
+Feel free to open a GitHub issue for questions or discussions.