ForageFacebook 1.0.0__py3-none-any.whl

forage/__init__.py

@@ -0,0 +1,3 @@
+"""Forage: CLI tool to scrape private Facebook groups."""
+
+__version__ = "1.0.0"

forage/auth.py

@@ -0,0 +1,134 @@
+"""Authentication and session management for Facebook."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Optional
+
+from playwright.sync_api import Browser, BrowserContext, Page, sync_playwright
+from rich.console import Console
+
+console = Console(stderr=True)
+
+DEFAULT_SESSION_DIR = Path.home() / ".config" / "forage" / "session"
+STORAGE_STATE_FILE = "storage_state.json"
+
+
+def get_session_path(session_dir: Optional[Path] = None) -> Path:
+    """Get the path to the session storage file."""
+    base_dir = session_dir or DEFAULT_SESSION_DIR
+    return base_dir / STORAGE_STATE_FILE
+
+
+def session_exists(session_dir: Optional[Path] = None) -> bool:
+    """Check if a saved session exists."""
+    return get_session_path(session_dir).exists()
+
+
+def login(
+    session_dir: Optional[Path] = None,
+    browser_type: str = "chromium",
+) -> None:
+    """
+    Open a browser for interactive Facebook login.
+
+    The user logs in manually, then presses Enter to save the session.
+    """
+    session_path = get_session_path(session_dir)
+    session_path.parent.mkdir(parents=True, exist_ok=True)
+
+    console.print("[bold]Opening browser for Facebook login...[/bold]")
+    console.print("Please log into Facebook in the browser window.")
+    console.print("Once logged in, press [bold green]Enter[/bold green] here to save your session.")
+
+    with sync_playwright() as p:
+        browser = getattr(p, browser_type).launch(headless=False)
+        context = browser.new_context()
+        page = context.new_page()
+
+        page.goto("https://www.facebook.com/login")
+
+        input()
+
+        if is_logged_in_page(page):
+            context.storage_state(path=str(session_path))
+            console.print("[bold green]Session saved successfully![/bold green]")
+        else:
+            console.print("[bold red]Login not detected. Please try again.[/bold red]")
+            raise SystemExit(3)
+
+        browser.close()
+
+
+def load_context(
+    browser: Browser,
+    session_dir: Optional[Path] = None,
+) -> BrowserContext:
+    """Load a browser context with saved session state."""
+    session_path = get_session_path(session_dir)
+
+    if session_path.exists():
+        return browser.new_context(storage_state=str(session_path))
+    else:
+        return browser.new_context()
+
+
+def is_logged_in_page(page: Page) -> bool:
+    """Check if the current page shows a logged-in state."""
+    try:
+        page.goto("https://www.facebook.com", timeout=10000, wait_until="domcontentloaded")
+        page.wait_for_timeout(2000)
+
+        logged_in_indicators = [
+            '[aria-label="Your profile"]',
+            '[aria-label="Account"]',
+            '[data-pagelet="ProfileTilesFeed"]',
+            'div[role="navigation"]',
+        ]
+
+        for selector in logged_in_indicators:
+            if page.query_selector(selector):
+                return True
+
+        login_indicators = [
+            'input[name="email"]',
+            'input[name="pass"]',
+            'button[name="login"]',
+        ]
+
+        for selector in login_indicators:
+            if page.query_selector(selector):
+                return False
+
+        return True
+
+    except Exception:
+        return False
+
+
+def is_logged_in(
+    session_dir: Optional[Path] = None,
+    browser_type: str = "chromium",
+) -> bool:
+    """Check if the saved session is still valid."""
+    if not session_exists(session_dir):
+        return False
+
+    with sync_playwright() as p:
+        browser = getattr(p, browser_type).launch(headless=True)
+        context = load_context(browser, session_dir)
+        page = context.new_page()
+
+        result = is_logged_in_page(page)
+
+        browser.close()
+        return result
+
+
+def clear_session(session_dir: Optional[Path] = None) -> None:
+    """Remove saved session data."""
+    session_path = get_session_path(session_dir)
+    if session_path.exists():
+        session_path.unlink()
+        console.print("Session cleared.")

forage/cli.py

@@ -0,0 +1,310 @@
+"""CLI interface for forage."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+import click
+from rich.console import Console
+
+from forage import __version__
+from forage.auth import (
+    DEFAULT_SESSION_DIR,
+    is_logged_in,
+    login as auth_login,
+    session_exists,
+)
+from forage.scraper import (
+    AuthenticationError,
+    GroupNotFoundError,
+    ScrapeOptions,
+    scrape_group,
+)
+
+console = Console(stderr=True)
+
+
+class Context:
+    """Shared context for CLI commands."""
+
+    def __init__(self):
+        self.verbose = False
+        self.quiet = False
+
+
+pass_context = click.make_pass_decorator(Context, ensure=True)
+
+
+@click.group()
+@click.option("-v", "--verbose", is_flag=True, help="Show progress and debug info")
+@click.option("-q", "--quiet", is_flag=True, help="Suppress non-error output")
+@click.option("--no-color", is_flag=True, help="Disable colored output")
+@click.version_option(version=__version__)
+@pass_context
+def main(ctx: Context, verbose: bool, quiet: bool, no_color: bool):
+    """Scrape posts, comments, and reactions from private Facebook groups."""
+    ctx.verbose = verbose
+    ctx.quiet = quiet
+
+    if no_color:
+        console.no_color = True
+
+
+@main.command()
+@click.option(
+    "--browser",
+    type=click.Choice(["chromium", "firefox", "webkit"]),
+    default="chromium",
+    help="Browser to use for login",
+)
+@click.option(
+    "--session-dir",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Directory to store session data",
+)
+@pass_context
+def login(ctx: Context, browser: str, session_dir: Optional[Path]):
+    """
+    Open browser for interactive Facebook login.
+
+    Opens a browser window where you can log into Facebook.
+    Once logged in, press Enter in the terminal to save your session.
+    """
+    try:
+        auth_login(session_dir=session_dir, browser_type=browser)
+    except SystemExit:
+        raise
+    except Exception as e:
+        if not ctx.quiet:
+            console.print(f"[red]Login failed: {e}[/red]")
+        raise SystemExit(1)
+
+
+@main.command()
+@click.argument("group")
+@click.option(
+    "--days",
+    type=int,
+    default=7,
+    help="Scrape posts from the last N days (default: 7)",
+)
+@click.option(
+    "--since",
+    type=str,
+    default=None,
+    help="Scrape posts since this date (ISO 8601: YYYY-MM-DD)",
+)
+@click.option(
+    "--until",
+    type=str,
+    default=None,
+    help="Scrape posts until this date (ISO 8601: YYYY-MM-DD)",
+)
+@click.option(
+    "--limit",
+    type=int,
+    default=0,
+    help="Maximum number of posts to fetch (0 = no limit)",
+)
+@click.option(
+    "--delay",
+    type=float,
+    default=2.0,
+    help="Seconds to wait between page loads (rate limiting)",
+)
+@click.option(
+    "--min-reactions",
+    type=int,
+    default=0,
+    help="Only include comments with at least N reactions",
+)
+@click.option(
+    "--top-comments",
+    type=int,
+    default=0,
+    help="Keep only the top N comments per post by reactions",
+)
+@click.option(
+    "--skip-comments",
+    is_flag=True,
+    help="Skip fetching comments entirely",
+)
+@click.option(
+    "--skip-reactions",
+    is_flag=True,
+    help="Skip fetching reaction counts",
+)
+@click.option(
+    "-o",
+    "--output",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Write output to file instead of stdout",
+)
+@click.option(
+    "-f",
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "sqlite", "csv"]),
+    default="json",
+    help="Output format (default: json)",
+)
+@click.option(
+    "--session-dir",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Directory containing session data",
+)
+@click.option(
+    "--headless/--no-headless",
+    default=True,
+    help="Run browser headlessly (use --no-headless to watch)",
+)
+@click.option(
+    "--browser",
+    type=click.Choice(["chromium", "firefox", "webkit"]),
+    default="chromium",
+    help="Browser to use",
+)
+@click.option(
+    "--no-input",
+    is_flag=True,
+    help="Disable interactive prompts",
+)
+@pass_context
+def scrape(
+    ctx: Context,
+    group: str,
+    days: int,
+    since: Optional[str],
+    until: Optional[str],
+    limit: int,
+    delay: float,
+    min_reactions: int,
+    top_comments: int,
+    skip_comments: bool,
+    skip_reactions: bool,
+    output: Optional[Path],
+    output_format: str,
+    session_dir: Optional[Path],
+    headless: bool,
+    browser: str,
+    no_input: bool,
+):
+    """
+    Scrape posts from a Facebook group.
+
+    GROUP can be a full URL, group ID, group slug, or '-' to read from stdin.
+
+    Examples:
+
+        forage scrape https://www.facebook.com/groups/mycityfoodies
+
+        forage scrape mycityfoodies --days 14
+
+        forage scrape 123456789 --since 2024-01-01 --until 2024-01-15
+
+        echo "mycityfoodies" | forage scrape -
+    """
+    # Handle stdin input
+    if group == "-":
+        if sys.stdin.isatty():
+            console.print("[red]No input provided on stdin[/red]")
+            raise SystemExit(2)
+        group = sys.stdin.read().strip()
+        if not group:
+            console.print("[red]Empty input from stdin[/red]")
+            raise SystemExit(2)
+        # Take first non-empty line if multiple lines provided
+        group = next((line.strip() for line in group.splitlines() if line.strip()), "")
+        if not group:
+            console.print("[red]No valid group identifier in stdin[/red]")
+            raise SystemExit(2)
+
+    options = ScrapeOptions(
+        days=days,
+        since=since,
+        until=until,
+        limit=limit,
+        delay=delay,
+        skip_comments=skip_comments,
+        skip_reactions=skip_reactions,
+        min_reactions=min_reactions,
+        top_comments=top_comments,
+        headless=headless,
+        verbose=ctx.verbose,
+        session_dir=session_dir,
+        browser_type=browser,
+    )
+
+    if not session_exists(session_dir):
+        if not ctx.quiet:
+            console.print("[yellow]No saved session found.[/yellow]")
+
+        if no_input or not sys.stdin.isatty():
+            console.print("[red]Please run 'forage login' first.[/red]")
+            raise SystemExit(3)
+
+        if click.confirm("Would you like to log in now?", default=True):
+            auth_login(session_dir=session_dir, browser_type=browser)
+        else:
+            raise SystemExit(3)
+
+    try:
+        result = scrape_group(group, options)
+    except AuthenticationError:
+        if not ctx.quiet:
+            console.print("[yellow]Session expired or invalid.[/yellow]")
+
+        if no_input or not sys.stdin.isatty():
+            console.print("[red]Please run 'forage login' to refresh your session.[/red]")
+            raise SystemExit(3)
+
+        if click.confirm("Session expired. Re-login?", default=True):
+            auth_login(session_dir=session_dir, browser_type=browser)
+            result = scrape_group(group, options)
+        else:
+            raise SystemExit(3)
+    except GroupNotFoundError as e:
+        console.print(f"[red]Group not found or access denied: {e}[/red]")
+        raise SystemExit(4)
+    except Exception as e:
+        console.print(f"[red]Error: {e}[/red]")
+        raise SystemExit(1)
+
+    if output_format == "sqlite":
+        from forage.exporter import export_to_sqlite
+
+        if not output:
+            console.print("[red]SQLite format requires --output file path[/red]")
+            raise SystemExit(2)
+        export_to_sqlite(result, output)
+        if not ctx.quiet:
+            console.print(f"[green]Data exported to {output}[/green]")
+    elif output_format == "csv":
+        from forage.exporter import export_to_csv
+
+        if not output:
+            console.print("[red]CSV format requires --output file path[/red]")
+            raise SystemExit(2)
+        export_to_csv(result, output)
+        if not ctx.quiet:
+            comments_path = output.with_suffix(".comments.csv")
+            console.print(f"[green]Posts exported to {output}[/green]")
+            console.print(f"[green]Comments exported to {comments_path}[/green]")
+    else:
+        json_output = result.model_dump_json(indent=2)
+        if output:
+            output.write_text(json_output)
+            if not ctx.quiet:
+                console.print(f"[green]Output written to {output}[/green]")
+        else:
+            click.echo(json_output)
+
+
+if __name__ == "__main__":
+    main()

forage/exporter.py

@@ -0,0 +1,222 @@
+"""Export functionality for scrape results."""
+
+from __future__ import annotations
+
+import csv
+import sqlite3
+from pathlib import Path
+
+from forage.models import ScrapeResult
+
+
+def export_to_csv(result: ScrapeResult, output_path: Path) -> None:
+    """Export scrape result to CSV files.
+
+    Creates two files:
+    - <output_path>: posts (one row per post)
+    - <output_path>.comments.csv: comments (one row per comment)
+    """
+    # Posts CSV
+    with open(output_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow([
+            "post_id",
+            "author_name",
+            "author_profile_url",
+            "content",
+            "timestamp",
+            "reactions_total",
+            "comments_count",
+            "group_name",
+            "group_id",
+        ])
+
+        for post in result.posts:
+            writer.writerow([
+                post.id,
+                post.author.name if post.author else "",
+                post.author.profile_url if post.author else "",
+                post.content or "",
+                post.timestamp.isoformat() if post.timestamp else "",
+                post.reactions.total if post.reactions else 0,
+                post.comments_count,
+                result.group.name,
+                result.group.id,
+            ])
+
+    # Comments CSV (separate file)
+    comments_path = output_path.with_suffix(".comments.csv")
+    with open(comments_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow([
+            "comment_id",
+            "post_id",
+            "parent_comment_id",
+            "author_name",
+            "author_profile_url",
+            "content",
+            "timestamp",
+            "reactions_total",
+        ])
+
+        def write_comment(comment, post_id, parent_id=""):
+            writer.writerow([
+                comment.id,
+                post_id,
+                parent_id,
+                comment.author.name if comment.author else "",
+                comment.author.profile_url if comment.author else "",
+                comment.content or "",
+                comment.timestamp.isoformat() if comment.timestamp else "",
+                comment.reactions.total if comment.reactions else 0,
+            ])
+            for reply in comment.replies:
+                write_comment(reply, post_id, parent_id=comment.id)
+
+        for post in result.posts:
+            for comment in post.comments:
+                write_comment(comment, post.id)
+
+
+def export_to_sqlite(result: ScrapeResult, db_path: Path) -> None:
+    """Export scrape result to SQLite database.
+
+    Creates tables for groups, posts, comments, and reactions.
+    If the database exists, appends to it (upserts based on IDs).
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+
+    # Create tables
+    cursor.executescript("""
+        CREATE TABLE IF NOT EXISTS groups (
+            id TEXT PRIMARY KEY,
+            name TEXT NOT NULL,
+            url TEXT NOT NULL
+        );
+
+        CREATE TABLE IF NOT EXISTS posts (
+            id TEXT PRIMARY KEY,
+            group_id TEXT NOT NULL,
+            author_name TEXT,
+            author_profile_url TEXT,
+            content TEXT,
+            timestamp TEXT,
+            reactions_total INTEGER DEFAULT 0,
+            reactions_like INTEGER DEFAULT 0,
+            reactions_love INTEGER DEFAULT 0,
+            reactions_haha INTEGER DEFAULT 0,
+            reactions_wow INTEGER DEFAULT 0,
+            reactions_sad INTEGER DEFAULT 0,
+            reactions_angry INTEGER DEFAULT 0,
+            comments_count INTEGER DEFAULT 0,
+            scraped_at TEXT,
+            FOREIGN KEY (group_id) REFERENCES groups(id)
+        );
+
+        CREATE TABLE IF NOT EXISTS comments (
+            id TEXT PRIMARY KEY,
+            post_id TEXT NOT NULL,
+            parent_comment_id TEXT,
+            author_name TEXT,
+            author_profile_url TEXT,
+            content TEXT,
+            timestamp TEXT,
+            reactions_total INTEGER DEFAULT 0,
+            reactions_like INTEGER DEFAULT 0,
+            reactions_love INTEGER DEFAULT 0,
+            reactions_haha INTEGER DEFAULT 0,
+            reactions_wow INTEGER DEFAULT 0,
+            reactions_sad INTEGER DEFAULT 0,
+            reactions_angry INTEGER DEFAULT 0,
+            FOREIGN KEY (post_id) REFERENCES posts(id),
+            FOREIGN KEY (parent_comment_id) REFERENCES comments(id)
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_posts_group ON posts(group_id);
+        CREATE INDEX IF NOT EXISTS idx_posts_timestamp ON posts(timestamp);
+        CREATE INDEX IF NOT EXISTS idx_comments_post ON comments(post_id);
+        CREATE INDEX IF NOT EXISTS idx_comments_parent ON comments(parent_comment_id);
+    """)
+
+    # Insert group
+    cursor.execute(
+        """
+        INSERT OR REPLACE INTO groups (id, name, url)
+        VALUES (?, ?, ?)
+        """,
+        (result.group.id, result.group.name, result.group.url),
+    )
+
+    scraped_at_str = result.scraped_at.isoformat() if result.scraped_at else None
+
+    # Insert posts and comments
+    for post in result.posts:
+        timestamp_str = post.timestamp.isoformat() if post.timestamp else None
+
+        cursor.execute(
+            """
+            INSERT OR REPLACE INTO posts (
+                id, group_id, author_name, author_profile_url, content,
+                timestamp, reactions_total, reactions_like, reactions_love,
+                reactions_haha, reactions_wow, reactions_sad, reactions_angry,
+                comments_count, scraped_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                post.id,
+                result.group.id,
+                post.author.name if post.author else None,
+                post.author.profile_url if post.author else None,
+                post.content,
+                timestamp_str,
+                post.reactions.total if post.reactions else 0,
+                post.reactions.like if post.reactions else 0,
+                post.reactions.love if post.reactions else 0,
+                post.reactions.haha if post.reactions else 0,
+                post.reactions.wow if post.reactions else 0,
+                post.reactions.sad if post.reactions else 0,
+                post.reactions.angry if post.reactions else 0,
+                post.comments_count,
+                scraped_at_str,
+            ),
+        )
+
+        # Insert comments recursively
+        def insert_comment(comment, parent_id=None):
+            comment_timestamp = comment.timestamp.isoformat() if comment.timestamp else None
+            cursor.execute(
+                """
+                INSERT OR REPLACE INTO comments (
+                    id, post_id, parent_comment_id, author_name, author_profile_url,
+                    content, timestamp, reactions_total, reactions_like, reactions_love,
+                    reactions_haha, reactions_wow, reactions_sad, reactions_angry
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    comment.id,
+                    post.id,
+                    parent_id,
+                    comment.author.name if comment.author else None,
+                    comment.author.profile_url if comment.author else None,
+                    comment.content,
+                    comment_timestamp,
+                    comment.reactions.total if comment.reactions else 0,
+                    comment.reactions.like if comment.reactions else 0,
+                    comment.reactions.love if comment.reactions else 0,
+                    comment.reactions.haha if comment.reactions else 0,
+                    comment.reactions.wow if comment.reactions else 0,
+                    comment.reactions.sad if comment.reactions else 0,
+                    comment.reactions.angry if comment.reactions else 0,
+                ),
+            )
+
+            # Insert nested replies
+            for reply in comment.replies:
+                insert_comment(reply, parent_id=comment.id)
+
+        for comment in post.comments:
+            insert_comment(comment)
+
+    conn.commit()
+    conn.close()