reddipy 1.0.0__py3-none-any.whl

reddipy/__init__.py

@@ -0,0 +1,4 @@
+__version__ = "1.0.0"
+
+from reddipy.sdk import Reddit
+from reddipy.models import Post, Comment, UserActivity

reddipy/__main__.py

@@ -0,0 +1,3 @@
+from reddipy.cli import main
+
+main()

reddipy/cli.py

@@ -0,0 +1,238 @@
+"""CLI entry point for RedMiner."""
+
+import argparse
+import json
+import sys
+
+from rich.console import Console
+
+from reddipy.client import RedditClient
+from reddipy.config import get_cookie, set_cookie, clear_config
+from reddipy.exceptions import (
+    NetworkError,
+    PostNotFoundError,
+    ProxyError,
+    RateLimitError,
+    RedMinerError,
+    SubredditNotFoundError,
+    UserNotFoundError,
+)
+from reddipy.exporters import ConsoleExporter, CSVExporter, JSONExporter
+from reddipy.scrapers import PostScraper, SearchScraper, SubredditScraper, UserScraper
+from reddipy.utils import generate_filename
+
+console = Console(stderr=True)
+
+
+def _add_common_flags(parser: argparse.ArgumentParser) -> None:
+    """Add flags shared by all subcommands."""
+    parser.add_argument("--limit", type=int, default=25)
+    parser.add_argument("--time", "-t", default="all",
+                        choices=["day", "week", "month", "year", "all"])
+    parser.add_argument("--export", default="console",
+                        choices=["json", "csv", "console"])
+    parser.add_argument("--output", "-o", default=None)
+    parser.add_argument("--proxy", default=None)
+    parser.add_argument("--proxy-file", default=None)
+    parser.add_argument("--cookie", default=None,
+                        help="Reddit session cookie (overrides stored config)")
+    parser.add_argument("--cookie-file", default=None,
+                        help="Path to JSON file with cookies")
+    parser.add_argument("--delay", type=float, default=2.0)
+    parser.add_argument("--user-agent", default=None)
+    parser.add_argument("--verbose", "-v", action="store_true", default=False)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the argument parser with all subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="reddipy",
+        description="RedMiner - Scrape Reddit data from the command line.",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    # configure
+    config_parser = subparsers.add_parser("configure",
+                                          help="Save Reddit session cookie for persistent use")
+    config_parser.add_argument("cookie", type=str,
+                               help="Your reddit_session cookie value")
+
+    # logout
+    subparsers.add_parser("logout", help="Remove stored authentication")
+
+    # search
+    search_parser = subparsers.add_parser("search", help="Search Reddit posts")
+    search_parser.add_argument("query", type=str)
+    search_parser.add_argument("--sort", default="relevance")
+    search_parser.add_argument("--subreddit", "-s", default=None)
+    _add_common_flags(search_parser)
+
+    # post
+    post_parser = subparsers.add_parser("post", help="Fetch a single post and its comments")
+    post_parser.add_argument("url", type=str)
+    post_parser.add_argument("--sort", default="best")
+    _add_common_flags(post_parser)
+
+    # subreddit
+    sub_parser = subparsers.add_parser("subreddit", help="Fetch posts from a subreddit")
+    sub_parser.add_argument("name", type=str)
+    sub_parser.add_argument("--sort", default="hot")
+    _add_common_flags(sub_parser)
+
+    # user
+    user_parser = subparsers.add_parser("user", help="Fetch user activity")
+    user_parser.add_argument("username", type=str)
+    user_parser.add_argument("--sort", default="new")
+    user_parser.add_argument("--type", default="all",
+                             choices=["posts", "comments", "all"])
+    _add_common_flags(user_parser)
+
+    return parser
+
+
+def _resolve_cookie(args: argparse.Namespace) -> str | None:
+    """Resolve cookie from args or stored config."""
+    if getattr(args, "cookie", None):
+        return args.cookie
+    if getattr(args, "cookie_file", None):
+        return None  # client handles cookie_file directly
+    return get_cookie()
+
+
+def handle_search(client: RedditClient, args: argparse.Namespace) -> list:
+    scraper = SearchScraper(client)
+    return scraper.scrape(
+        args.query,
+        sort=args.sort,
+        time_filter=args.time,
+        limit=args.limit,
+        subreddit=getattr(args, "subreddit", None),
+    )
+
+
+def handle_post(client: RedditClient, args: argparse.Namespace) -> tuple:
+    scraper = PostScraper(client)
+    post, comments = scraper.scrape(args.url)
+    return (post, comments)
+
+
+def handle_subreddit(client: RedditClient, args: argparse.Namespace) -> list:
+    scraper = SubredditScraper(client)
+    return scraper.scrape(
+        args.name,
+        sort=args.sort,
+        time_filter=args.time,
+        limit=args.limit,
+    )
+
+
+def handle_user(client: RedditClient, args: argparse.Namespace) -> list:
+    scraper = UserScraper(client)
+    return scraper.scrape(
+        args.username,
+        limit=args.limit,
+        sort=args.sort,
+        activity_type=args.type,
+    )
+
+
+def export_data(data, args: argparse.Namespace) -> None:
+    """Export scraped data using the selected exporter."""
+    if args.export == "json":
+        exporter = JSONExporter()
+        ext = "json"
+    elif args.export == "csv":
+        exporter = CSVExporter()
+        ext = "csv"
+    else:
+        exporter = ConsoleExporter()
+        ext = None
+
+    output_path = getattr(args, "output", None)
+    if output_path is None and ext is not None:
+        output_path = generate_filename(args.command, ext)
+
+    if isinstance(data, tuple):
+        post, comments = data
+        if args.export == "console":
+            console.print("\n[bold]Post[/bold]")
+            exporter.export([post])
+            if comments:
+                console.print(f"\n[bold]Comments[/bold] ({len(comments)})")
+                exporter.export(comments)
+        elif args.export == "json":
+            combined = {
+                "post": post.to_dict(),
+                "comments": [c.to_dict() for c in comments],
+            }
+            if output_path:
+                with open(output_path, "w", encoding="utf-8") as f:
+                    json.dump(combined, f, indent=2)
+                console.print(f"[green]Saved to {output_path}[/green]")
+            else:
+                print(json.dumps(combined, indent=2))
+        else:
+            if comments:
+                exporter.export(comments, output_path)
+            else:
+                exporter.export([post], output_path)
+            if output_path:
+                console.print(f"[green]Saved to {output_path}[/green]")
+    else:
+        exporter.export(data, output_path)
+        if output_path and ext:
+            console.print(f"[green]Saved to {output_path}[/green]")
+
+
+def main() -> None:
+    """CLI entry point."""
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if not hasattr(args, "command") or args.command is None:
+        parser.print_help()
+        sys.exit(1)
+
+    # Handle configure/logout commands
+    if args.command == "configure":
+        set_cookie(args.cookie)
+        console.print("[green]Cookie saved successfully.[/green] You can now use reddipy without --cookie.")
+        return
+
+    if args.command == "logout":
+        clear_config()
+        console.print("[yellow]Stored authentication removed.[/yellow]")
+        return
+
+    try:
+        cookie = _resolve_cookie(args)
+        client = RedditClient(
+            delay=args.delay,
+            proxy=args.proxy,
+            proxy_file=args.proxy_file,
+            user_agent=args.user_agent,
+            verbose=args.verbose,
+            cookie=cookie,
+            cookie_file=getattr(args, "cookie_file", None),
+        )
+
+        if args.command == "search":
+            data = handle_search(client, args)
+        elif args.command == "post":
+            data = handle_post(client, args)
+        elif args.command == "subreddit":
+            data = handle_subreddit(client, args)
+        elif args.command == "user":
+            data = handle_user(client, args)
+
+        export_data(data, args)
+
+    except (SubredditNotFoundError, PostNotFoundError, UserNotFoundError) as e:
+        console.print(f"[red]Error:[/red] {e}")
+        sys.exit(3)
+    except (NetworkError, RateLimitError, ProxyError) as e:
+        console.print(f"[red]Error:[/red] {e}")
+        sys.exit(2)
+    except RedMinerError as e:
+        console.print(f"[red]Error:[/red] {e}")
+        sys.exit(1)

reddipy/client.py

@@ -0,0 +1,254 @@
+"""RedditClient: HTTP layer with rate limiting, pagination, and proxy support."""
+
+import random
+import sys
+import time
+
+import requests
+
+from reddipy.exceptions import (
+    NetworkError,
+    PostNotFoundError,
+    ProxyError,
+    RateLimitError,
+    SubredditNotFoundError,
+    UserNotFoundError,
+)
+
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/149.0.0.0 Safari/537.36"
+)
+
+DEFAULT_HEADERS = {
+    "User-Agent": DEFAULT_USER_AGENT,
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "Sec-CH-UA": '"Google Chrome";v="149", "Chromium";v="149", "Not)A;Brand";v="24"',
+    "Sec-CH-UA-Mobile": "?0",
+    "Sec-CH-UA-Platform": '"Windows"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    "Connection": "keep-alive",
+}
+
+MAX_RETRIES = 3
+BACKOFF_SCHEDULE = [2, 4, 8]
+DEAD_PROXY_COOLDOWN = 60  # seconds
+
+
+class RedditClient:
+    """HTTP client for Reddit's public JSON endpoints."""
+
+    BASE_URL = "https://www.reddit.com"
+
+    def __init__(
+        self,
+        delay: float = 2.0,
+        proxy: str | None = None,
+        proxy_file: str | None = None,
+        user_agent: str | None = None,
+        verbose: bool = False,
+        cookie: str | None = None,
+        cookie_file: str | None = None,
+    ):
+        self.delay = delay
+        self.verbose = verbose
+        self._headers = dict(DEFAULT_HEADERS)
+        if user_agent:
+            self._headers["User-Agent"] = user_agent
+        self._last_request_time: float = 0.0
+        self._dead_proxies: dict[str, float] = {}
+
+        # Build proxy pool
+        self._proxies: list[str] = []
+        if proxy:
+            self._proxies = [proxy]
+        elif proxy_file:
+            self._proxies = self._load_proxy_file(proxy_file)
+
+        # Build cookie jar
+        self._cookies: dict[str, str] = {}
+        if cookie:
+            self._cookies["reddit_session"] = cookie
+        elif cookie_file:
+            self._cookies = self._load_cookie_file(cookie_file)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def fetch(self, url: str, params: dict | None = None) -> dict:
+        """Single HTTP GET with rate limiting, retries, and proxy support."""
+        self._rate_limit()
+
+        proxy_dict = self._select_proxy()
+
+        for attempt in range(MAX_RETRIES + 1):
+            try:
+                resp = requests.get(
+                    url,
+                    headers=self._headers,
+                    params=params or {},
+                    proxies=proxy_dict,
+                    cookies=self._cookies or None,
+                    timeout=30,
+                )
+            except (requests.ConnectionError, requests.Timeout) as exc:
+                # Mark the proxy dead if one was used
+                if proxy_dict:
+                    proxy_url = proxy_dict.get("http")
+                    if proxy_url:
+                        self._dead_proxies[proxy_url] = time.time()
+                raise NetworkError(str(exc)) from exc
+
+            status = resp.status_code
+
+            # 403 without cookies = auth issue, not "not found"
+            if status == 403 and not self._cookies:
+                raise NetworkError(
+                    "Reddit returned 403 Forbidden. You likely need to authenticate. "
+                    "Run: Reddit.configure(cookie='your_reddit_session') or use --cookie"
+                )
+
+            # Permanent errors (not retryable)
+            if status in (403, 404):
+                self._raise_not_found(url, status)
+
+            # Rate limited
+            if status == 429:
+                if attempt < MAX_RETRIES:
+                    wait = BACKOFF_SCHEDULE[attempt]
+                    self._log(f"Rate limited (429). Cooling down {wait}s before retry {attempt + 1}/{MAX_RETRIES}...")
+                    time.sleep(wait)
+                    continue
+                raise RateLimitError(
+                    f"Rate limited after {MAX_RETRIES} retries: {url}"
+                )
+
+            # Server errors
+            if status >= 500:
+                if attempt < MAX_RETRIES:
+                    wait = BACKOFF_SCHEDULE[attempt]
+                    self._log(f"Server error ({status}). Cooling down {wait}s before retry {attempt + 1}/{MAX_RETRIES}...")
+                    time.sleep(BACKOFF_SCHEDULE[attempt])
+                    continue
+                raise NetworkError(
+                    f"Server error {status} after {MAX_RETRIES} retries: {url}"
+                )
+
+            # Success
+            return resp.json()
+
+        # Should not be reachable, but just in case
+        raise NetworkError(f"Unexpected state after retries: {url}")  # pragma: no cover
+
+    def fetch_all(
+        self, url: str, params: dict | None = None, limit: int = 25
+    ) -> list[dict]:
+        """Paginated fetch that collects items across multiple pages."""
+        collected: list[dict] = []
+        after: str | None = None
+        params = dict(params) if params else {}
+
+        while len(collected) < limit:
+            remaining = limit - len(collected)
+            page_params = {**params, "limit": min(100, remaining)}
+            if after:
+                page_params["after"] = after
+
+            data = self.fetch(url, params=page_params)
+            children = data.get("data", {}).get("children", [])
+
+            for child in children:
+                if len(collected) >= limit:
+                    break
+                collected.append(child.get("data", {}))
+
+            after = data.get("data", {}).get("after")
+            if not after:
+                break
+
+        return collected
+
+    # ------------------------------------------------------------------
+    # Proxy management
+    # ------------------------------------------------------------------
+
+    def _select_proxy(self) -> dict | None:
+        """Pick a live proxy from the pool, or None if no proxies configured."""
+        if not self._proxies:
+            return None
+
+        now = time.time()
+        alive = [
+            p
+            for p in self._proxies
+            if p not in self._dead_proxies
+            or (now - self._dead_proxies[p]) >= DEAD_PROXY_COOLDOWN
+        ]
+
+        if not alive:
+            raise ProxyError("All proxies are dead")
+
+        chosen = random.choice(alive)
+        return {"http": chosen, "https": chosen}
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _rate_limit(self) -> None:
+        """Sleep if needed to enforce minimum delay between requests."""
+        if self._last_request_time > 0 and self.delay > 0:
+            elapsed = time.time() - self._last_request_time
+            if elapsed < self.delay:
+                time.sleep(self.delay - elapsed)
+        self._last_request_time = time.time()
+
+    def _raise_not_found(self, url: str, status: int) -> None:
+        """Raise the appropriate not-found exception based on URL pattern."""
+        if "/comments/" in url:
+            raise PostNotFoundError(f"Post not found ({status}): {url}")
+        if "/user/" in url or "/u/" in url:
+            raise UserNotFoundError(f"User not found ({status}): {url}")
+        # Default to subreddit
+        raise SubredditNotFoundError(f"Subreddit not found ({status}): {url}")
+
+    def _log(self, message: str) -> None:
+        """Print to stderr if verbose mode is on."""
+        if self.verbose:
+            print(message, file=sys.stderr)
+
+    @staticmethod
+    def _load_proxy_file(path: str) -> list[str]:
+        """Load proxy URLs from a file, one per line."""
+        with open(path, "r", encoding="utf-8") as f:
+            proxies = [line.strip() for line in f if line.strip()]
+        return proxies
+
+    @staticmethod
+    def _load_cookie_file(path: str) -> dict[str, str]:
+        """Load cookies from a JSON file.
+
+        Expects either:
+        - A JSON object: {"reddit_session": "value", ...}
+        - A JSON array (browser export): [{"name": "reddit_session", "value": "..."}, ...]
+        """
+        import json as _json
+
+        with open(path, "r", encoding="utf-8") as f:
+            data = _json.load(f)
+
+        if isinstance(data, dict):
+            return data
+        if isinstance(data, list):
+            return {item["name"]: item["value"] for item in data if "name" in item and "value" in item}
+        return {}

reddipy/config.py

@@ -0,0 +1,44 @@
+"""Persistent configuration for RedMiner (stored at ~/.reddipy/config.json)."""
+
+import json
+import os
+
+CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".reddipy")
+CONFIG_FILE = os.path.join(CONFIG_DIR, "config.json")
+
+
+def _ensure_dir() -> None:
+    os.makedirs(CONFIG_DIR, exist_ok=True)
+
+
+def load_config() -> dict:
+    """Load config from ~/.reddipy/config.json. Returns empty dict if missing."""
+    if not os.path.exists(CONFIG_FILE):
+        return {}
+    with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def save_config(config: dict) -> None:
+    """Save config to ~/.reddipy/config.json."""
+    _ensure_dir()
+    with open(CONFIG_FILE, "w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2)
+
+
+def get_cookie() -> str | None:
+    """Get stored reddit_session cookie, or None."""
+    return load_config().get("reddit_session")
+
+
+def set_cookie(cookie: str) -> None:
+    """Store reddit_session cookie persistently."""
+    config = load_config()
+    config["reddit_session"] = cookie
+    save_config(config)
+
+
+def clear_config() -> None:
+    """Remove stored config."""
+    if os.path.exists(CONFIG_FILE):
+        os.remove(CONFIG_FILE)

reddipy/exceptions.py

@@ -0,0 +1,29 @@
+"""Domain-specific exceptions for RedMiner."""
+
+
+class RedMinerError(Exception):
+    """Base exception for all RedMiner errors."""
+
+
+class SubredditNotFoundError(RedMinerError):
+    """Raised when a subreddit is private, banned, or nonexistent (403/404)."""
+
+
+class PostNotFoundError(RedMinerError):
+    """Raised when a post URL is deleted or invalid."""
+
+
+class UserNotFoundError(RedMinerError):
+    """Raised when a user is suspended, shadow-banned, or nonexistent."""
+
+
+class RateLimitError(RedMinerError):
+    """Raised when all retries are exhausted after repeated 429 responses."""
+
+
+class NetworkError(RedMinerError):
+    """Raised on connection failures or timeouts."""
+
+
+class ProxyError(RedMinerError):
+    """Raised when all proxies are exhausted or dead."""

reddipy/exporters/__init__.py

@@ -0,0 +1,5 @@
+"""Exporter modules for RedMiner."""
+
+from reddipy.exporters.json_exporter import JSONExporter
+from reddipy.exporters.csv_exporter import CSVExporter
+from reddipy.exporters.console import ConsoleExporter

reddipy/exporters/base.py

@@ -0,0 +1,9 @@
+"""Abstract base class for all exporters."""
+
+from abc import ABC, abstractmethod
+
+
+class BaseExporter(ABC):
+    @abstractmethod
+    def export(self, data: list, output_path: str | None = None) -> None:
+        pass

reddipy/exporters/console.py

@@ -0,0 +1,96 @@
+"""Console exporter: prints rich formatted tables to stdout."""
+
+from rich.console import Console
+from rich.table import Table
+
+from reddipy.exporters.base import BaseExporter
+
+
+def _truncate(text: str | None, max_len: int) -> str:
+    if text is None:
+        return ""
+    text = text.replace("\n", " ").replace("\r", "")
+    if len(text) <= max_len:
+        return text
+    return text[:max_len] + "..."
+
+
+class ConsoleExporter(BaseExporter):
+    def export(self, data: list, output_path: str | None = None) -> None:
+        if not data:
+            return
+
+        first = data[0]
+        class_name = type(first).__name__
+
+        if class_name == "Post":
+            self._print_posts(data)
+        elif class_name == "Comment":
+            self._print_comments(data)
+        elif class_name == "UserActivity":
+            self._print_user_activity(data)
+
+    def _print_posts(self, data: list) -> None:
+        table = Table(show_header=True, header_style="bold cyan", border_style="dim",
+                      pad_edge=False, show_lines=False, expand=True)
+        table.add_column("ID", style="dim", no_wrap=True, ratio=1)
+        table.add_column("Title", ratio=5)
+        table.add_column("Author", style="green", ratio=2)
+        table.add_column("Subreddit", style="magenta", ratio=2)
+        table.add_column("Score", justify="right", style="yellow", ratio=1)
+        table.add_column("Comments", justify="right", style="blue", ratio=1)
+
+        for item in data:
+            table.add_row(
+                item.id,
+                _truncate(item.title, 62),
+                item.author,
+                f"r/{item.subreddit}",
+                str(item.score),
+                str(item.num_comments),
+            )
+
+        Console().print(table)
+
+    def _print_comments(self, data: list) -> None:
+        table = Table(show_header=True, header_style="bold cyan", border_style="dim",
+                      pad_edge=False, show_lines=False, expand=True)
+        table.add_column("ID", style="dim", no_wrap=True, ratio=1)
+        table.add_column("Author", style="green", ratio=2)
+        table.add_column("Comment", ratio=5)
+        table.add_column("Score", justify="right", style="yellow", ratio=1)
+        table.add_column("Depth", justify="right", style="blue", ratio=1)
+
+        for item in data:
+            table.add_row(
+                item.id,
+                item.author,
+                _truncate(item.body, 52),
+                str(item.score),
+                str(item.depth),
+            )
+
+        Console().print(table)
+
+    def _print_user_activity(self, data: list) -> None:
+        table = Table(show_header=True, header_style="bold cyan", border_style="dim",
+                      pad_edge=False, show_lines=False, expand=True)
+        table.add_column("Type", style="bold", ratio=1)
+        table.add_column("ID", style="dim", no_wrap=True, ratio=1)
+        table.add_column("Author", style="green", ratio=2)
+        table.add_column("Subreddit", style="magenta", ratio=2)
+        table.add_column("Score", justify="right", style="yellow", ratio=1)
+        table.add_column("Content", ratio=4)
+
+        for item in data:
+            content = item.title if item.title is not None else item.body
+            table.add_row(
+                item.type,
+                item.id,
+                item.author,
+                f"r/{item.subreddit}",
+                str(item.score),
+                _truncate(content, 47),
+            )
+
+        Console().print(table)