praw-cli 0.6.0__py3-none-any.whl

praw_cli-0.6.0.dist-info/METADATA

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: praw-cli
+Version: 0.6.0
+Summary: Composable Reddit crawler CLI with lazy pipelines, powerful filtering, reproducible datasets, and flexible output formats.
+Project-URL: Repository, https://github.com/othonhugo/praw-cli
+Project-URL: Bug-Tracker, https://github.com/othonhugo/praw-cli/issues
+Project-URL: Documentation, https://github.com/othonhugo/praw-cli#documentation
+Author: Othon H.
+Keywords: cli,crawler,data-science,dataset,nlp,praw,reddit
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Utilities
+Requires-Python: >=3.12
+Requires-Dist: praw>=8.0.2
+Requires-Dist: tenacity>=9.1.4
+Requires-Dist: typer[all]>=0.12.0
+Description-Content-Type: text/markdown
+
+# praw-cli
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-purple.svg)](https://github.com/astral-sh/ruff)
+
+A composable Reddit crawler for the command line. Fetch posts and comments from any source, shape the data through a lazy filter pipeline, and emit it in any format — all without writing a single line of Python.
+
+```bash
+# stream the top 500 posts from r/MachineLearning as JSONL
+praw-cli posts r/MachineLearning --sort top --time year --limit 500 --format jsonl
+
+# search across r/programming, keep only high-signal posts, project to four fields
+praw-cli search "New web framework" --sub r/programming \
+  --filter "score>=100" --filter "num_comments>=10" \
+  --fields id,title,score,url --format csv --output results.csv
+
+# full comment tree of a submission, depth-first, minimum score 5
+praw-cli comments https://reddit.com/r/Python/comments/xyz/ \
+  --depth 10 --min-score 5 --format jsonl
+```
+
+## Key Features & How They Solve Your Problems
+
+### Smart Memory & Reliability
+
+- **Constant Memory Footprint (Lazy Pipelines)**: Typical scrapers buffer huge arrays in memory, causing out-of-memory crashes on large crawls. praw-cli processes data lazily item-by-item (`Iterator[Record]`). Streaming 100,000 posts uses the same constant memory as streaming 10.
+- **Resumable Extractions (Checkpointing)**: Long-running scrapes often fail halfway due to network drops or API limits. praw-cli checkpoints progress, letting you `--resume` interrupted sessions without refetching from scratch.
+
+### Zero-Code Data Preparation
+
+- **Data Science-Focused DSL**: Stop writing custom Python scripts just to filter text. Chain `--filter` conditions directly in the CLI:
+  - _NLP Cleaning_: Keep long-form content using `selftext len>= 500`.
+  - _Temporal Sorting_: Restrict dates natively using ISO-8601 strings, like `created_utc >= 2024-01-01`.
+  - _Targeted Mining_: Target specific topics with keyword groupings (`has`, `has_all`) or regular expressions (`title ~= \bbot\b`).
+- **Field Projections**: Keep output datasets lightweight and clean by extracting only the schema columns you need (e.g., `--fields id,title,score,author`).
+
+### Instant Pandas & R Integrations
+
+- **Diverse Output Formats**: Stream outputs directly to `jsonl` (preferred for streaming/Pandas), standard `json`, `csv` (for R/Excel), or rich console tables and Markdown reports.
+- **Multi-Sink Pipeline**: Write the raw data to a `.jsonl` database while simultaneously writing a preview to a `.csv` summary in a single pass.
+
+### Built-in Scientific Rigor
+
+- **Crawl Manifests**: Every execution automatically generates a `.manifest.json` detailing exact parameters, versioning, records filtered, and a config fingerprint, preventing configuration drift in research environments.
+- **Deterministic Sampling**: Extract reproducible subsets of huge subreddits using systematic or randomized sampling (e.g., Bernoulli trial at `rate = 0.1` with a fixed seed).
+
+## Installation
+
+Requires Python 3.12 or later and a [Reddit API application](https://www.reddit.com/prefs/apps) (free, read-only access is sufficient for most use cases).
+
+### Via PyPI (Recommended)
+
+Install the package directly using `pip` or `pipx`:
+
+```bash
+pip install praw-cli
+```
+
+### From Source (Using `uv`)
+
+If you want to run it locally or contribute to development:
+
+```bash
+git clone https://github.com/othonhugo/praw-cli
+cd praw-cli
+uv sync
+source .venv/bin/activate
+```
+
+## Documentation
+
+Full documentation is available in the `docs/` directory.
+
+- **[Getting Started](docs/getting-started.md)**: Installation and Authentication
+- **Usage Guide**:
+  - [Commands](docs/usage/commands.md)
+  - [Pipelines & Filters](docs/usage/pipelines-filters.md)
+  - [Formatters & Outputs](docs/usage/outputs.md)
+  - [Scientific Use & Reproducibility](docs/usage/reproducibility.md)
+- **Development**:
+  - [System Architecture](docs/development/architecture.md)
+  - [Decision Log (ADR)](docs/development/decisions.md)
+  - [Contributing Guide](docs/development/contributing.md)

praw_cli-0.6.0.dist-info/RECORD

@@ -0,0 +1,34 @@
+prawler/__init__.py,sha256=owmIFA0vd2sD8dBFANuZbwABXBwvpj3O5oAvTEcFRMc,158
+prawler/__main__.py,sha256=SYfzNEfMAT8TgmMXWbCgkIRxRZighyEbNfAdYHP-mPo,66
+prawler/config.py,sha256=1JMoUAetz8fdmK6c42xZhpwWCnamr55gWQd2h65HREY,844
+prawler/cli/__init__.py,sha256=jArxVms_ZS6eug_h3jSt4pbNpU0XFAAkZ_qbrA9Xrk8,40
+prawler/cli/app.py,sha256=4IwyS30TkOg0oViMErMLzqXBa_xzKkSy5faB8Qep934,346
+prawler/cli/options.py,sha256=0txgvzy8E1HALi6XCDKV047dd6NNTQZJLCORLTeyASg,2012
+prawler/cli/commands/__init__.py,sha256=dGdARbvOMhKCBM-kLldFyRqqdWxhr9acaYry1ieXHiM,176
+prawler/cli/commands/comments.py,sha256=P-PZ-yW6aHzosGuP4prQTqsTwH05JuMoCvig0ybN1is,1605
+prawler/cli/commands/posts.py,sha256=p9_FVYJlEA5jm9rNTMbHL7jiOrqqHsRVyMU8h1rh64Y,1333
+prawler/cli/commands/search.py,sha256=IAd4bTjMA4eDCds5dQB62mOjwigGKayWIJN2FNPqE2s,1362
+prawler/cli/commands/user.py,sha256=STidxqcJInGKjpckHPzayiJmEigywkL21SxASywkLEo,1843
+prawler/client/__init__.py,sha256=5-oDVQ7RTbnCLpKScTYtbg-0_lOHEJdaORIOhBxexZw,81
+prawler/client/reddit_praw.py,sha256=3ZQyw-zm29X8XWb9T5ym-zbMMklfFnKcuxqzss-lgn4,1908
+prawler/crawler/__init__.py,sha256=4sbAbfHZJfoRz19rtCfO-XU1fnukgNrqvHy-e4iqssQ,670
+prawler/crawler/comment.py,sha256=4xEs2JNQ1DslLUd6yK2uLxhjB3dDjR5e_XLp18ZL1zg,1821
+prawler/crawler/post.py,sha256=AjAX75AWgsiQob6393nnPqpbsnmeaXWY74KCTauKfm0,3463
+prawler/crawler/redditor.py,sha256=Qru29mQVM6Ut19EI9fV5TJGF9mxXuI5LnawMMZa3Z4I,649
+prawler/model/__init__.py,sha256=fW9OOdESvnf0hQ-qqo7Ck7tUc3PtzSPb6US3dTzTSYY,141
+prawler/model/comment.py,sha256=QJXBjGJJhaRThRUoo1JztdiTlN0pjDe9iOPyPyuw0lw,1859
+prawler/model/post.py,sha256=NsQ9-XtY5ZBeawEG6bYdB5SEDAtqVCo2xPeFjnM1MPw,1798
+prawler/model/redditor.py,sha256=yqYXtCODrjlKvR1AWfG9hYe94CqdxezFM4Loy_znFFM,1525
+prawler/output/__init__.py,sha256=DvaqvqnIQQj-HUJhIzAZKWKXmrJ-YskKDzZ6BxGTUDQ,295
+prawler/output/base.py,sha256=Ghki6bGr-U0bz3iDTMNa9WpxpFHD19TgYx98Jo3DT48,336
+prawler/output/formatters.py,sha256=vJD2lP6esYL7SvVxp2xvvQ40O-twRfcu7Zya9hqT5gQ,1941
+prawler/output/registry.py,sha256=ClLBRFuJ72ih55RflBegs0LtiLN7huLZDJtDc9V2UQc,602
+prawler/output/sinks.py,sha256=t3bhOrhg4IofEU13gtohfjSCns7CK9iEPvJ7bDoAedw,938
+prawler/pipeline/__init__.py,sha256=cUW5_3e8BYXfjcNU2KfbDeBuFaDJ2rbPuuKajxE7LZM,250
+prawler/pipeline/filters.py,sha256=rIj3hjHpAI0Z4rF3Gac4X7EBN4Nh8erIKBQSYbwMa4I,3650
+prawler/pipeline/stage.py,sha256=2dHPDBqcEESBv2F8e7s0HjAaBTUR0aqzzzvZ3TG9hm4,391
+prawler/pipeline/transforms.py,sha256=M71MVAHl-EASQ2fXKm2Llz1zL4iInSEV95_n0m7zS7E,425
+praw_cli-0.6.0.dist-info/METADATA,sha256=v0GzbBymAHvDjtkeY9t65j7uWjRfk5lOye03a4jZfU0,5079
+praw_cli-0.6.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+praw_cli-0.6.0.dist-info/entry_points.txt,sha256=X9Pzmn9IzPX_iyysKUVAgt_ZHAwhrTWIfFL_ICZPfeU,45
+praw_cli-0.6.0.dist-info/RECORD,,

praw_cli-0.6.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

praw_cli-0.6.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+praw-cli = prawler.cli:app

prawler/__init__.py

@@ -0,0 +1,10 @@
+from . import cli, client, crawler, model, output, pipeline
+
+__all__ = [
+    "cli",
+    "client",
+    "crawler",
+    "model",
+    "output",
+    "pipeline",
+]

prawler/__main__.py

@@ -0,0 +1,4 @@
+from prawler.cli import app
+
+if __name__ == "__main__":
+    app()

prawler/cli/__init__.py

@@ -0,0 +1,3 @@
+from .app import app
+
+__all__ = ["app"]

prawler/cli/app.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+import typer
+
+from prawler.cli.commands import comments, posts, search, user
+
+app = typer.Typer(
+    name="prawler",
+    help="Composable Reddit crawler. Fetch posts, comments, and user data.",
+    no_args_is_help=True,
+)
+
+app.command()(posts)
+app.command()(comments)
+app.command()(search)
+app.command()(user)

prawler/cli/commands/__init__.py

@@ -0,0 +1,11 @@
+from .comments import comments
+from .posts import posts
+from .search import search
+from .user import user
+
+__all__ = [
+    "comments",
+    "posts",
+    "search",
+    "user",
+]

prawler/cli/commands/comments.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from prawler.cli.options import FieldsOption, FilterOption, FormatOption, LimitOption, OutputOption, SortOption, SourceArg
+from prawler.client import RedditPrawClient
+from prawler.config import get_config
+from prawler.crawler import CommentCrawler, SubmissionCommentConfig, UserCommentConfig
+from prawler.output import get_formatter, make_sink
+from prawler.pipeline import build_pipeline, make_field_select_stage, make_filter_stage
+
+
+def comments(
+    source: SourceArg,
+    sort: SortOption = "new",
+    limit: LimitOption = 100,
+    format: FormatOption = "jsonl",
+    output: OutputOption = "-",
+    fields: FieldsOption = None,
+    filter: FilterOption = None,
+) -> None:
+    """Crawl comments from a submission (URL or ID) or a user (u/username)."""
+
+    cfg = get_config()
+    client = RedditPrawClient.from_config(cfg)
+    crawler = CommentCrawler(client)
+
+    if source.startswith("u/"):
+        username = source.removeprefix("u/")
+        stream = crawler.from_user(UserCommentConfig(username, sort, limit))
+    elif source.startswith("http"):
+        stream = crawler.from_submission(SubmissionCommentConfig(submission_url=source, limit=limit))
+    else:
+        stream = crawler.from_submission(SubmissionCommentConfig(submission_id=source, limit=limit))
+
+    records = (comment.to_dict() for comment in stream)
+
+    pipeline = build_pipeline(
+        *[make_filter_stage(f) for f in (filter or [])],
+        make_field_select_stage(fields.split(",") if fields else None),
+    )
+
+    make_sink(output).write(get_formatter(format).format(pipeline(records)))

prawler/cli/commands/posts.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from prawler.cli.options import FieldsOption, FilterOption, FormatOption, LimitOption, OutputOption, SubredditArg, SubredditSortOption, TimeFilterOption
+from prawler.client import RedditPrawClient
+from prawler.config import get_config
+from prawler.crawler import PostCrawler, SubredditCrawlConfig, SubredditSort, TimeFilter
+from prawler.output import get_formatter, make_sink
+from prawler.pipeline import build_pipeline, make_field_select_stage, make_filter_stage
+
+
+def posts(
+    subreddit: SubredditArg,
+    sort: SubredditSortOption = SubredditSort.HOT,
+    time_filter: TimeFilterOption = TimeFilter.ALL,
+    limit: LimitOption = 100,
+    format: FormatOption = "jsonl",
+    output: OutputOption = "-",
+    fields: FieldsOption = None,
+    filter: FilterOption = None,
+) -> None:
+    """Crawl posts from a subreddit."""
+
+    cfg = get_config()
+    client = RedditPrawClient.from_config(cfg)
+    stream = PostCrawler(client).from_subreddit(SubredditCrawlConfig(subreddit, sort, time_filter, limit))
+    records = (post.to_dict() for post in stream)
+
+    pipeline = build_pipeline(
+        *[make_filter_stage(f) for f in (filter or [])],
+        make_field_select_stage(fields.split(",") if fields else None),
+    )
+
+    make_sink(output).write(get_formatter(format).format(pipeline(records)))

prawler/cli/commands/search.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from prawler.cli.options import FieldsOption, FilterOption, FormatOption, LimitOption, OutputOption, QueryArg, SearchSortOption, SubredditOption, TimeFilterOption
+from prawler.client import RedditPrawClient
+from prawler.config import get_config
+from prawler.crawler import PostCrawler, SearchCrawlConfig, SearchSort, TimeFilter
+from prawler.output import get_formatter, make_sink
+from prawler.pipeline import build_pipeline, make_field_select_stage, make_filter_stage
+
+
+def search(
+    query: QueryArg,
+    subreddit: SubredditOption = "all",
+    sort: SearchSortOption = SearchSort.RELEVANCE,
+    time_filter: TimeFilterOption = TimeFilter.ALL,
+    limit: LimitOption = 100,
+    format: FormatOption = "jsonl",
+    output: OutputOption = "-",
+    fields: FieldsOption = None,
+    filter: FilterOption = None,
+) -> None:
+    """Search Reddit posts."""
+
+    cfg = get_config()
+    client = RedditPrawClient.from_config(cfg)
+    stream = PostCrawler(client).from_search(SearchCrawlConfig(query, subreddit, sort, time_filter, limit))
+    records = (post.to_dict() for post in stream)
+
+    pipeline = build_pipeline(
+        *[make_filter_stage(f) for f in (filter or [])],
+        make_field_select_stage(fields.split(",") if fields else None),
+    )
+
+    make_sink(output).write(get_formatter(format).format(pipeline(records)))

prawler/cli/commands/user.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from prawler.cli.options import FieldsOption, FilterOption, FormatOption, LimitOption, OutputOption, SortOption, UserMode, UserModeOption, UsernameArg
+from prawler.client import RedditPrawClient
+from prawler.config import get_config
+from prawler.crawler import CommentCrawler, PostCrawler, RedditorCrawler, RedditorProfileConfig, SubredditSort, UserCommentConfig, UserCrawlConfig
+from prawler.output import get_formatter, make_sink
+from prawler.pipeline import build_pipeline, make_field_select_stage, make_filter_stage
+
+
+def user(
+    username: UsernameArg,
+    mode: UserModeOption = UserMode.POSTS,
+    sort: SortOption = "new",
+    limit: LimitOption = 100,
+    format: FormatOption = "jsonl",
+    output: OutputOption = "-",
+    fields: FieldsOption = None,
+    filter: FilterOption = None,
+) -> None:
+    """Crawl a Redditor's posts, comments, or profile."""
+
+    cfg = get_config()
+    client = RedditPrawClient.from_config(cfg)
+
+    match mode:
+        case UserMode.POSTS:
+            post_stream = PostCrawler(client).from_user(UserCrawlConfig(username, SubredditSort(sort), limit))
+            records = (item.to_dict() for item in post_stream)
+
+        case UserMode.COMMENTS:
+            comment_stream = CommentCrawler(client).from_user(UserCommentConfig(username, sort, limit))
+            records = (item.to_dict() for item in comment_stream)
+
+        case UserMode.PROFILE:
+            redditor_stream = RedditorCrawler(client).from_usernames(RedditorProfileConfig([username]))
+            records = (item.to_dict() for item in redditor_stream)
+
+    pipeline = build_pipeline(
+        *[make_filter_stage(f) for f in (filter or [])],
+        make_field_select_stage(fields.split(",") if fields else None),
+    )
+
+    make_sink(output).write(get_formatter(format).format(pipeline(records)))

prawler/cli/options.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated
+
+import typer
+
+from prawler.crawler import SearchSort, SubredditSort, TimeFilter
+
+# Arguments
+
+SubredditArg = Annotated[
+    str,
+    typer.Argument(help="Subreddit name (without r/)."),
+]
+
+QueryArg = Annotated[
+    str,
+    typer.Argument(help="Search query string."),
+]
+
+UsernameArg = Annotated[
+    str,
+    typer.Argument(help="Reddit username (without u/)."),
+]
+
+SourceArg = Annotated[
+    str,
+    typer.Argument(help="Submission URL/ID or username (prefix with u/ for user)."),
+]
+
+# Shared options
+
+FormatOption = Annotated[
+    str,
+    typer.Option("--format", "-f", help="Output format: json | jsonl | csv | table | markdown."),
+]
+
+OutputOption = Annotated[
+    str,
+    typer.Option("--output", "-o", help="Output destination. Use '-' for stdout."),
+]
+
+LimitOption = Annotated[
+    int | None,
+    typer.Option("--limit", "-n", help="Maximum number of items to fetch."),
+]
+
+FieldsOption = Annotated[
+    str | None,
+    typer.Option("--fields", help="Comma-separated list of fields to include."),
+]
+
+FilterOption = Annotated[
+    list[str] | None,
+    typer.Option("--filter", "-F", help="Filter expression (repeatable)."),
+]
+
+TimeFilterOption = Annotated[
+    TimeFilter,
+    typer.Option("--time", "-t", help="Time window (top/controversial only)."),
+]
+
+SortOption = Annotated[
+    str,
+    typer.Option("--sort", "-s", help="Sort order."),
+]
+
+# Command-specific options
+
+SubredditSortOption = Annotated[
+    SubredditSort,
+    typer.Option("--sort", "-s", help="Sort order."),
+]
+
+SearchSortOption = Annotated[
+    SearchSort,
+    typer.Option("--sort", "-s", help="Sort order."),
+]
+
+SubredditOption = Annotated[
+    str,
+    typer.Option("--sub", help="Subreddit to search within."),
+]
+
+
+class UserMode(StrEnum):
+    POSTS = "posts"
+    COMMENTS = "comments"
+    PROFILE = "profile"
+
+
+UserModeOption = Annotated[
+    UserMode,
+    typer.Option("--mode", help="What to fetch: posts | comments | profile."),
+]

prawler/client/__init__.py

@@ -0,0 +1,5 @@
+from .reddit_praw import RedditPrawClient
+
+__all__ = [
+    "RedditPrawClient",
+]

prawler/client/reddit_praw.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import praw
+import praw.models
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+from prawler.config import Config
+
+
+class RedditPrawClient:
+    def __init__(
+        self,
+        client_id: str,
+        client_secret: str,
+        user_agent: str,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
+        if username and password:
+            self._reddit = praw.Reddit(
+                client_id=client_id,
+                client_secret=client_secret,
+                user_agent=user_agent,
+                username=username,
+                password=password,
+            )
+            self._reddit.read_only = False
+        else:
+            self._reddit = praw.Reddit(
+                client_id=client_id,
+                client_secret=client_secret,
+                user_agent=user_agent,
+            )
+            self._reddit.read_only = True
+
+    def subreddit(self, name: str) -> praw.models.Subreddit:
+        return self._reddit.subreddit(name)
+
+    def redditor(self, name: str) -> praw.models.Redditor:
+        return self._reddit.redditor(name)
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=30),
+        reraise=True,
+    )
+    def submission(self, *, url: str | None = None, id: str | None = None) -> praw.models.Submission:
+        if url:
+            return self._reddit.submission(url=url)
+
+        if id:
+            return self._reddit.submission(id=id)
+
+        raise ValueError("provide url or id")
+
+    @classmethod
+    def from_config(cls, cfg: Config) -> "RedditPrawClient":
+        return cls(
+            client_id=cfg.reddit_client_id,
+            client_secret=cfg.reddit_client_secret,
+            user_agent=cfg.reddit_user_agent,
+            username=cfg.reddit_username,
+            password=cfg.reddit_password,
+        )

prawler/config.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import os
+
+
+class Config:
+    reddit_client_id: str
+    reddit_client_secret: str
+    reddit_user_agent: str
+    reddit_username: str | None
+    reddit_password: str | None
+
+    def __init__(self) -> None:
+        self.reddit_client_id = self._require("PRAWLER_CLIENT_ID")
+        self.reddit_client_secret = self._require("PRAWLER_CLIENT_SECRET")
+        self.reddit_user_agent = os.getenv("PRAWLER_USER_AGENT", "prawler/0.2")
+        self.reddit_username = os.getenv("PRAWLER_USERNAME")
+        self.reddit_password = os.getenv("PRAWLER_PASSWORD")
+
+    @staticmethod
+    def _require(key: str) -> str:
+        value = os.getenv(key)
+
+        if not value:
+            raise SystemExit(f"Missing required environment variable: {key}")
+
+        return value
+
+
+def get_config() -> Config:
+    return Config()

prawler/crawler/__init__.py

@@ -0,0 +1,35 @@
+from .comment import (
+    CommentCrawler,
+    SubmissionCommentConfig,
+    UserCommentConfig,
+)
+from .post import (
+    PostCrawler,
+    SearchCrawlConfig,
+    SearchSort,
+    SubredditCrawlConfig,
+    SubredditSort,
+    TimeFilter,
+    UrlCrawlConfig,
+    UserCrawlConfig,
+)
+from .redditor import (
+    RedditorCrawler,
+    RedditorProfileConfig,
+)
+
+__all__ = [
+    "CommentCrawler",
+    "PostCrawler",
+    "RedditorCrawler",
+    "RedditorProfileConfig",
+    "SearchCrawlConfig",
+    "SearchSort",
+    "SubmissionCommentConfig",
+    "SubredditCrawlConfig",
+    "SubredditSort",
+    "TimeFilter",
+    "UrlCrawlConfig",
+    "UserCommentConfig",
+    "UserCrawlConfig",
+]

prawler/crawler/comment.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass
+from typing import Iterable, Iterator
+
+import praw.models
+
+from prawler.client import RedditPrawClient
+from prawler.model import Comment
+
+
+@dataclass(frozen=True)
+class SubmissionCommentConfig:
+    submission_id: str | None = None
+    submission_url: str | None = None
+    limit: int | None = None
+
+
+@dataclass(frozen=True)
+class UserCommentConfig:
+    username: str
+    sort: str = "new"
+    limit: int | None = 100
+
+
+class CommentCrawler:
+    def __init__(self, client: RedditPrawClient) -> None:
+        self._client = client
+
+    def from_submission(self, cfg: SubmissionCommentConfig) -> Iterator[Comment]:
+        submission = self._client.submission(
+            url=cfg.submission_url,
+            id=cfg.submission_id,
+        )
+
+        submission.comments.replace_more(limit=cfg.limit)
+
+        yield from self._map(item for item in submission.comments.list() if isinstance(item, praw.models.Comment))
+
+    def from_user(self, cfg: UserCommentConfig) -> Iterator[Comment]:
+        redditor = self._client.redditor(cfg.username)
+
+        match cfg.sort:
+            case "hot":
+                comments = redditor.comments.hot(limit=cfg.limit)
+            case "top":
+                comments = redditor.comments.top(limit=cfg.limit)
+            case "controversial":
+                comments = redditor.comments.controversial(limit=cfg.limit)
+            case _:
+                comments = redditor.comments.new(limit=cfg.limit)
+
+        yield from self._map(item for item in comments if isinstance(item, praw.models.Comment))
+
+    def _map(self, comments: Iterable[praw.models.Comment]) -> Iterator[Comment]:
+        for comment in comments:
+            try:
+                yield Comment.from_praw(comment)
+            except Exception:  # noqa: BLE001
+                pass