praw-cli 0.6.0__py3-none-any.whl

prawler/crawler/post.py

@@ -0,0 +1,125 @@
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Iterator
+
+from praw.models import Submission
+
+from prawler.client import RedditPrawClient
+from prawler.model import Post
+
+
+class SubredditSort(StrEnum):
+    HOT = "hot"
+    NEW = "new"
+    TOP = "top"
+    RISING = "rising"
+    CONTROVERTIAL = "controversial"
+
+
+class SearchSort(StrEnum):
+    RELEVANCE = "relevance"
+    HOT = "hot"
+    TOP = "top"
+    NEW = "new"
+    COMMENTS = "comments"
+
+
+class TimeFilter(StrEnum):
+    HOUR = "hour"
+    DAY = "day"
+    WEEK = "week"
+    MONTH = "month"
+    YEAR = "year"
+    ALL = "all"
+
+
+@dataclass(frozen=True)
+class SubredditCrawlConfig:
+    subreddit: str
+    sort: SubredditSort = SubredditSort.HOT
+    time_filter: TimeFilter = TimeFilter.ALL
+    limit: int | None = 100
+
+
+@dataclass(frozen=True)
+class SearchCrawlConfig:
+    query: str
+    subreddit: str = "all"
+    sort: SearchSort = SearchSort.RELEVANCE
+    time_filter: TimeFilter = TimeFilter.ALL
+    limit: int | None = 100
+
+
+@dataclass(frozen=True)
+class UserCrawlConfig:
+    username: str
+    sort: SubredditSort = SubredditSort.NEW
+    limit: int | None = 100
+
+
+@dataclass(frozen=True)
+class UrlCrawlConfig:
+    url: str
+
+
+class PostCrawler:
+    def __init__(self, client: RedditPrawClient) -> None:
+        self._client = client
+
+    def from_subreddit(self, cfg: SubredditCrawlConfig) -> Iterator[Post]:
+        sub = self._client.subreddit(cfg.subreddit)
+
+        match cfg.sort:
+            case SubredditSort.HOT:
+                submissions = sub.hot(limit=cfg.limit)
+            case SubredditSort.NEW:
+                submissions = sub.new(limit=cfg.limit)
+            case SubredditSort.TOP:
+                submissions = sub.top(limit=cfg.limit, time_filter=cfg.time_filter.value)
+            case SubredditSort.RISING:
+                submissions = sub.rising(limit=cfg.limit)
+            case SubredditSort.CONTROVERTIAL:
+                submissions = sub.controversial(limit=cfg.limit, time_filter=cfg.time_filter.value)
+
+        yield from self._map(submissions)
+
+    def from_search(self, cfg: SearchCrawlConfig) -> Iterator[Post]:
+        sub = self._client.subreddit(cfg.subreddit)
+
+        results = sub.search(
+            query=cfg.query,
+            sort=cfg.sort.value,
+            time_filter=cfg.time_filter.value,
+            limit=cfg.limit,
+        )
+
+        yield from self._map(results)
+
+    def from_user(self, cfg: UserCrawlConfig) -> Iterator[Post]:
+        redditor = self._client.redditor(cfg.username)
+
+        match cfg.sort:
+            case SubredditSort.HOT:
+                submissions = redditor.hot()
+            case SubredditSort.NEW:
+                submissions = redditor.new()
+            case SubredditSort.TOP:
+                submissions = redditor.top()
+            case SubredditSort.RISING:
+                submissions = redditor.hot()
+            case SubredditSort.CONTROVERTIAL:
+                submissions = redditor.hot()
+
+        yield from self._map(item for item in submissions if isinstance(item, Submission))
+
+    def from_url(self, cfg: UrlCrawlConfig) -> Iterator[Post]:
+        submission = self._client.submission(url=cfg.url)
+
+        yield Post.from_submission(submission)
+
+    def _map(self, submissions: Iterator[Submission]) -> Iterator[Post]:
+        for sub in submissions:
+            try:
+                yield Post.from_submission(sub)
+            except Exception:  # noqa: BLE001
+                pass

prawler/crawler/redditor.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+from typing import Iterator
+
+from prawler.client import RedditPrawClient
+from prawler.model import Redditor
+
+
+@dataclass(frozen=True)
+class RedditorProfileConfig:
+    usernames: list[str]
+
+
+class RedditorCrawler:
+    def __init__(self, client: RedditPrawClient) -> None:
+        self._client = client
+
+    def from_usernames(self, cfg: RedditorProfileConfig) -> Iterator[Redditor]:
+        for username in cfg.usernames:
+            try:
+                redditor = self._client.redditor(username)
+                yield Redditor.from_praw(redditor)
+            except Exception:  # noqa: BLE001
+                pass

prawler/model/__init__.py

@@ -0,0 +1,9 @@
+from .comment import Comment
+from .post import Post
+from .redditor import Redditor
+
+__all__ = [
+    "Comment",
+    "Post",
+    "Redditor",
+]

prawler/model/comment.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+
+from praw.models import Comment as PrawComment
+
+
+@dataclass(frozen=True)
+class Comment:
+    # Identity
+    id: str
+    permalink: str
+
+    # Relationships
+    submission_id: str
+    submission_title: str
+    subreddit: str
+    author: str | None
+    parent_id: str
+
+    # Content
+    body: str
+    depth: int
+
+    # Engagement
+    score: int
+
+    # Metadata
+    created_utc: datetime
+    is_submitter: bool
+    stickied: bool
+    locked: bool
+    nsfw: bool
+    edited: bool
+
+    def to_dict(self, fields: list[str] | None = None) -> dict[str, object]:
+        data = asdict(self)
+        data["created_utc"] = self.created_utc.isoformat()
+
+        if fields:
+            data = {k: data[k] for k in fields if k in data}
+
+        return data
+
+    @classmethod
+    def from_praw(cls, comment: PrawComment) -> "Comment":
+        return cls(
+            id=comment.id,
+            permalink=comment.permalink,
+            submission_id=comment.submission.id,
+            submission_title=comment.submission.title,
+            subreddit=comment.subreddit.display_name,
+            author=comment.author.name if comment.author else None,
+            parent_id=comment.parent_id,
+            body=comment.body or "",
+            depth=comment.depth,
+            score=comment.score,
+            created_utc=datetime.fromtimestamp(comment.created_utc, tz=timezone.utc),
+            is_submitter=comment.is_submitter,
+            stickied=comment.stickied,
+            locked=comment.locked,
+            nsfw=comment.submission.over_18,
+            edited=bool(comment.edited),
+        )
+
+    @property
+    def age_days(self) -> float:
+        delta = datetime.now(tz=timezone.utc) - self.created_utc
+
+        return delta.total_seconds() / 86_400

prawler/model/post.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+
+from praw.models import Submission
+
+
+@dataclass(frozen=True)
+class Post:
+    # Identity
+    id: str
+    title: str
+    url: str
+    permalink: str
+
+    # Engagement
+    score: int
+    upvote_ratio: float
+    num_comments: int
+
+    # Metadata
+    created_utc: datetime
+    subreddit: str
+    author: str | None
+    selftext: str
+    is_self: bool
+    flair: str | None
+    nsfw: bool
+    spoiler: bool
+    locked: bool
+    stickied: bool
+    domain: str
+
+    def to_dict(self, fields: list[str] | None = None) -> dict[str, object]:
+        data = asdict(self)
+        data["created_utc"] = self.created_utc.isoformat()
+
+        if fields:
+            data = {k: data[k] for k in fields if k in data}
+
+        return data
+
+    @classmethod
+    def from_submission(cls, sub: Submission) -> "Post":
+        return cls(
+            id=sub.id,
+            title=sub.title,
+            url=sub.url,
+            permalink=sub.permalink,
+            score=sub.score,
+            upvote_ratio=sub.upvote_ratio,
+            num_comments=sub.num_comments,
+            created_utc=datetime.fromtimestamp(sub.created_utc, tz=timezone.utc),
+            subreddit=sub.subreddit.display_name,
+            author=sub.author.name if sub.author else None,
+            selftext=sub.selftext or "",
+            is_self=sub.is_self,
+            flair=sub.link_flair_text,
+            nsfw=sub.over_18,
+            spoiler=sub.spoiler,
+            locked=sub.locked,
+            stickied=sub.stickied,
+            domain=sub.domain,
+        )
+
+    @property
+    def age_days(self) -> float:
+        delta = datetime.now(tz=timezone.utc) - self.created_utc
+
+        return delta.total_seconds() / 86_400

prawler/model/redditor.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+
+from praw.models import Redditor as PrawRedditor
+
+
+@dataclass(frozen=True)
+class Redditor:
+    # Identity
+    name: str
+    id: str
+
+    # Karma
+    link_karma: int
+    comment_karma: int
+
+    # Metadata
+    created_utc: datetime
+    is_mod: bool
+    is_gold: bool
+    has_verified_email: bool | None
+    icon_img: str
+
+    def to_dict(self, fields: list[str] | None = None) -> dict[str, object]:
+        data = asdict(self)
+        data["created_utc"] = self.created_utc.isoformat()
+
+        if fields:
+            data = {k: data[k] for k in fields if k in data}
+
+        return data
+
+    @classmethod
+    def from_praw(cls, redditor: PrawRedditor) -> "Redditor":
+        return cls(
+            name=redditor.name,
+            id=redditor.id,
+            link_karma=redditor.link_karma,
+            comment_karma=redditor.comment_karma,
+            created_utc=datetime.fromtimestamp(redditor.created_utc, tz=timezone.utc),
+            is_mod=redditor.is_mod,
+            is_gold=redditor.is_gold,
+            has_verified_email=getattr(redditor, "has_verified_email", None),
+            icon_img=getattr(redditor, "icon_img", ""),
+        )
+
+    @property
+    def total_karma(self) -> int:
+        return self.link_karma + self.comment_karma
+
+    @property
+    def age_days(self) -> float:
+        delta = datetime.now(tz=timezone.utc) - self.created_utc
+
+        return delta.total_seconds() / 86_400

prawler/output/__init__.py

@@ -0,0 +1,14 @@
+from .base import Formatter, Sink
+from .registry import FORMATTERS, get_formatter
+from .sinks import FileSink, MultiSink, StdoutSink, make_sink
+
+__all__ = [
+    "FORMATTERS",
+    "FileSink",
+    "Formatter",
+    "MultiSink",
+    "Sink",
+    "StdoutSink",
+    "get_formatter",
+    "make_sink",
+]

prawler/output/base.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import Iterator, Protocol, runtime_checkable
+
+Record = dict[str, object]
+
+
+@runtime_checkable
+class Formatter(Protocol):
+    def format(self, items: Iterator[Record]) -> Iterator[str]: ...
+
+
+@runtime_checkable
+class Sink(Protocol):
+    def write(self, chunks: Iterator[str]) -> None: ...

prawler/output/formatters.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import csv
+import io
+import json
+from typing import Iterator
+
+from prawler.output.base import Record
+
+
+class JsonFormatter:
+    def format(self, items: Iterator[Record]) -> Iterator[str]:
+        yield json.dumps(list(items), ensure_ascii=False, indent=2) + "\n"
+
+
+class JsonLFormatter:
+    def format(self, items: Iterator[Record]) -> Iterator[str]:
+        for item in items:
+            yield json.dumps(item, ensure_ascii=False) + "\n"
+
+
+class CsvFormatter:
+    def format(self, items: Iterator[Record]) -> Iterator[str]:
+        rows = list(items)
+
+        if not rows:
+            return
+
+        buf = io.StringIO()
+        writer = csv.DictWriter(buf, fieldnames=list(rows[0].keys()))
+        writer.writeheader()
+        writer.writerows(rows)
+
+        yield buf.getvalue()
+
+
+class MarkdownFormatter:
+    def format(self, items: Iterator[Record]) -> Iterator[str]:
+        rows = list(items)
+
+        if not rows:
+            return
+
+        headers = list(rows[0].keys())
+        sep = " | ".join("---" for _ in headers)
+
+        yield "| " + " | ".join(headers) + " |\n"
+        yield "| " + sep + " |\n"
+
+        for row in rows:
+            cells = " | ".join(str(row.get(h, "")) for h in headers)
+            yield "| " + cells + " |\n"
+
+
+class TableFormatter:
+    def format(self, items: Iterator[Record]) -> Iterator[str]:
+        try:
+            from rich.console import Console
+            from rich.table import Table
+        except ImportError:
+            raise SystemExit("Install rich for table output: uv add rich")
+
+        rows = list(items)
+
+        if not rows:
+            return
+
+        table = Table()
+
+        for col in rows[0].keys():
+            table.add_column(col)
+
+        for row in rows:
+            table.add_row(*[str(v) for v in row.values()])
+
+        buf = io.StringIO()
+        Console(file=buf, highlight=False).print(table)
+
+        yield buf.getvalue()

prawler/output/registry.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from prawler.output.base import Formatter
+from .formatters import CsvFormatter, JsonFormatter, JsonLFormatter, MarkdownFormatter, TableFormatter
+
+FORMATTERS: dict[str, type[Formatter]] = {
+    "json": JsonFormatter,
+    "jsonl": JsonLFormatter,
+    "csv": CsvFormatter,
+    "table": TableFormatter,
+    "markdown": MarkdownFormatter,
+}
+
+
+def get_formatter(name: str) -> Formatter:
+    cls = FORMATTERS.get(name)
+
+    if cls is None:
+        valid = ", ".join(FORMATTERS)
+        raise SystemExit(f"Unknown format '{name}'. Valid options: {valid}")
+
+    return cls()

prawler/output/sinks.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Iterator
+
+from prawler.output.base import Sink
+
+
+class StdoutSink:
+    def write(self, chunks: Iterator[str]) -> None:
+        for chunk in chunks:
+            sys.stdout.write(chunk)
+
+
+class FileSink:
+    def __init__(self, path: Path) -> None:
+        self._path = path
+
+    def write(self, chunks: Iterator[str]) -> None:
+        with self._path.open("w", encoding="utf-8") as fh:
+            for chunk in chunks:
+                fh.write(chunk)
+
+
+class MultiSink:
+    def __init__(self, sinks: list[Sink]) -> None:
+        self._sinks = sinks
+
+    def write(self, chunks: Iterator[str]) -> None:
+        collected = list(chunks)
+
+        for sink in self._sinks:
+            sink.write(iter(collected))
+
+
+def make_sink(output: str) -> StdoutSink | FileSink:
+    if output == "-":
+        return StdoutSink()
+
+    return FileSink(Path(output))

prawler/pipeline/__init__.py

@@ -0,0 +1,10 @@
+from .filters import make_filter_stage
+from .stage import PipelineStage, build_pipeline
+from .transforms import make_field_select_stage
+
+__all__ = [
+    "PipelineStage",
+    "build_pipeline",
+    "make_field_select_stage",
+    "make_filter_stage",
+]

prawler/pipeline/filters.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterator
+
+from prawler.pipeline.stage import PipelineStage, Record
+
+OPERATORS = [
+    " starts_with ",
+    " ends_with ",
+    " has_all ",
+    " len>= ",
+    " len<= ",
+    " len> ",
+    " len< ",
+    " len= ",
+    " has ",
+    " in ",
+    "~=",
+    "!=",
+    ">=",
+    "<=",
+    ">",
+    "<",
+    "=",
+]
+
+Predicate = Callable[[Record], bool]
+
+
+def _parse(expr: str) -> tuple[str, str, str]:
+    for op in OPERATORS:
+        idx = expr.find(op)
+
+        if idx != -1:
+            field = expr[:idx].strip()
+            value = expr[idx + len(op) :].strip()
+
+            return field, op.strip(), value
+
+    raise ValueError(f"Invalid filter expression: {expr}")
+
+
+def _coerce(value: str) -> bool | int | float | str:
+    if value.lower() in ("true", "false"):
+        return value.lower() == "true"
+
+    try:
+        return int(value)
+    except ValueError:
+        pass
+
+    try:
+        return float(value)
+    except ValueError:
+        pass
+
+    return value
+
+
+def _make_predicate(field: str, op: str, raw_value: str) -> Predicate:
+    match op:
+        case "=":
+            target = _coerce(raw_value)
+            return lambda r: _coerce(str(r.get(field, ""))) == target
+
+        case "!=":
+            target = _coerce(raw_value)
+            return lambda r: _coerce(str(r.get(field, ""))) != target
+
+        case ">=" | "<=" | ">" | "<":
+            target = _coerce(raw_value)
+
+            ops_float = {">=": float.__ge__, "<=": float.__le__, ">": float.__gt__, "<": float.__lt__}
+            ops_str = {">=": str.__ge__, "<=": str.__le__, ">": str.__gt__, "<": str.__lt__}
+
+            def _cmp(val: object, tgt: object) -> bool:
+                if isinstance(tgt, (int, float)):
+                    try:
+                        return ops_float[op](float(str(val)), float(tgt))
+                    except (ValueError, TypeError):
+                        pass
+
+                return ops_str[op](str(val), str(tgt))
+
+            return lambda r: _cmp(r.get(field, ""), target)
+
+        case "len>=" | "len<=" | "len>" | "len<" | "len=":
+            target_len = int(raw_value)
+
+            ops_len = {"len>=": int.__ge__, "len<=": int.__le__, "len>": int.__gt__, "len<": int.__lt__, "len=": int.__eq__}
+            cmp_len = ops_len[op]
+
+            return lambda r: cmp_len(len(str(r.get(field, ""))), target_len)
+
+        case "starts_with":
+            return lambda r: str(r.get(field, "")).startswith(raw_value)
+
+        case "ends_with":
+            return lambda r: str(r.get(field, "")).endswith(raw_value)
+
+        case "~=":
+            pattern = re.compile(raw_value, re.IGNORECASE)
+
+            return lambda r: bool(pattern.search(str(r.get(field, ""))))
+
+        case "in":
+            values = {v.strip() for v in raw_value.split(",")}
+
+            return lambda r: str(r.get(field, "")) in values
+
+        case "has":
+            keywords_any = [v.strip().lower() for v in raw_value.split(",")]
+
+            return lambda r: any(k in str(r.get(field, "")).lower() for k in keywords_any)
+
+        case "has_all":
+            keywords_all = [v.strip().lower() for v in raw_value.split(",")]
+
+            return lambda r: all(k in str(r.get(field, "")).lower() for k in keywords_all)
+
+        case _:
+            raise ValueError(f"Unknown operator: {op}")
+
+
+def make_filter_stage(expr: str) -> PipelineStage:
+    field, op, value = _parse(expr)
+    predicate = _make_predicate(field, op, value)
+
+    def stage(stream: Iterator[Record]) -> Iterator[Record]:
+        return (record for record in stream if predicate(record))
+
+    return stage

prawler/pipeline/stage.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from functools import reduce
+
+Record = dict[str, object]
+PipelineStage = Callable[[Iterator[Record]], Iterator[Record]]
+
+
+def build_pipeline(*stages: PipelineStage) -> PipelineStage:
+    if not stages:
+        return lambda stream: stream
+
+    return reduce(lambda f, g: lambda stream: g(f(stream)), stages)