grokfeed 0.2.0__py3-none-any.whl

grokfeed/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

grokfeed/app.py

@@ -0,0 +1,290 @@
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+from textual.app import App, ComposeResult
+from textual.binding import Binding
+from textual.containers import Container
+from textual.widgets import Footer, Header, Label, LoadingIndicator
+
+from .config import Config, load_cache, save_cache
+from .sources.hn import fetch_hn_stories_by_ids, fetch_hn_top_ids
+from .sources.lobsters import fetch_lobsters_posts
+from .sources.reddit import fetch_reddit_posts
+from .widgets.feed import FeedList
+from .widgets.post_split_modal import PostSplitModal
+from .widgets.story import source_color as get_source_color
+
+# Source filter sentinel
+ALL = "all"
+
+
+def _interleave_by_score(items: list[dict]) -> list[dict]:
+    """Sort items by score normalized within each source (0–1 scale)."""
+    from collections import defaultdict
+
+    groups: dict[str, list[dict]] = defaultdict(list)
+    for item in items:
+        groups[item["source"]].append(item)
+    for source_items in groups.values():
+        scores = [i["score"] for i in source_items]
+        lo, hi = min(scores), max(scores)
+        span = hi - lo or 1
+        for item in source_items:
+            item["_norm_score"] = (item["score"] - lo) / span
+    return sorted(items, key=lambda i: i.get("_norm_score", 0), reverse=True)
+
+
+class GrokFeedApp(App):
+    """Hacker News + Reddit terminal feed."""
+
+    CSS = """
+    Screen {
+        background: $surface;
+        overflow-y: hidden;
+    }
+    #status-bar {
+        height: 1;
+        background: $primary-darken-3;
+        color: $text-muted;
+        padding: 0 1;
+        dock: top;
+        margin-top: 3;
+    }
+    #loading {
+        align: center middle;
+    }
+    LoadingIndicator {
+        color: #ff6600;
+    }
+    """
+
+    BINDINGS = [
+        Binding("j", "cursor_down", "Down"),
+        Binding("k", "cursor_up", "Up"),
+        Binding("down", "cursor_down", "Down", show=False),
+        Binding("up", "cursor_up", "Up", show=False),
+        Binding("enter", "open_or_body", "Open", priority=True),
+        Binding("f", "cycle_source", "Filter"),
+        Binding("r", "refresh", "Refresh"),
+        Binding("m", "load_more", "More"),
+        Binding("q", "quit", "Quit"),
+    ]
+
+    TITLE = "grokfeed"
+    SUB_TITLE = "HN + Reddit + lobste.rs terminal reader"
+
+    def __init__(self, config: Config) -> None:
+        super().__init__()
+        self.config = config
+        self._all_items: list[dict] = []
+        self._source_filter: str = ALL
+        self._sources: list[str] = []
+        self._loading = False
+        self._hn_ids: list[int] = []
+        self._hn_offset: int = 0
+        self._reddit_after: dict[str, str] = {}
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        yield Label("", id="status-bar")
+        yield Container(LoadingIndicator(), id="loading")
+        yield FeedList(id="feed")
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self.query_one("#feed").display = False
+        self.run_worker(self._load_all(), exclusive=True, name="fetch")
+
+    async def _load_all(self) -> None:
+        loading = self.query_one("#loading")
+        feed = self.query_one(FeedList)
+        loading.display = True
+        feed.display = False
+
+        # Serve from cache if fresh enough
+        cached = load_cache(self.config.cache_ttl_minutes)
+        if cached:
+            self._all_items = cached
+            self._sources = [ALL] + list(dict.fromkeys(i["source"] for i in cached))
+            self._apply_filter(from_cache=True)
+            loading.display = False
+            feed.display = True
+            return
+
+        self._set_status("Fetching stories…")
+        try:
+            async with httpx.AsyncClient() as client:
+                self._hn_ids = await fetch_hn_top_ids(client)
+                hn_ids = self._hn_ids[: self.config.hn_story_count]
+                self._hn_offset = len(hn_ids)
+
+                hn_task = asyncio.create_task(fetch_hn_stories_by_ids(hn_ids, client))
+                reddit_task = asyncio.create_task(
+                    fetch_reddit_posts(
+                        self.config.subreddits, self.config.reddit_post_count, client
+                    )
+                )
+                lobsters_task = asyncio.create_task(
+                    fetch_lobsters_posts(self.config.lobsters_post_count, client)
+                )
+                (
+                    hn_stories,
+                    (reddit_posts, self._reddit_after),
+                    lobsters_posts,
+                ) = await asyncio.gather(hn_task, reddit_task, lobsters_task)
+        except Exception as e:
+            self._set_status(f"Error: {e}")
+            loading.display = False
+            return
+
+        items: list[dict] = []
+        for s in hn_stories:
+            items.append(
+                {
+                    "title": s.title,
+                    "source": "HN",
+                    "score": s.score,
+                    "comments": s.comments,
+                    "url": s.url,
+                    "body": s.body,
+                    "post_id": str(s.id),
+                }
+            )
+        for p in reddit_posts:
+            items.append(
+                {
+                    "title": p.title,
+                    "source": p.source,
+                    "score": p.score,
+                    "comments": p.comments,
+                    "url": p.url,
+                    "body": p.body,
+                    "post_id": p.id,
+                    "subreddit": p.subreddit,
+                }
+            )
+        for lp in lobsters_posts:
+            items.append(
+                {
+                    "title": lp.title,
+                    "source": lp.source,
+                    "score": lp.score,
+                    "comments": lp.comments,
+                    "url": lp.url,
+                    "body": lp.body,
+                    "post_id": lp.id,
+                }
+            )
+
+        self._all_items = items
+        self._sources = [ALL] + list(dict.fromkeys(i["source"] for i in items))
+        self._apply_filter()
+        save_cache(items)
+
+        loading.display = False
+        feed.display = True
+
+    def _apply_filter(self, from_cache: bool = False) -> None:
+        feed = self.query_one(FeedList)
+        if self._source_filter == ALL:
+            visible = _interleave_by_score(self._all_items)
+            label = "all sources"
+        else:
+            visible = [i for i in self._all_items if i["source"] == self._source_filter]
+            label = self._source_filter
+        feed.load_items(visible)
+        suffix = " (cached)" if from_cache else ""
+        self._set_status(f"{len(visible)} stories — {label}{suffix}")
+
+    def _set_status(self, msg: str) -> None:
+        self.query_one("#status-bar", Label).update(msg)
+
+    def action_cursor_down(self) -> None:
+        self.query_one(FeedList).action_cursor_down()
+
+    def action_cursor_up(self) -> None:
+        self.query_one(FeedList).action_cursor_up()
+
+    def action_open_or_body(self) -> None:
+        feed = self.query_one(FeedList)
+        item = feed.current_item()
+        if not item:
+            return
+        color = get_source_color(item["source"], 0)
+        self.push_screen(PostSplitModal(item, color))
+
+    def action_load_more(self) -> None:
+        self.run_worker(self._fetch_more(), exclusive=True, name="fetch-more")
+
+    async def _fetch_more(self) -> None:
+        self._set_status("Loading more…")
+        try:
+            async with httpx.AsyncClient() as client:
+                hn_ids = self._hn_ids[
+                    self._hn_offset : self._hn_offset + self.config.hn_story_count
+                ]
+                hn_task = asyncio.create_task(fetch_hn_stories_by_ids(hn_ids, client))
+                reddit_task = asyncio.create_task(
+                    fetch_reddit_posts(
+                        self.config.subreddits,
+                        self.config.reddit_post_count,
+                        client,
+                        after=self._reddit_after,
+                    )
+                )
+                hn_stories, (reddit_posts, new_after) = await asyncio.gather(hn_task, reddit_task)
+        except Exception as e:
+            self._set_status(f"Error: {e}")
+            return
+
+        self._hn_offset += len(hn_ids)
+        self._reddit_after = new_after
+
+        existing_ids = {i["post_id"] for i in self._all_items}
+        new_items: list[dict] = []
+        for s in hn_stories:
+            if str(s.id) not in existing_ids:
+                new_items.append(
+                    {
+                        "title": s.title,
+                        "source": "HN",
+                        "score": s.score,
+                        "comments": s.comments,
+                        "url": s.url,
+                        "body": s.body,
+                        "post_id": str(s.id),
+                    }
+                )
+        for p in reddit_posts:
+            if p.id not in existing_ids:
+                new_items.append(
+                    {
+                        "title": p.title,
+                        "source": p.source,
+                        "score": p.score,
+                        "comments": p.comments,
+                        "url": p.url,
+                        "body": p.body,
+                        "post_id": p.id,
+                        "subreddit": p.subreddit,
+                    }
+                )
+
+        self._all_items.extend(new_items)
+        self._sources = [ALL] + list(dict.fromkeys(i["source"] for i in self._all_items))
+        self._apply_filter()
+
+    def action_refresh(self) -> None:
+        self.run_worker(self._load_all(), exclusive=True, name="fetch")
+
+    def action_cycle_source(self) -> None:
+        if not self._sources:
+            return
+        try:
+            idx = self._sources.index(self._source_filter)
+        except ValueError:
+            idx = 0
+        self._source_filter = self._sources[(idx + 1) % len(self._sources)]
+        self._apply_filter()

grokfeed/config.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import json
+import time
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore[no-redef]
+from dataclasses import dataclass, field
+from pathlib import Path
+
+CONFIG_DIR = Path.home() / ".grokfeed"
+CONFIG_PATH = CONFIG_DIR / "config.toml"
+CACHE_PATH = CONFIG_DIR / "cache.json"
+
+DEFAULT_CONFIG = """\
+subreddits = ["programming", "ClaudeAI", "machinelearning"]
+hn_story_count = 30
+reddit_post_count = 15
+lobsters_post_count = 25
+cache_ttl_minutes = 10
+"""
+
+
+@dataclass
+class Config:
+    subreddits: list[str] = field(
+        default_factory=lambda: ["programming", "python", "machinelearning"]
+    )
+    hn_story_count: int = 30
+    reddit_post_count: int = 15
+    lobsters_post_count: int = 25
+    cache_ttl_minutes: int = 10
+
+
+def load_config() -> tuple[Config, bool]:
+    """Return (Config, created_fresh). Creates default config on first run."""
+    created = False
+    if not CONFIG_PATH.exists():
+        CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+        CONFIG_PATH.write_text(DEFAULT_CONFIG)
+        created = True
+
+    raw = tomllib.loads(CONFIG_PATH.read_text())
+    return Config(
+        subreddits=raw.get("subreddits", ["programming", "python", "machinelearning"]),
+        hn_story_count=int(raw.get("hn_story_count", 30)),
+        reddit_post_count=int(raw.get("reddit_post_count", 15)),
+        lobsters_post_count=int(raw.get("lobsters_post_count", 25)),
+        cache_ttl_minutes=int(raw.get("cache_ttl_minutes", 10)),
+    ), created
+
+
+def load_cache(ttl_minutes: int) -> list[dict] | None:
+    if not CACHE_PATH.exists():
+        return None
+    try:
+        data = json.loads(CACHE_PATH.read_bytes())
+        if time.time() - data["ts"] > ttl_minutes * 60:
+            return None
+        return data["items"]
+    except Exception:
+        return None
+
+
+def save_cache(items: list[dict]) -> None:
+    try:
+        CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+        CACHE_PATH.write_text(json.dumps({"ts": time.time(), "items": items}))
+    except Exception:
+        pass

grokfeed/main.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import click
+
+
+@click.command()
+def app() -> None:
+    """Hacker News + Reddit terminal feed viewer."""
+    from .app import GrokFeedApp
+    from .config import CONFIG_PATH, load_config
+
+    config, created = load_config()
+    if created:
+        click.echo(f"Config created: {CONFIG_PATH}\nEdit it to add/remove subreddits.")
+    GrokFeedApp(config).run()
+
+
+if __name__ == "__main__":
+    app()

grokfeed/sources/__init__.py

@@ -0,0 +1,5 @@
+from .hn import fetch_hn_stories
+from .lobsters import fetch_lobsters_posts
+from .reddit import fetch_reddit_posts
+
+__all__ = ["fetch_hn_stories", "fetch_reddit_posts", "fetch_lobsters_posts"]

grokfeed/sources/comments.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import asyncio
+import html as _html
+import re as _re
+from dataclasses import dataclass
+
+import httpx
+
+USER_AGENT = "grokfeed:v0.1.0 (terminal feed reader)"
+HN_ITEM = "https://hacker-news.firebaseio.com/v0/item/{}.json"
+REDDIT_COMMENTS = "https://www.reddit.com/r/{subreddit}/comments/{post_id}.json?limit=50&depth=3"
+LOBSTERS_POST = "https://lobste.rs/s/{short_id}.json"
+
+
+@dataclass
+class Comment:
+    author: str
+    score: int
+    body: str
+    depth: int = 0
+
+
+def _strip_html(raw: str) -> str:
+    text = _re.sub(r"<p>", "\n\n", raw)
+    text = _re.sub(r"<[^>]+>", "", text)
+    return _html.unescape(text).strip()
+
+
+# ── HN ────────────────────────────────────────────────────────────────────────
+
+
+async def _fetch_hn_comment(
+    client: httpx.AsyncClient, sem: asyncio.Semaphore, cid: int
+) -> Comment | None:
+    async with sem:
+        try:
+            r = await client.get(HN_ITEM.format(cid), timeout=10)
+            r.raise_for_status()
+            d = r.json()
+            if not d or d.get("deleted") or d.get("dead") or d.get("type") != "comment":
+                return None
+            raw = d.get("text", "")
+            return Comment(
+                author=d.get("by", "[deleted]"),
+                score=0,  # HN comments have no score in API
+                body=_strip_html(raw) if raw else "",
+                depth=0,
+            )
+        except Exception:
+            return None
+
+
+async def fetch_hn_comments(story_id: int, limit: int = 30) -> list[Comment]:
+    sem = asyncio.Semaphore(10)
+    async with httpx.AsyncClient() as client:
+        r = await client.get(HN_ITEM.format(story_id), timeout=10)
+        r.raise_for_status()
+        d = r.json()
+        kids = (d.get("kids") or [])[:limit]
+        tasks = [_fetch_hn_comment(client, sem, kid) for kid in kids]
+        results = await asyncio.gather(*tasks)
+    return [c for c in results if c is not None]
+
+
+# ── Reddit ─────────────────────────────────────────────────────────────────────
+
+
+def _flatten_reddit(children: list, depth: int = 0, max_depth: int = 2) -> list[Comment]:
+    out: list[Comment] = []
+    for child in children:
+        if child.get("kind") != "t1":
+            continue
+        d = child.get("data", {})
+        body = d.get("body", "")
+        if body in ("[deleted]", "[removed]", ""):
+            continue
+        out.append(
+            Comment(
+                author=d.get("author", "[deleted]"),
+                score=d.get("score", 0),
+                body=body,
+                depth=depth,
+            )
+        )
+        if depth < max_depth:
+            replies = d.get("replies", "")
+            if isinstance(replies, dict):
+                reply_children = replies.get("data", {}).get("children", [])
+                out.extend(_flatten_reddit(reply_children, depth + 1, max_depth))
+    return out
+
+
+async def fetch_reddit_comments(subreddit: str, post_id: str) -> list[Comment]:
+    headers = {"User-Agent": USER_AGENT}
+    url = REDDIT_COMMENTS.format(subreddit=subreddit, post_id=post_id)
+    async with httpx.AsyncClient(headers=headers) as client:
+        try:
+            r = await client.get(url, timeout=15)
+            r.raise_for_status()
+            data = r.json()
+        except Exception:
+            return []
+    # data is [post_listing, comments_listing]
+    if not isinstance(data, list) or len(data) < 2:
+        return []
+    children = data[1].get("data", {}).get("children", [])
+    return _flatten_reddit(children)
+
+
+# ── lobste.rs ──────────────────────────────────────────────────────────────────
+
+
+async def fetch_lobsters_comments(short_id: str) -> list[Comment]:
+    headers = {"User-Agent": USER_AGENT}
+    url = LOBSTERS_POST.format(short_id=short_id)
+    async with httpx.AsyncClient(headers=headers) as client:
+        try:
+            r = await client.get(url, timeout=15)
+            r.raise_for_status()
+            data = r.json()
+        except Exception:
+            return []
+    out: list[Comment] = []
+    for c in data.get("comments", []):
+        raw = c.get("comment", "")
+        body = _strip_html(raw) if raw else ""
+        if not body:
+            continue
+        raw_user = c.get("commenting_user", "?")
+        author = raw_user if isinstance(raw_user, str) else raw_user.get("username", "?")
+        out.append(
+            Comment(
+                author=author,
+                score=c.get("score", 0),
+                body=body,
+                depth=c.get("indent_level", 0),
+            )
+        )
+    return out
+
+
+# ── Dispatcher ─────────────────────────────────────────────────────────────────
+
+
+async def fetch_comments(item: dict) -> list[Comment]:
+    source = item.get("source", "")
+    if source == "HN":
+        return await fetch_hn_comments(int(item["post_id"]))
+    elif source.startswith("r/"):
+        return await fetch_reddit_comments(item["subreddit"], item["post_id"])
+    elif source == "lobste.rs":
+        return await fetch_lobsters_comments(item["post_id"])
+    return []

grokfeed/sources/hn.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import asyncio
+import html as _html
+import re as _re
+from dataclasses import dataclass
+
+import httpx
+
+HN_BASE = "https://hacker-news.firebaseio.com/v0"
+HN_ITEM = HN_BASE + "/item/{}.json"
+HN_TOP = HN_BASE + "/topstories.json"
+
+
+def _strip_html(raw: str) -> str:
+    text = _re.sub(r"<p>", "\n\n", raw)
+    text = _re.sub(r"<[^>]+>", "", text)
+    return _html.unescape(text).strip()
+
+
+@dataclass
+class Story:
+    id: int
+    title: str
+    url: str
+    score: int
+    comments: int
+    body: str = ""  # present for Ask HN / Tell HN posts
+    source: str = "HN"
+
+
+async def _fetch_item(client: httpx.AsyncClient, item_id: int) -> Story | None:
+    try:
+        r = await client.get(HN_ITEM.format(item_id), timeout=10)
+        r.raise_for_status()
+        d = r.json()
+        if not d or d.get("type") != "story":
+            return None
+        raw_text = d.get("text", "")
+        return Story(
+            id=d["id"],
+            title=d.get("title", "(no title)"),
+            url=d.get("url") or f"https://news.ycombinator.com/item?id={d['id']}",
+            score=d.get("score", 0),
+            comments=d.get("descendants", 0),
+            body=_strip_html(raw_text) if raw_text else "",
+        )
+    except Exception:
+        return None
+
+
+async def fetch_hn_top_ids(client: httpx.AsyncClient) -> list[int]:
+    r = await client.get(HN_TOP, timeout=10)
+    r.raise_for_status()
+    return r.json()
+
+
+async def fetch_hn_stories_by_ids(
+    ids: list[int],
+    client: httpx.AsyncClient | None = None,
+) -> list[Story]:
+    async def _run(c: httpx.AsyncClient) -> list[Story]:
+        sem = asyncio.Semaphore(10)
+
+        async def _bounded(item_id: int) -> Story | None:
+            async with sem:
+                return await _fetch_item(c, item_id)
+
+        results = await asyncio.gather(*[_bounded(i) for i in ids])
+        return [s for s in results if s is not None]
+
+    if client is not None:
+        return await _run(client)
+    async with httpx.AsyncClient() as c:
+        return await _run(c)
+
+
+async def fetch_hn_stories(
+    count: int = 30,
+    client: httpx.AsyncClient | None = None,
+) -> list[Story]:
+    async def _run(c: httpx.AsyncClient) -> list[Story]:
+        ids = (await fetch_hn_top_ids(c))[:count]
+        return await fetch_hn_stories_by_ids(ids, c)
+
+    if client is not None:
+        return await _run(client)
+    async with httpx.AsyncClient() as c:
+        return await _run(c)

grokfeed/sources/lobsters.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import html as _html
+import re as _re
+from dataclasses import dataclass
+
+import httpx
+
+
+def _strip_html(raw: str) -> str:
+    text = _re.sub(r"<p>", "\n\n", raw)
+    text = _re.sub(r"<[^>]+>", "", text)
+    return _html.unescape(text).strip()
+
+
+LOBSTERS_URL = "https://lobste.rs/hottest.json"
+USER_AGENT = "grokfeed:v0.1.0 (terminal feed reader)"
+
+
+@dataclass
+class LobstersPost:
+    id: str
+    title: str
+    url: str
+    score: int
+    comments: int
+    body: str = ""  # description for discussion posts
+    source: str = "lobste.rs"
+
+
+async def fetch_lobsters_posts(
+    count: int = 25,
+    client: httpx.AsyncClient | None = None,
+) -> list[LobstersPost]:
+    async def _run(c: httpx.AsyncClient) -> list[LobstersPost]:
+        try:
+            r = await c.get(LOBSTERS_URL, timeout=15, headers={"User-Agent": USER_AGENT})
+            r.raise_for_status()
+            data = r.json()
+        except Exception:
+            return []
+        posts: list[LobstersPost] = []
+        for item in data[:count]:
+            raw_desc = item.get("description", "")
+            ext_url = item.get("url", "")
+            posts.append(
+                LobstersPost(
+                    id=item.get("short_id", ""),
+                    title=item.get("title", "(no title)"),
+                    url=ext_url or item.get("comments_url", ""),
+                    score=item.get("score", 0),
+                    comments=item.get("comment_count", 0),
+                    body=_strip_html(raw_desc) if raw_desc else "",
+                )
+            )
+        return posts
+
+    if client is not None:
+        return await _run(client)
+    async with httpx.AsyncClient() as c:
+        return await _run(c)