feedscope 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- feedscope/__init__.py +3 -0
- feedscope/__main__.py +4 -0
- feedscope/cli.py +154 -0
- feedscope/core/__init__.py +1 -0
- feedscope/core/browser.py +367 -0
- feedscope/core/config.py +367 -0
- feedscope/core/const.py +64 -0
- feedscope/core/display.py +62 -0
- feedscope/core/http.py +104 -0
- feedscope/core/i18n/__init__.py +202 -0
- feedscope/core/i18n/en.toml +447 -0
- feedscope/core/i18n/zh-CN.toml +444 -0
- feedscope/core/media.py +150 -0
- feedscope/core/models.py +86 -0
- feedscope/core/registry.py +53 -0
- feedscope/core/samples.py +68 -0
- feedscope/core/source.py +668 -0
- feedscope/core/text.py +54 -0
- feedscope/core/timefmt.py +53 -0
- feedscope/core/tree.py +134 -0
- feedscope/doctor.py +54 -0
- feedscope/login.py +168 -0
- feedscope/probe.py +131 -0
- feedscope/report.py +19 -0
- feedscope/sources/__init__.py +38 -0
- feedscope/sources/hn/__init__.py +66 -0
- feedscope/sources/hn/fetch.py +82 -0
- feedscope/sources/hn/parse.py +154 -0
- feedscope/sources/reddit/__init__.py +178 -0
- feedscope/sources/reddit/fetch.py +181 -0
- feedscope/sources/reddit/parse.py +318 -0
- feedscope/sources/rss/__init__.py +64 -0
- feedscope/sources/rss/fetch.py +94 -0
- feedscope/sources/rss/parse.py +116 -0
- feedscope/sources/telegram/__init__.py +61 -0
- feedscope/sources/telegram/fetch.py +79 -0
- feedscope/sources/telegram/parse.py +182 -0
- feedscope/sources/v2ex/__init__.py +64 -0
- feedscope/sources/v2ex/fetch.py +46 -0
- feedscope/sources/v2ex/parse.py +116 -0
- feedscope/sources/x/__init__.py +68 -0
- feedscope/sources/x/fetch.py +490 -0
- feedscope/sources/x/parse.py +535 -0
- feedscope/sources/zhihu/__init__.py +81 -0
- feedscope/sources/zhihu/fetch.py +402 -0
- feedscope/sources/zhihu/parse.py +235 -0
- feedscope/subscriptions.py +423 -0
- feedscope/ui/__init__.py +10 -0
- feedscope/ui/account.py +125 -0
- feedscope/ui/app.py +105 -0
- feedscope/ui/dialogs.py +127 -0
- feedscope/ui/feedback.py +64 -0
- feedscope/ui/focus_nav.py +113 -0
- feedscope/ui/labels.py +38 -0
- feedscope/ui/layout.py +66 -0
- feedscope/ui/list_render.py +42 -0
- feedscope/ui/screens/__init__.py +1 -0
- feedscope/ui/screens/answer.py +296 -0
- feedscope/ui/screens/detail.py +274 -0
- feedscope/ui/screens/language.py +59 -0
- feedscope/ui/screens/main.py +736 -0
- feedscope/ui/screens/main_setup.py +226 -0
- feedscope/ui/screens/reading.py +384 -0
- feedscope/ui/screens/source_setup.py +217 -0
- feedscope/ui/screens/subscriptions.py +196 -0
- feedscope/ui/theme.py +45 -0
- feedscope/ui/wrap.py +157 -0
- feedscope-0.1.0.dist-info/METADATA +302 -0
- feedscope-0.1.0.dist-info/RECORD +72 -0
- feedscope-0.1.0.dist-info/WHEEL +4 -0
- feedscope-0.1.0.dist-info/entry_points.txt +2 -0
- feedscope-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""HN(Hacker News)源(hn):首页/最新/Ask/Show 四个信息流 + 讨论树详情。
|
|
2
|
+
|
|
3
|
+
包内分层(全项目统一的源模板):
|
|
4
|
+
- fetch.py 取数层:Algolia 公开索引 API(免 key 免登录、无登录态)
|
|
5
|
+
- parse.py 解析层:Algolia JSON -> FeedItem / 楼层与评论树,纯函数可测
|
|
6
|
+
- __init__.py 编排层:Source 能力声明与 open_* 分派(楼层/评论翻页走
|
|
7
|
+
core 的内存切片基类)
|
|
8
|
+
|
|
9
|
+
结构映射是知乎式三层:story 本体(正文 = 原文一行 + Ask/Show 自述)->
|
|
10
|
+
顶层评论当楼层(unit「评论」)-> 楼层内回复树当评论层(unit「回复」)。
|
|
11
|
+
items/:id 一次给全树,楼层与评论翻页都是内存切片——详情打开后评论层
|
|
12
|
+
零额外请求。无楼层引导(descendants=0 的楼层头不带「N 条回复」)。
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from feedscope.core.models import FeedItem
|
|
18
|
+
from feedscope.core.i18n import t
|
|
19
|
+
from feedscope.core.registry import register
|
|
20
|
+
from feedscope.core.source import (
|
|
21
|
+
FeedLoader,
|
|
22
|
+
SliceDetailLoader,
|
|
23
|
+
Source,
|
|
24
|
+
SourceError,
|
|
25
|
+
)
|
|
26
|
+
from feedscope.sources.hn.fetch import HnFeedLoader, fetch_item
|
|
27
|
+
from feedscope.sources.hn.parse import SOURCE_ID, parse_story
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class HnDetailLoader(SliceDetailLoader):
|
|
31
|
+
"""HN 详情加载器:树已全在内存——楼层翻页与评论层都是切片,零额外请求。
|
|
32
|
+
|
|
33
|
+
全部结构在 SliceDetailLoader.from_detail(楼层/评论树成对进切片),
|
|
34
|
+
本类只声明报错文案里的源标签(楼层页宽用基类默认)。"""
|
|
35
|
+
|
|
36
|
+
source_label = "HN"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@register
|
|
40
|
+
class HnSource(Source):
|
|
41
|
+
source_id = SOURCE_ID
|
|
42
|
+
label = "HN"
|
|
43
|
+
supports_detail = True
|
|
44
|
+
supports_comments = True # 详情屏 Enter 进楼层的回复树(树已全在内存)
|
|
45
|
+
feeds: list[str] = ["front", "new", "ask", "show"]
|
|
46
|
+
# ask/show = Ask HN(向社区提问)/ Show HN(展示自己的项目),HN 官方两大
|
|
47
|
+
# 经典分类;标签与其余源同款两字中文(列表里帖子标题仍带 Ask/Show HN 前缀)
|
|
48
|
+
feed_labels: dict[str, str] = {
|
|
49
|
+
"front": "sources.hn.feeds.front",
|
|
50
|
+
"new": "sources.hn.feeds.new",
|
|
51
|
+
"ask": "sources.hn.feeds.ask",
|
|
52
|
+
"show": "sources.hn.feeds.show",
|
|
53
|
+
}
|
|
54
|
+
detail_kinds = frozenset({"story", "ask", "show"}) # 全部条目 Enter 都进终端详情
|
|
55
|
+
detail_unit_label = "units.comment" # 楼层单位词:顶层评论串(知乎「回答」、X「回复」)
|
|
56
|
+
comment_unit_label = "units.reply" # 楼层内的讨论(知乎叫「评论」)
|
|
57
|
+
|
|
58
|
+
async def open_feed(self, name: str) -> FeedLoader:
|
|
59
|
+
label = self.label_of(name)
|
|
60
|
+
return await self.first_batch(HnFeedLoader(name, label), label)
|
|
61
|
+
|
|
62
|
+
async def open_detail(self, item: FeedItem) -> SliceDetailLoader:
|
|
63
|
+
root = await fetch_item(item.id)
|
|
64
|
+
if not root.get("id"):
|
|
65
|
+
raise SourceError(t("errors.hn_no_id"))
|
|
66
|
+
return HnDetailLoader.from_detail(parse_story(root), fallback_title=item.title)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""取数层:HN 走 Algolia 公开索引 API(免 key 免登录)——search 列表 + items/:id 整树。
|
|
2
|
+
|
|
3
|
+
两个端点都是单请求水合 JSON:列表一次 30 条全字段;详情一次给全评论树
|
|
4
|
+
(楼层/评论层的翻页全在内存切片,见 __init__ 的内存 Loader),fetch 层只有
|
|
5
|
+
两个无状态 GET 与一个翻页 Loader。索引有分钟级延迟、列表按 points 排序
|
|
6
|
+
(非官方榜序)——接受这两点换零解析成本(官方 Firebase 逐条取、官方
|
|
7
|
+
HTML 要解析 DOM);真在意榜序时只换本文件的列表通道,树端点不动。
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from feedscope.core import http
|
|
13
|
+
from feedscope.core.models import FeedItem
|
|
14
|
+
from feedscope.core.source import FeedLoader, SourceError, dedup_append
|
|
15
|
+
from feedscope.core.i18n import t
|
|
16
|
+
from feedscope.sources.hn.parse import parse_search
|
|
17
|
+
|
|
18
|
+
_API = "https://hn.algolia.com/api/v1"
|
|
19
|
+
|
|
20
|
+
# 各内置流的 (search 端点, 过滤参数)(feeds 词汇表与顺序见 __init__):
|
|
21
|
+
# front_page 是「当前首页」快照(30 条);ask/show 按 tag 过滤;new 按时间倒序
|
|
22
|
+
FEEDS: dict[str, tuple[str, dict[str, str]]] = {
|
|
23
|
+
"front": (f"{_API}/search", {"tags": "front_page"}),
|
|
24
|
+
"new": (f"{_API}/search_by_date", {"tags": "story"}),
|
|
25
|
+
"ask": (f"{_API}/search", {"tags": "ask_hn"}),
|
|
26
|
+
"show": (f"{_API}/search", {"tags": "show_hn"}),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
HITS_PER_PAGE = 30 # 与 HN 官方首页同宽(front_page 恰好 30 条,一页装下)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
async def fetch_feed(name: str, label: str, page: int) -> dict:
|
|
33
|
+
"""一个信息流的一页 search 结果(原始 JSON dict;解析/校验见 http.get_json)。"""
|
|
34
|
+
url, base_params = FEEDS[name]
|
|
35
|
+
return await http.get_json(
|
|
36
|
+
url,
|
|
37
|
+
source=f"HN「{label}」",
|
|
38
|
+
params={**base_params, "hitsPerPage": HITS_PER_PAGE, "page": page},
|
|
39
|
+
expect=dict,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
async def fetch_item(object_id: str) -> dict:
|
|
44
|
+
"""一条 story 的 items/:id 整棵评论树(原始 JSON dict;解析/校验见 http.get_json)。"""
|
|
45
|
+
return await http.get_json(
|
|
46
|
+
f"{_API}/items/{object_id}", source=t("sources.hn.detail"), expect=dict
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class HnFeedLoader(FeedLoader):
|
|
51
|
+
"""信息流加载器:search 按页翻(page 0 起、nbPages 到头),id 去重。
|
|
52
|
+
|
|
53
|
+
索引重排会让跨页撞见已见条目(分数涨跌挤进挤出),dedup_append 与
|
|
54
|
+
各源共用「同一 id 只出现一次」的不变量;无资源可释(连接池进程级共享)。
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
def __init__(self, name: str, label: str) -> None:
|
|
58
|
+
self.items: list[FeedItem] = []
|
|
59
|
+
self._name = name
|
|
60
|
+
self._label = label
|
|
61
|
+
self._seen: set[str] = set()
|
|
62
|
+
self._page = 0 # 下一页页号(Algolia 0 起)
|
|
63
|
+
self._nb_pages = 1
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def done(self) -> bool:
|
|
67
|
+
return self._page >= self._nb_pages
|
|
68
|
+
|
|
69
|
+
async def load_more(self) -> list[FeedItem]:
|
|
70
|
+
if self.done:
|
|
71
|
+
return []
|
|
72
|
+
data = await fetch_feed(self._name, self._label, self._page)
|
|
73
|
+
hits = data.get("hits")
|
|
74
|
+
if not isinstance(hits, list):
|
|
75
|
+
raise SourceError(t("errors.hn_no_hits", label=self._label))
|
|
76
|
+
new = dedup_append(self.items, self._seen, parse_search(hits))
|
|
77
|
+
self._page += 1
|
|
78
|
+
try:
|
|
79
|
+
self._nb_pages = max(1, int(data.get("nbPages") or 1))
|
|
80
|
+
except (TypeError, ValueError):
|
|
81
|
+
self._nb_pages = 1
|
|
82
|
+
return new
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""解析层:Algolia JSON -> FeedItem / 楼层与评论树,纯函数不碰网络。
|
|
2
|
+
|
|
3
|
+
HN 的结构映射是知乎式三层(词汇重映射,UI 零特判):
|
|
4
|
+
- 列表 search hit -> FeedItem(objectID 即条目 id,tag 分 kind)
|
|
5
|
+
- items/:id 根 -> story 本体(链接帖正文 = 原文一行;Ask/Show 自述全文)
|
|
6
|
+
- 顶层评论 -> 楼层(Answer,unit「评论」);楼层的回复树 -> 评论层
|
|
7
|
+
(Comment,unit「回复」)。Comment 模型只有两层而 HN 常嵌四五层:
|
|
8
|
+
直接子回复做顶层,更深的拍平进 replies;隔层的回复加「@父作者」
|
|
9
|
+
前缀保住「在回复谁」的指向(紧贴楼上的不加,同知乎观感)。整棵树
|
|
10
|
+
一次已在内存,翻页是切片不是请求(见 __init__ 的内存 Loader)。
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
|
|
17
|
+
from feedscope.core.const import safe_int
|
|
18
|
+
from feedscope.core.display import fmt_count
|
|
19
|
+
from feedscope.core.i18n import t
|
|
20
|
+
from feedscope.core.media import text_to_md
|
|
21
|
+
from feedscope.core.models import Answer, Comment, FeedItem, Floor
|
|
22
|
+
from feedscope.core.timefmt import fmt_iso
|
|
23
|
+
from feedscope.core.tree import (
|
|
24
|
+
deleted_author,
|
|
25
|
+
deleted_body,
|
|
26
|
+
TreeAccessors,
|
|
27
|
+
build_floors,
|
|
28
|
+
count_kept,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
SOURCE_ID = "hn"
|
|
32
|
+
|
|
33
|
+
# HN 原生条目页:自帖的兜底 url 与楼层直达链接(评论 permalink 也是 item 页)
|
|
34
|
+
_ITEM_URL = "https://news.ycombinator.com/item?id="
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _md(html: object) -> str:
|
|
38
|
+
"""HN 受限 HTML(p/a/i/code)-> markdown;非 str / 空回空串(管线见 core.media)。"""
|
|
39
|
+
return text_to_md(html) if isinstance(html, str) else ""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _kids(node: dict) -> list[dict]:
|
|
43
|
+
kids = node.get("children")
|
|
44
|
+
return [c for c in kids if isinstance(c, dict)] if isinstance(kids, list) else []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _keep(node: dict) -> bool:
|
|
48
|
+
"""空壳节点(无正文也无孩子,典型是被删且无回复的评论)整支跳过。"""
|
|
49
|
+
return bool(_md(node.get("text")) or _kids(node))
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _author(node: dict) -> str:
|
|
53
|
+
return str(node.get("author") or "") or deleted_author()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _time(node: dict) -> str:
|
|
57
|
+
created = node.get("created_at")
|
|
58
|
+
return fmt_iso(created if isinstance(created, str) else None)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _comment_of(node: dict) -> Comment:
|
|
62
|
+
"""Algolia 节点 -> Comment 本体(@ 前缀由 core.tree 统一拼装)。"""
|
|
63
|
+
return Comment(
|
|
64
|
+
author=_author(node),
|
|
65
|
+
votes="", # Algolia 树节点不带分值,HN 评论也无赞数语义
|
|
66
|
+
content_md=_md(node.get("text")) or deleted_body(),
|
|
67
|
+
time=_time(node),
|
|
68
|
+
raw_id=str(node.get("id") or ""),
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _answer_of(node: dict, n: int) -> Answer:
|
|
73
|
+
"""楼层本体(core.tree.build_floors 注入):字段来源是站差异,n 为回复数。"""
|
|
74
|
+
cid = str(node.get("id") or "")
|
|
75
|
+
return Answer(
|
|
76
|
+
author=_author(node),
|
|
77
|
+
votes="",
|
|
78
|
+
content_md=_md(node.get("text")) or deleted_body(),
|
|
79
|
+
time=_time(node),
|
|
80
|
+
url=f"{_ITEM_URL}{cid}" if cid else "",
|
|
81
|
+
comment_count=t("display.replies_n", n=n) if n else "", # 0 条不引导 Enter
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# 差异注入:HN 的树是 Algolia dict(结构见各 accessor;拍平骨架在 core.tree)
|
|
86
|
+
_ACC = TreeAccessors(keep=_keep, kids_of=_kids, author_of=_author, comment_of=_comment_of)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _meta(points: int, comments: int, author: str) -> str:
|
|
90
|
+
"""元信息列「N 点 · N 评 · 作者」:列表 hit 与详情 root 共用。"""
|
|
91
|
+
parts = [
|
|
92
|
+
t("words.points", n=fmt_count(points)),
|
|
93
|
+
t("words.comments_short", n=fmt_count(comments)),
|
|
94
|
+
]
|
|
95
|
+
if author:
|
|
96
|
+
parts.append(author)
|
|
97
|
+
return " · ".join(parts)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_search(hits: list[dict]) -> list[FeedItem]:
|
|
101
|
+
"""search hits -> FeedItem 列表(保持返回序,野数据逐条防御)。"""
|
|
102
|
+
items: list[FeedItem] = []
|
|
103
|
+
for hit in hits:
|
|
104
|
+
oid = str(hit.get("objectID") or "")
|
|
105
|
+
if not oid.isdigit():
|
|
106
|
+
continue # 无 id 无法寻址,跳条不跳流
|
|
107
|
+
tags = hit.get("_tags")
|
|
108
|
+
tags = tags if isinstance(tags, list) else []
|
|
109
|
+
kind = "ask" if "ask_hn" in tags else "show" if "show_hn" in tags else "story"
|
|
110
|
+
items.append(
|
|
111
|
+
FeedItem(
|
|
112
|
+
source=SOURCE_ID,
|
|
113
|
+
id=oid,
|
|
114
|
+
title=str(hit.get("title") or "").strip() or t("words.no_title"),
|
|
115
|
+
url=str(hit.get("url") or "") or f"{_ITEM_URL}{oid}", # 自帖无外链,兜底条目页
|
|
116
|
+
kind=kind,
|
|
117
|
+
meta=_meta(
|
|
118
|
+
safe_int(hit.get("points")), safe_int(hit.get("num_comments")),
|
|
119
|
+
str(hit.get("author") or ""),
|
|
120
|
+
),
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
return items
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass
|
|
127
|
+
class StoryDetail:
|
|
128
|
+
"""items/:id 的解析结果:story 本体 + 全部楼层(含各自的回复树)。"""
|
|
129
|
+
|
|
130
|
+
title: str
|
|
131
|
+
meta: str
|
|
132
|
+
time: str
|
|
133
|
+
description_md: str
|
|
134
|
+
floors: list[Floor] = field(default_factory=list)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def parse_story(root: dict) -> StoryDetail:
|
|
138
|
+
"""items/:id 根 -> StoryDetail(story 本体 + 顶层评论楼层与各自回复树)。"""
|
|
139
|
+
blocks = []
|
|
140
|
+
if url := str(root.get("url") or ""):
|
|
141
|
+
blocks.append(f"{t('words.original')}({url})") # 链接帖正文就是外链一行,Ctrl+O 直达
|
|
142
|
+
if text_md := _md(root.get("text")):
|
|
143
|
+
blocks.append(text_md) # Ask/Show 的自述全文
|
|
144
|
+
# 顶层节点 -> 楼层(keep 过滤 / 计数 / 拍平的骨架在 core.tree.build_floors)
|
|
145
|
+
return StoryDetail(
|
|
146
|
+
title=str(root.get("title") or "").strip(),
|
|
147
|
+
meta=_meta(
|
|
148
|
+
safe_int(root.get("points")), count_kept(root, _ACC),
|
|
149
|
+
str(root.get("author") or ""),
|
|
150
|
+
),
|
|
151
|
+
time=_time(root),
|
|
152
|
+
description_md="\n\n".join(blocks),
|
|
153
|
+
floors=build_floors(_kids(root), _ACC, answer_of=_answer_of),
|
|
154
|
+
)
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Reddit 源(reddit):首页/热门两个内置流 + subreddit 订阅表 + 帖子详情(评论树)。
|
|
2
|
+
|
|
3
|
+
包内分层(全项目统一的源模板):
|
|
4
|
+
- fetch.py 取数层:网站自己的 HTML 通道(shreddit SSR 页面 + faceplate
|
|
5
|
+
partial 翻页),由常驻 reddit.com 通道页签的真 Chrome 发出(知乎同款,
|
|
6
|
+
TLS/头序/cookie 与登录会话自洽;.json 遗留端点已全线 403)
|
|
7
|
+
- parse.py 解析层:shreddit HTML -> FeedItem / 帖子与评论树,纯函数可测
|
|
8
|
+
- __init__.py 编排层:Source 能力声明与 open_* 分派 + 内存分页 Loader
|
|
9
|
+
|
|
10
|
+
结构映射是 HN 式三层:帖子(自述帖 HTML 转 markdown / 链接帖原文一行 /
|
|
11
|
+
媒体记号)-> 顶层评论当楼层(unit「评论」)-> 楼层内回复树当评论层
|
|
12
|
+
(unit「回复」)。permalink 页单请求 SSR 自带整树,楼层与评论翻页全在
|
|
13
|
+
内存切片——详情打开后评论层零额外请求(同 HN)。
|
|
14
|
+
|
|
15
|
+
feeds 是混形合成(介于知乎减法与 RSS 加法之间):内置流 home=首页 /
|
|
16
|
+
popular=r/popular 走 hide 减法,[reddit.feeds] 订阅表(subreddit 名即
|
|
17
|
+
key,url 存站内地址自解释)走加法——零订阅仍有内置流(与 RSS 的纯订阅
|
|
18
|
+
合成不同)。subscribable=True:F2 里 → 进订阅管理屏(与 RSS 共用一屏,
|
|
19
|
+
按源分派见 ui/screens/subscriptions.py 的 store)。
|
|
20
|
+
|
|
21
|
+
登录:本源必须登录。`feedscope login reddit` 弹 Chrome 手动输账号密码
|
|
22
|
+
(+2FA)。判据一律是 token_v2 的 JWT 本地解码(登录态 sub="user"、
|
|
23
|
+
匿名态 sub="loid",见 _token_logged_in)——Reddit 的判登 cookie 全不可
|
|
24
|
+
信:reddit_session 匿名流量也发、token_v2 光看在不在更不行;解码零网络
|
|
25
|
+
零请求,匿名流量伪造不出。
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import asyncio
|
|
31
|
+
import base64
|
|
32
|
+
import json as _json
|
|
33
|
+
from typing import ClassVar
|
|
34
|
+
|
|
35
|
+
from feedscope.core import config
|
|
36
|
+
from feedscope.core.i18n import t
|
|
37
|
+
from feedscope.core.models import FeedItem
|
|
38
|
+
from feedscope.core.registry import register
|
|
39
|
+
from feedscope.core.source import (
|
|
40
|
+
FeedLoader,
|
|
41
|
+
SliceDetailLoader,
|
|
42
|
+
Source,
|
|
43
|
+
SourceError,
|
|
44
|
+
login_gate,
|
|
45
|
+
require_feed_items,
|
|
46
|
+
)
|
|
47
|
+
from feedscope.sources.reddit.fetch import RedditFeedLoader, fetch_thread_html
|
|
48
|
+
from feedscope.sources.reddit.parse import SOURCE_ID, parse_thread_html
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class RedditDetailLoader(SliceDetailLoader):
|
|
52
|
+
"""Reddit 详情加载器:树已全在内存——楼层翻页与评论层都是切片,零额外请求。
|
|
53
|
+
|
|
54
|
+
全部结构在 SliceDetailLoader.from_detail(楼层/评论树成对进切片),
|
|
55
|
+
本类只声明报错文案里的源标签(楼层页宽用基类默认)。"""
|
|
56
|
+
|
|
57
|
+
source_label = "Reddit"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _token_logged_in(token_v2: str | None) -> bool:
|
|
61
|
+
"""token_v2 的 JWT claims 是登录的本地铁证:登录态 sub="user"(匿名态 sub="loid")。
|
|
62
|
+
|
|
63
|
+
Reddit 的判登 cookie 全都不可信:reddit_session 匿名流量也发(实测打开
|
|
64
|
+
登录页即出现)、token_v2 光看在不在更不行——但 token_v2 本体是 JWT,
|
|
65
|
+
登录后 Reddit 重签的版本 claims 里 sub 从 "loid"(Logged-Out ID)变成
|
|
66
|
+
"user" 且带 at=1/aid=账号id(2026-11 实测两种形态各解码一次)。本地
|
|
67
|
+
解码零请求、匿名流量伪造不出,是判登/登录轮询的地面真值。
|
|
68
|
+
"""
|
|
69
|
+
if not token_v2:
|
|
70
|
+
return False
|
|
71
|
+
try:
|
|
72
|
+
payload = token_v2.split(".")[1]
|
|
73
|
+
payload += "=" * (-len(payload) % 4)
|
|
74
|
+
claims = _json.loads(base64.urlsafe_b64decode(payload))
|
|
75
|
+
return claims.get("sub") == "user"
|
|
76
|
+
except (IndexError, ValueError):
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def _login_verified(ctx) -> bool:
|
|
81
|
+
"""登录成功的浏览器级判据(login.py 轮询 login_verify):token_v2 解码出登录态。
|
|
82
|
+
|
|
83
|
+
读登录窗口上下文的 cookie、本地解码 JWT(见 _token_logged_in)——零网络
|
|
84
|
+
(这台网络 www 的 me.json 连登录会话都 403,问不了服务端)、零页面
|
|
85
|
+
依赖(登录中途跳 accounts 域也不影响)。登录完成 Reddit 重签 token_v2
|
|
86
|
+
的瞬间即可检出。
|
|
87
|
+
"""
|
|
88
|
+
cookies = await ctx.cookies("https://www.reddit.com")
|
|
89
|
+
jar = {c["name"]: c["value"] for c in cookies}
|
|
90
|
+
return _token_logged_in(jar.get("token_v2"))
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
async def require_login(what: str) -> None:
|
|
94
|
+
"""判登门:token_v2 JWT 过门(通道页签的 cookie 由浏览器自带,无需另建会话)。
|
|
95
|
+
|
|
96
|
+
公共 API:subscriptions 的 subreddit 添加入口也过这道门(订阅管理屏
|
|
97
|
+
没有账号动线,LoginRequired 的消息自带指引)。cookie 读取与消息模板
|
|
98
|
+
在 core.source.login_gate,判据(JWT 解码)是本源差异。
|
|
99
|
+
"""
|
|
100
|
+
await login_gate(
|
|
101
|
+
"https://www.reddit.com",
|
|
102
|
+
lambda c: _token_logged_in(c.get("token_v2")),
|
|
103
|
+
label="Reddit",
|
|
104
|
+
what=what,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@register
|
|
109
|
+
class RedditSource(Source):
|
|
110
|
+
source_id = SOURCE_ID
|
|
111
|
+
label = "Reddit"
|
|
112
|
+
supports_detail = True
|
|
113
|
+
supports_comments = True # 详情屏 Enter 进楼层的回复树(树已全在内存)
|
|
114
|
+
subscribable = True # feeds 除内置流外还来自 [reddit.feeds] 订阅表:F2 里 → 进订阅管理
|
|
115
|
+
detail_kinds = frozenset({"link", "self"}) # 全部条目 Enter 都进终端详情
|
|
116
|
+
detail_unit_label = "units.comment" # 楼层单位词:顶层评论串(同 HN)
|
|
117
|
+
comment_unit_label = "units.reply" # 楼层内的讨论(同 HN)
|
|
118
|
+
login_url = "https://www.reddit.com/login"
|
|
119
|
+
login_cookie = "token_v2" # 兜底显示用(真实判据是 login_verify 的 JWT 解码)
|
|
120
|
+
login_verify = staticmethod(
|
|
121
|
+
_login_verified
|
|
122
|
+
) # 登录成功 = token_v2 解码出登录态(cookie 出现不算数)
|
|
123
|
+
logout_cookies: ClassVar[list[str]] = ["reddit_session", "token_v2"] # 登出两枚都清
|
|
124
|
+
|
|
125
|
+
# 内置流:流名 -> 列表页 URL(订阅流的页面是 https://www.reddit.com/r/{key}/)
|
|
126
|
+
_BUILTIN: ClassVar[dict[str, str]] = {
|
|
127
|
+
"home": "https://www.reddit.com/",
|
|
128
|
+
"popular": "https://www.reddit.com/r/popular/",
|
|
129
|
+
}
|
|
130
|
+
_BUILTIN_LABELS: ClassVar[dict[str, str]] = {
|
|
131
|
+
"home": "sources.reddit.feeds.home", "popular": "sources.reddit.feeds.popular",
|
|
132
|
+
}
|
|
133
|
+
# 订阅 key 不得撞内置流名(/r/home、/r/popular 真实存在,撞名会遮住内置流):
|
|
134
|
+
# 从 _BUILTIN 派生,新增内置流时保留字自动跟上(subscriptions 的拒收读这里)
|
|
135
|
+
reserved_feed_keys = tuple(_BUILTIN)
|
|
136
|
+
|
|
137
|
+
def __init__(self) -> None:
|
|
138
|
+
"""混形合成:内置流剔 hide(减法)+ 订阅表各成一流(加法,RSS 同款)。
|
|
139
|
+
|
|
140
|
+
不调 super().__init__()——它的合成规则只对类默认流做减法;这里 feeds
|
|
141
|
+
还要拼 [reddit.feeds] 的订阅 key。订阅 key 与内置流撞名(/r/home、
|
|
142
|
+
/r/popular 真实存在)在添加入口已拦,手编配置漏网的在这里静默跳过。
|
|
143
|
+
"""
|
|
144
|
+
hidden = set(config.hidden_feeds(self.source_id))
|
|
145
|
+
subs = [
|
|
146
|
+
f
|
|
147
|
+
for f in config.feeds_for(self.source_id)
|
|
148
|
+
if f.key not in hidden and f.key not in self._BUILTIN
|
|
149
|
+
]
|
|
150
|
+
self._subs = {f.key: f for f in subs}
|
|
151
|
+
self.feeds = [f for f in self._BUILTIN if f not in hidden] + list(self._subs)
|
|
152
|
+
self.feed_labels = {f: self._BUILTIN_LABELS[f] for f in self._BUILTIN if f not in hidden}
|
|
153
|
+
self.feed_labels.update({k: f.label for k, f in self._subs.items()})
|
|
154
|
+
|
|
155
|
+
async def open_feed(self, name: str) -> FeedLoader:
|
|
156
|
+
label = self.label_of(name)
|
|
157
|
+
page = (
|
|
158
|
+
self._BUILTIN.get(name) or f"https://www.reddit.com/r/{name}/"
|
|
159
|
+
) # 订阅 key 即 subreddit 名
|
|
160
|
+
await require_login(t("errors.what_feed", label=label))
|
|
161
|
+
loader = RedditFeedLoader(page, label)
|
|
162
|
+
await loader.load_more() # 首批(判登在此抛 LoginRequired)
|
|
163
|
+
if name == "home" and not loader.items:
|
|
164
|
+
# 首页是账号在 Reddit 站内「关注版块」的聚合:账号没关注任何版块
|
|
165
|
+
# 就是真空(实测新号如此),不是改版——通用 0 条守卫会误导
|
|
166
|
+
raise SourceError(t("errors.reddit_home_empty"))
|
|
167
|
+
require_feed_items(loader.items, self.label_text, label) # 0 条 = 大概率改版
|
|
168
|
+
return loader
|
|
169
|
+
|
|
170
|
+
async def open_detail(self, item: FeedItem) -> SliceDetailLoader:
|
|
171
|
+
await require_login(t("errors.what_post_detail"))
|
|
172
|
+
html = await fetch_thread_html(item.url) # 列表给的绝对 permalink(slug 全)
|
|
173
|
+
# 1.5MB 级 SSR 页的解析 ~160ms:放工作线程跑,不卡事件循环(解析是
|
|
174
|
+
# 纯函数;HTML2Text 已线程本地化,见 core.media.h2t)。换篇取消的是
|
|
175
|
+
# await——线程会把旧结果跑完但协程已死、结果自然丢弃(线程无法取消,
|
|
176
|
+
# 白算一次是换篇的既有代价)
|
|
177
|
+
detail = await asyncio.to_thread(parse_thread_html, html, fallback_title=item.title)
|
|
178
|
+
return RedditDetailLoader.from_detail(detail)
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""取数层:Reddit 走通道页签(知乎同款)——常驻 reddit.com 页签的页面 fetch。
|
|
2
|
+
|
|
3
|
+
2026 年起边缘对 .json 遗留端点全线 403(第三方通道关闭的一部分),且对
|
|
4
|
+
非浏览器画像(TLS/头序/HTTP2 指纹)的检查随时可能收紧——取数全部改由
|
|
5
|
+
常驻页签里的真 Chrome 发出(fetch credentials include):TLS、头序、
|
|
6
|
+
cookie 与登录会话完全自洽,httpx 侧的头画像维护整体退役。网站自己的
|
|
7
|
+
HTML 全放行:页面(SSR,帖子/评论树直接织在 HTML 里)与翻页 partial
|
|
8
|
+
(text/vnd.reddit.partial+html)页签里都拿得到。
|
|
9
|
+
|
|
10
|
+
- 通道:进程级单例页签停在 www.reddit.com 上(同源 fetch 与 cookie 的
|
|
11
|
+
前提,生命周期见 core/browser.ChannelTab),evaluate 撞上在途跳转/页签
|
|
12
|
+
被关都重建重试一次;判登门在编排层(token_v2 JWT 解码,见
|
|
13
|
+
__init__.py),门没过到不了这层
|
|
14
|
+
- 列表:流页面起步(首页 / 、热门 /r/popular/ 、订阅 /r/{sub}/ ),翻页
|
|
15
|
+
跟随页面/片段里 faceplate-partial slot="load-after" 的 src 链(服务端
|
|
16
|
+
给全参 URL,feeds/*-feed 与 community-more-posts 两族不用自己认)
|
|
17
|
+
- 详情:帖子 permalink 页 SSR 自带整棵评论树(1.5MB 级;深度大的树被
|
|
18
|
+
服务端截断,解析层给「另有 N 条未展开」)
|
|
19
|
+
- x-reddit-client-version:随 Reddit 部署轮换,且浏览器不会自动带
|
|
20
|
+
(SPA 自定义头)——照旧收割,但比 httpx 时代更便宜:服务端把版本织
|
|
21
|
+
进每个页面 HTML,每个页面响应顺手正则一次(进程内缓存,首批 partial
|
|
22
|
+
通常零额外请求),partial 403 时再专割一次自愈(版本确实轮换过才重试)
|
|
23
|
+
- 401/403 且重割无效时给「登录态可能已过期或被风控」的指引——绝不自动
|
|
24
|
+
清 cookie(历史教训:误清即登录死循环)
|
|
25
|
+
- 样本落盘在 ChannelTab.fetch 里统一做(probe 巡检对着真实 HTML 修
|
|
26
|
+
parse)
|
|
27
|
+
|
|
28
|
+
请求画像:刷流 1 页面 + N-1 片段(每片段 50-90 帖)、开帖 1 请求,远在
|
|
29
|
+
Reddit 登录态限流(约 100 次/分)之下。
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import re
|
|
35
|
+
|
|
36
|
+
from feedscope.core.browser import ChannelTab
|
|
37
|
+
from feedscope.core.models import FeedItem
|
|
38
|
+
from feedscope.core.source import FeedLoader, SourceError, dedup_append
|
|
39
|
+
from feedscope.core.i18n import t
|
|
40
|
+
from feedscope.sources.reddit.parse import (
|
|
41
|
+
parse_next_html,
|
|
42
|
+
parse_posts_html,
|
|
43
|
+
parse_subreddit_page,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
_API = "https://www.reddit.com"
|
|
47
|
+
|
|
48
|
+
_tab = ChannelTab(
|
|
49
|
+
f"{_API}/", name="reddit"
|
|
50
|
+
) # 停靠 www.reddit.com/:同源 fetch(referer/cookie)的前提
|
|
51
|
+
|
|
52
|
+
# 一次 load_more 至少攒够的条数:SSR 页面只嵌 3-10 帖(站点首屏后自己
|
|
53
|
+
# 也会续拉一批),不足就顺着 load-after 链补齐——「一屏上下、翻页有货」
|
|
54
|
+
# (HN 30、X 20 的同款折中)
|
|
55
|
+
_MIN_BATCH = 20
|
|
56
|
+
|
|
57
|
+
# x-reddit-client-version 的形态(如 2026-11-04T13:46Z~456d6f1d):部署时
|
|
58
|
+
# 轮换,服务端织进每个页面的 HTML
|
|
59
|
+
_VERSION_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z~[0-9a-f]{6,8}")
|
|
60
|
+
|
|
61
|
+
# partial 翻页的请求头(与 SPA 自己发的同形);页面请求只需要 Accept html。
|
|
62
|
+
# referer 不手设:页签停在 www.reddit.com/ 上,同源 fetch 的自然 referer
|
|
63
|
+
# 就是它(2026-11 实测矩阵里 referer 在场即可,同站即同形)
|
|
64
|
+
_PARTIAL_ACCEPT = "text/vnd.reddit.partial+html, text/html;q=0.9"
|
|
65
|
+
_HTML_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9"
|
|
66
|
+
|
|
67
|
+
_version: str | None = None # 进程级缓存(页面响应顺手更新,403 自愈时重割)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async def _raw_fetch(
|
|
71
|
+
url: str, accept: str, version: str | None, *, source: str
|
|
72
|
+
) -> tuple[int | None, str]:
|
|
73
|
+
"""一次通道 fetch -> (HTTP 状态码或 None, HTML 正文);解包/报错/样本落盘见 ChannelTab.fetch。
|
|
74
|
+
|
|
75
|
+
状态码原样上交(403 的自愈决策在 _get_html),成功时正文连同状态码
|
|
76
|
+
一起返回——收割路径要 403 页不炸,取数路径才转错误。手动头只有
|
|
77
|
+
Accept 与 partial 的版本头/retry 头(浏览器不会自动带的)。
|
|
78
|
+
"""
|
|
79
|
+
headers = {"Accept": accept}
|
|
80
|
+
if version:
|
|
81
|
+
headers["x-reddit-client-version"] = version
|
|
82
|
+
headers["x-reddit-retry"] = "attempt=0, max=2"
|
|
83
|
+
status, body = await _tab.fetch(url, source=source, headers=headers, as_text=True)
|
|
84
|
+
return status, (body or "")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
async def _harvest_version() -> None:
|
|
88
|
+
"""专割一次版本头(从热门页 HTML);失败保留旧值。
|
|
89
|
+
|
|
90
|
+
页面响应的顺手收割(见 _get_html)让这条路通常只剩 403 自愈在走:
|
|
91
|
+
partial 首用且缓存为空、或版本轮换后的重割。
|
|
92
|
+
"""
|
|
93
|
+
global _version
|
|
94
|
+
try:
|
|
95
|
+
_, body = await _raw_fetch(f"{_API}/r/popular/", _HTML_ACCEPT, None, source="Reddit")
|
|
96
|
+
except SourceError:
|
|
97
|
+
return # 收割失败保留旧值(可能是网络抖动,不是版本问题)
|
|
98
|
+
if m := _VERSION_RE.search(body):
|
|
99
|
+
_version = m.group(0)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
async def _get_html(url: str, *, source: str, partial: bool = False) -> str:
|
|
103
|
+
"""一次通道 fetch -> HTML 文本;partial 首用前收割版本头,403 时重割自愈一次。"""
|
|
104
|
+
global _version
|
|
105
|
+
if partial and not _version:
|
|
106
|
+
await _harvest_version() # partial 必须带版本头:先割再用(空头发必 403)
|
|
107
|
+
accept = _PARTIAL_ACCEPT if partial else _HTML_ACCEPT
|
|
108
|
+
status, body = await _raw_fetch(url, accept, _version if partial else None, source=source)
|
|
109
|
+
if status in (401, 403):
|
|
110
|
+
if partial: # 版本头是 partial 独有的变量:轮换过就新头原址重试一次
|
|
111
|
+
saved = _version
|
|
112
|
+
await _harvest_version()
|
|
113
|
+
if _version and _version != saved:
|
|
114
|
+
status, body = await _raw_fetch(url, accept, _version, source=source)
|
|
115
|
+
if status:
|
|
116
|
+
raise SourceError(
|
|
117
|
+
t("errors.reddit_status_login", source=source, status=status)
|
|
118
|
+
) from None
|
|
119
|
+
if not partial and (m := _VERSION_RE.search(body)):
|
|
120
|
+
_version = m.group(0) # 服务端把版本织进每个页面:顺手收割,首批 partial 免一次专割
|
|
121
|
+
return body
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
async def fetch_listing(url: str, *, source: str = "Reddit") -> tuple[list[FeedItem], str | None]:
|
|
125
|
+
"""一页列表(页面或片段 URL)-> (FeedItem, 下一页 URL);None = 到底。"""
|
|
126
|
+
html = await _get_html(url, source=source, partial="/svc/" in url)
|
|
127
|
+
return parse_posts_html(html), parse_next_html(html)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
async def fetch_thread_html(permalink: str, *, source: str = "") -> str:
|
|
131
|
+
"""帖子 permalink 页的 SSR HTML(解析见 parse.parse_thread_html)。"""
|
|
132
|
+
return await _get_html(permalink, source=source or t("sources.reddit.post"))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
async def fetch_subreddit(name: str) -> str:
|
|
136
|
+
"""r/{name}/ 页面校验存在性;返回页面自述的 subreddit 名(订阅添加用)。"""
|
|
137
|
+
|
|
138
|
+
html = await _get_html(f"{_API}/r/{name}/", source=f"Reddit r/{name}")
|
|
139
|
+
real = parse_subreddit_page(html)
|
|
140
|
+
if not real:
|
|
141
|
+
raise SourceError(t("errors.reddit_subreddit_gone", name=name))
|
|
142
|
+
return real
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class RedditFeedLoader(FeedLoader):
|
|
146
|
+
"""列表翻页:页面起步、跟随 load-after partial 链;链尽即到底。
|
|
147
|
+
|
|
148
|
+
SSR 页面只内嵌 3-10 帖(站点自己也这样:首屏渲染几帖,余下靠前端
|
|
149
|
+
续拉),一批不足 _MIN_BATCH 就顺着链继续拉到够——用户视角「首屏即
|
|
150
|
+
一屏」,稀疏首屏只在链尽(真没更多)时出现。不持有页签(通道是
|
|
151
|
+
进程级单例,随会话生灭)。
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
def __init__(self, page_url: str, label: str) -> None:
|
|
155
|
+
self._page_url = page_url
|
|
156
|
+
self._label = label
|
|
157
|
+
self.items: list[FeedItem] = []
|
|
158
|
+
self._seen: set[str] = set()
|
|
159
|
+
self._next: str | None = None
|
|
160
|
+
self._started = False
|
|
161
|
+
self._done = False
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def done(self) -> bool:
|
|
165
|
+
return self._done
|
|
166
|
+
|
|
167
|
+
async def load_more(self) -> list[FeedItem]:
|
|
168
|
+
if self._done:
|
|
169
|
+
return []
|
|
170
|
+
added: list[FeedItem] = []
|
|
171
|
+
while not self._done and len(added) < _MIN_BATCH:
|
|
172
|
+
url = self._page_url if not self._started else (self._next or "")
|
|
173
|
+
if not url:
|
|
174
|
+
self._done = True
|
|
175
|
+
break
|
|
176
|
+
self._started = True
|
|
177
|
+
items, nxt = await fetch_listing(url, source=f"Reddit「{self._label}」")
|
|
178
|
+
self._next = nxt
|
|
179
|
+
self._done = nxt is None
|
|
180
|
+
added.extend(dedup_append(self.items, self._seen, items))
|
|
181
|
+
return added
|