feedscope 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. feedscope/__init__.py +3 -0
  2. feedscope/__main__.py +4 -0
  3. feedscope/cli.py +154 -0
  4. feedscope/core/__init__.py +1 -0
  5. feedscope/core/browser.py +367 -0
  6. feedscope/core/config.py +367 -0
  7. feedscope/core/const.py +64 -0
  8. feedscope/core/display.py +62 -0
  9. feedscope/core/http.py +104 -0
  10. feedscope/core/i18n/__init__.py +202 -0
  11. feedscope/core/i18n/en.toml +447 -0
  12. feedscope/core/i18n/zh-CN.toml +444 -0
  13. feedscope/core/media.py +150 -0
  14. feedscope/core/models.py +86 -0
  15. feedscope/core/registry.py +53 -0
  16. feedscope/core/samples.py +68 -0
  17. feedscope/core/source.py +668 -0
  18. feedscope/core/text.py +54 -0
  19. feedscope/core/timefmt.py +53 -0
  20. feedscope/core/tree.py +134 -0
  21. feedscope/doctor.py +54 -0
  22. feedscope/login.py +168 -0
  23. feedscope/probe.py +131 -0
  24. feedscope/report.py +19 -0
  25. feedscope/sources/__init__.py +38 -0
  26. feedscope/sources/hn/__init__.py +66 -0
  27. feedscope/sources/hn/fetch.py +82 -0
  28. feedscope/sources/hn/parse.py +154 -0
  29. feedscope/sources/reddit/__init__.py +178 -0
  30. feedscope/sources/reddit/fetch.py +181 -0
  31. feedscope/sources/reddit/parse.py +318 -0
  32. feedscope/sources/rss/__init__.py +64 -0
  33. feedscope/sources/rss/fetch.py +94 -0
  34. feedscope/sources/rss/parse.py +116 -0
  35. feedscope/sources/telegram/__init__.py +61 -0
  36. feedscope/sources/telegram/fetch.py +79 -0
  37. feedscope/sources/telegram/parse.py +182 -0
  38. feedscope/sources/v2ex/__init__.py +64 -0
  39. feedscope/sources/v2ex/fetch.py +46 -0
  40. feedscope/sources/v2ex/parse.py +116 -0
  41. feedscope/sources/x/__init__.py +68 -0
  42. feedscope/sources/x/fetch.py +490 -0
  43. feedscope/sources/x/parse.py +535 -0
  44. feedscope/sources/zhihu/__init__.py +81 -0
  45. feedscope/sources/zhihu/fetch.py +402 -0
  46. feedscope/sources/zhihu/parse.py +235 -0
  47. feedscope/subscriptions.py +423 -0
  48. feedscope/ui/__init__.py +10 -0
  49. feedscope/ui/account.py +125 -0
  50. feedscope/ui/app.py +105 -0
  51. feedscope/ui/dialogs.py +127 -0
  52. feedscope/ui/feedback.py +64 -0
  53. feedscope/ui/focus_nav.py +113 -0
  54. feedscope/ui/labels.py +38 -0
  55. feedscope/ui/layout.py +66 -0
  56. feedscope/ui/list_render.py +42 -0
  57. feedscope/ui/screens/__init__.py +1 -0
  58. feedscope/ui/screens/answer.py +296 -0
  59. feedscope/ui/screens/detail.py +274 -0
  60. feedscope/ui/screens/language.py +59 -0
  61. feedscope/ui/screens/main.py +736 -0
  62. feedscope/ui/screens/main_setup.py +226 -0
  63. feedscope/ui/screens/reading.py +384 -0
  64. feedscope/ui/screens/source_setup.py +217 -0
  65. feedscope/ui/screens/subscriptions.py +196 -0
  66. feedscope/ui/theme.py +45 -0
  67. feedscope/ui/wrap.py +157 -0
  68. feedscope-0.1.0.dist-info/METADATA +302 -0
  69. feedscope-0.1.0.dist-info/RECORD +72 -0
  70. feedscope-0.1.0.dist-info/WHEEL +4 -0
  71. feedscope-0.1.0.dist-info/entry_points.txt +2 -0
  72. feedscope-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,318 @@
1
+ """解析层:Reddit 网页 HTML(shreddit SSR / faceplate partial)-> FeedItem /
2
+ 帖子与评论树,纯函数不碰网络。
3
+
4
+ 2026-11 起 www 的 .json 遗留端点对第三方全线 403,本源改吃网站自己的
5
+ HTML 通道(请求由常驻 reddit.com 通道页签的真 Chrome 发出,见 fetch.py):
6
+ - 列表:流页面 SSR(首页 / 热门 r/popular / 订阅 r/{sub})+ 翻页跟随
7
+ faceplate-partial slot="load-after" 的 src 链(feeds/*-feed 与
8
+ community-more-posts 两族,src 里带全参数不用自己拼)
9
+ - 详情:帖子 permalink 页 SSR 自带整棵评论树(1.5MB 级,深度大的树被
10
+ 服务端截断,用「另有 N 条未展开」诚实提示,不续拉)
11
+
12
+ 结构映射是 HN 式三层(词汇重映射,UI 零特判),语义与旧 JSON 版一致:
13
+ - <shreddit-post> 开标签属性即帖子全部元数据(id/post-title/score/
14
+ comment-count/subreddit-name/permalink/post-type/content-href)
15
+ - 顶层评论 -> 楼层(Answer,unit「评论」);楼层内回复树按 depth 属性
16
+ 重建 -> 评论层(Comment,unit「回复」)。Comment 模型只有两层而
17
+ Reddit 常嵌四五层:照 HN 拍平——直接子回复做顶层,更深的拍平进
18
+ replies;隔层回复加「@父作者」前缀
19
+ - 正文是结构化 HTML(rtjson 渲染产物),统一走 html_to_md(同 Telegram)
20
+
21
+ Reddit 特有语义:
22
+ - score:楼层与评论都有值
23
+ - [deleted]/[removed]:作者/正文的删除占位(作者栏 [已删除]、正文
24
+ 「(已删除)」;有回复的删除评论保留树形)
25
+ - 广告:promoted 帖不带 t3_ id(或 class 带 promoted),静默跳过
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import re
31
+ from dataclasses import dataclass, field
32
+ from urllib.parse import urljoin
33
+
34
+ from feedscope.core.const import safe_int
35
+ from feedscope.core.display import fmt_count, votes_text
36
+ from feedscope.core.media import attrs as tag_attrs
37
+ from feedscope.core.media import html_to_md, mark
38
+ from feedscope.core.models import Answer, Comment, FeedItem, Floor
39
+ from feedscope.core.timefmt import fmt_iso
40
+ from feedscope.core.i18n import t
41
+ from feedscope.core.tree import deleted_author, deleted_body, TreeAccessors, build_floors
42
+
43
+ SOURCE_ID = "reddit"
44
+
45
+ _SITE = "https://www.reddit.com"
46
+
47
+ _DELETED_BODIES = {"[deleted]", "[removed]"} # Reddit 原生删除态(作者/正文两栏的判定源)
48
+
49
+ # subreddit 名域(字母数字下划线):下限放宽到 2——校验过严拒掉真订阅的
50
+ # 代价(加不上),比过松收进一个假名(页面校验会拦住)更高
51
+ _NAME_RE = re.compile(r"^[A-Za-z0-9_]{2,21}$")
52
+
53
+ # 站内地址里的 /r/<名>/ 段(起点或前面是 / . = 才认,避免误吃普通词)
54
+ _R_PATH_RE = re.compile(r"(?:^|[/=.])r/([A-Za-z0-9_]+)/?")
55
+
56
+ # shreddit 开标签(自闭合形态不出现;\s 排除 shreddit-post-xxx 同前缀标签)
57
+ _POST_TAG_RE = re.compile(r"<shreddit-post\s[^>]*>")
58
+ _COMMENT_TAG_RE = re.compile(r"<shreddit-comment\s[^>]*>")
59
+ _PARTIAL_TAG_RE = re.compile(r"<faceplate-partial\s[^>]*>")
60
+
61
+ _SELF_TYPES = {"text", "self"} # post-type -> kind 的两类映射(其余一律 link)
62
+ _MEDIA_TYPES = {"image", "video", "gallery"}
63
+
64
+
65
+ # ---- 列表 ----
66
+
67
+
68
+ def _post_meta(a: dict[str, str], *, with_author: bool = False) -> str:
69
+ """帖子元信息列「N 点 · N 评 · r/版块(· u/作者)」:列表与详情共用。"""
70
+ parts = [
71
+ t("words.points", n=fmt_count(safe_int(a.get('score')))),
72
+ t("words.comments_short", n=fmt_count(safe_int(a.get('comment-count')))),
73
+ ]
74
+ if subreddit := str(a.get("subreddit-name") or ""):
75
+ parts.append(f"r/{subreddit}")
76
+ if with_author and (author := str(a.get("author") or "")):
77
+ parts.append(f"u/{author}")
78
+ return " · ".join(parts)
79
+
80
+
81
+ def parse_posts_html(html: str) -> list[FeedItem]:
82
+ """页面/片段 HTML -> FeedItem 列表(保持文档序,广告与残条逐条防御跳过)。"""
83
+ items: list[FeedItem] = []
84
+ for m in _POST_TAG_RE.finditer(html):
85
+ a = tag_attrs(m.group(0))
86
+ pid = str(a.get("id") or "")
87
+ title = str(a.get("post-title") or "").strip()
88
+ if not pid.startswith("t3_") or not title:
89
+ continue # 广告(无 t3 id)或残条:跳条不跳流
90
+ if "promoted" in str(a.get("class") or ""):
91
+ continue
92
+ subreddit = str(a.get("subreddit-name") or "")
93
+ permalink = str(a.get("permalink") or "")
94
+ items.append(
95
+ FeedItem(
96
+ source=SOURCE_ID,
97
+ id=pid[3:],
98
+ title=title,
99
+ url=urljoin(_SITE, permalink) if permalink else f"{_SITE}/comments/{pid[3:]}",
100
+ kind="self" if str(a.get("post-type") or "") in _SELF_TYPES else "link",
101
+ meta=_post_meta(a),
102
+ extra={"subreddit": subreddit},
103
+ )
104
+ )
105
+ return items
106
+
107
+
108
+ def parse_next_html(html: str) -> str | None:
109
+ """HTML -> 下一页 partial 的绝对 URL(faceplate-partial slot="load-after" 的 src)。
110
+
111
+ 没有该元素 = 服务端说到底了。src 自带全参数(after/distance/sort/...),
112
+ 直接整串跟走,不重新拼。
113
+ """
114
+ for m in _PARTIAL_TAG_RE.finditer(html):
115
+ a = tag_attrs(m.group(0))
116
+ if a.get("slot") == "load-after" and a.get("src"):
117
+ return urljoin(_SITE, a["src"])
118
+ return None
119
+
120
+
121
+ # ---- 帖子与评论树 ----
122
+
123
+
124
+ @dataclass
125
+ class ThreadDetail:
126
+ """帖子 permalink 页的解析结果:帖子本体 + 全部楼层(含各自的回复树)。"""
127
+
128
+ title: str
129
+ meta: str
130
+ time: str
131
+ description_md: str
132
+ floors: list[Floor] = field(default_factory=list)
133
+
134
+
135
+ @dataclass
136
+ class _CNode:
137
+ """评论树的工作节点:属性 + 正文 + 子节点(depth 属性重建嵌套)。"""
138
+
139
+ attrs: dict[str, str]
140
+ body: str
141
+ children: list[_CNode] = field(default_factory=list)
142
+
143
+ @property
144
+ def author(self) -> str:
145
+ a = str(self.attrs.get("author") or "")
146
+ return a if a and a not in _DELETED_BODIES else deleted_author()
147
+
148
+ @property
149
+ def votes(self) -> str:
150
+ return votes_text(safe_int(self.attrs.get("score")))
151
+
152
+
153
+ _DIV_TOKEN_RE = re.compile(r"<div\b|</div>")
154
+
155
+
156
+ def _slot_inner(segment: str, slot: str) -> str:
157
+ """segment 里第一个 slot=<slot> 的 div 的内层 HTML(按 div 配平找闭合)。
158
+
159
+ shreddit 的正文容器是 slot="comment" / slot="text-body" 的 div,内层
160
+ 是结构化 HTML(rtjson 产物);嵌套 div 靠配平而不是非贪婪匹配。
161
+ """
162
+ m = re.search(rf'<div[^>]*slot="{slot}"[^>]*>', segment)
163
+ if not m:
164
+ return ""
165
+ depth = 1
166
+ for tok in _DIV_TOKEN_RE.finditer(segment, m.end()):
167
+ depth += 1 if tok.group(0) == "<div" else -1
168
+ if depth == 0:
169
+ return segment[m.end() : tok.start()]
170
+ return ""
171
+
172
+
173
+ def _content(node: _CNode) -> str:
174
+ """一条评论的正文:删除占位的兜底(@ 前缀由 core.tree 统一拼装)。"""
175
+ return node.body or deleted_body()
176
+
177
+
178
+ def _comment_of(node: _CNode) -> Comment:
179
+ return Comment(
180
+ author=node.author,
181
+ votes=node.votes,
182
+ content_md=_content(node),
183
+ time=fmt_iso(str(node.attrs.get("created") or "") or None),
184
+ raw_id=str(node.attrs.get("thingid") or "").removeprefix("t1_"),
185
+ )
186
+
187
+
188
+ def _keep(node: _CNode) -> bool:
189
+ """空壳节点(被删且无回复的评论)整支跳过(规则见 core.tree)。"""
190
+ return bool(node.body or node.children)
191
+
192
+
193
+ def _kids_of(node: _CNode) -> list[_CNode]:
194
+ return node.children
195
+
196
+
197
+ def _author_of(node: _CNode) -> str:
198
+ return node.author
199
+
200
+
201
+ # 差异注入:Reddit 的树是 HTML 按 depth 重建的 _CNode(拍平与 @ 规则在 core.tree)
202
+ _ACC = TreeAccessors(keep=_keep, kids_of=_kids_of, author_of=_author_of, comment_of=_comment_of)
203
+
204
+
205
+ def _answer_of(node: _CNode, n: int) -> Answer:
206
+ """楼层本体(core.tree.build_floors 注入):字段来源是站差异,n 为回复数。"""
207
+ cp = str(node.attrs.get("permalink") or "")
208
+ return Answer(
209
+ author=node.author,
210
+ votes=node.votes,
211
+ content_md=_content(node),
212
+ time=fmt_iso(str(node.attrs.get("created") or "") or None),
213
+ url=urljoin(_SITE, cp) if cp.startswith("/") else "",
214
+ comment_count=t("display.replies_n", n=n) if n else "", # 0 条不引导 Enter
215
+ )
216
+
217
+
218
+ def _post_body(attrs: dict[str, str], html: str) -> str:
219
+ """帖子正文:自述帖 HTML 转 markdown;媒体帖记号直达原物;链接帖一行原文。"""
220
+ ptype = str(attrs.get("post-type") or "")
221
+ if ptype in _SELF_TYPES:
222
+ md = html_to_md(_slot_inner(html, "text-body"))
223
+ return md or t("words.no_body")
224
+ href = str(attrs.get("content-href") or "")
225
+ if ptype in _MEDIA_TYPES:
226
+ label = {
227
+ "image": t("media.image"), "video": t("media.video"),
228
+ "gallery": t("media.gallery"),
229
+ }.get(ptype, t("media.generic"))
230
+ target = (
231
+ urljoin(_SITE, href)
232
+ if href
233
+ else urljoin(_SITE, str(attrs.get("permalink") or ""))
234
+ )
235
+ return mark(label, target) if target else ""
236
+ return mark(t("words.original"), urljoin(_SITE, href)) if href else ""
237
+
238
+
239
+ def parse_thread_html(html: str, fallback_title: str = "") -> ThreadDetail:
240
+ """帖子 permalink 页 HTML -> ThreadDetail(野数据防御;评论缺失给空楼层)。"""
241
+ post: dict[str, str] = {}
242
+ m = _POST_TAG_RE.search(html)
243
+ if m:
244
+ post = tag_attrs(m.group(0))
245
+
246
+ # 评论树:开标签按文档序拿属性与正文,depth 属性重建嵌套
247
+ roots: list[_CNode] = []
248
+ stack: list[tuple[int, list[_CNode]]] = []
249
+ tags = list(_COMMENT_TAG_RE.finditer(html))
250
+ for i, tag in enumerate(tags):
251
+ # 末条评论的正文区上界:200KB 足够容纳一条评论的 rtjson(再大只会
252
+ # 白扫),配平逻辑保证越界内容也只是被丢弃而不是误吞
253
+ end = tags[i + 1].start() if i + 1 < len(tags) else min(tag.end() + 200_000, len(html))
254
+ node = _CNode(
255
+ attrs=tag_attrs(tag.group(0)),
256
+ body=html_to_md(_slot_inner(html[tag.end() : end], "comment")),
257
+ )
258
+ depth = max(0, safe_int(node.attrs.get("depth")))
259
+ while stack and stack[-1][0] >= depth:
260
+ stack.pop()
261
+ (stack[-1][1] if stack else roots).append(node)
262
+ stack.append((depth, node.children))
263
+
264
+ # 顶层节点 -> 楼层(keep 过滤 / 计数 / 拍平的骨架在 core.tree.build_floors)
265
+ floors = build_floors(roots, _ACC, answer_of=_answer_of)
266
+ # 截断口:服务端只渲染了前一部分楼层(大帖常见),按 comment-count 诚实提示
267
+ total = safe_int(post.get("comment-count"))
268
+ loaded = sum(1 + len(f.comments) + sum(len(c.replies) for c in f.comments) for f in floors)
269
+ if floors and total > loaded:
270
+ last = floors[-1].answer
271
+ last.comment_count = " · ".join(
272
+ x
273
+ for x in (
274
+ last.comment_count,
275
+ t("words.more_unexpanded", n=fmt_count(total - loaded)),
276
+ )
277
+ if x
278
+ )
279
+ return ThreadDetail(
280
+ title=str(post.get("post-title") or "").strip() or fallback_title,
281
+ meta=_post_meta(post, with_author=True),
282
+ time=fmt_iso(str(post.get("created-timestamp") or "") or None),
283
+ description_md=_post_body(post, html),
284
+ floors=floors,
285
+ )
286
+
287
+
288
+ # ---- 订阅校验 ----
289
+
290
+
291
+ def parse_subreddit_page(html: str) -> str | None:
292
+ """版块页 HTML -> subreddit 名(校验存在性用);不存在(壳页)回 None。
293
+
294
+ subreddit-name 首次出现可能在 400KB+ 深处(页首是大量 head/CSS),
295
+ 全量搜索——600KB 级页面的单次正则扫描代价可忽略。
296
+ """
297
+ m = re.search(r'subreddit-name="([A-Za-z0-9_]+)"', html)
298
+ if m:
299
+ return m.group(1)
300
+ m = re.search(r'subreddit-prefixed-name="r/([A-Za-z0-9_]+)"', html)
301
+ return m.group(1) if m else None
302
+
303
+
304
+ # ---- 订阅输入归一 ----
305
+
306
+
307
+ def parse_subreddit_input(text: str) -> str:
308
+ """四形态归一成裸 subreddit 名(小写):rust / r/rust / /r/rust/ / 站点地址。
309
+
310
+ 认不出回空串(调用方给「没认出」反馈,不在这里猜)。
311
+ """
312
+ s = (text or "").strip()
313
+ if not s:
314
+ return ""
315
+ if m := _R_PATH_RE.search(s):
316
+ return m.group(1).lower()
317
+ s = s.rstrip("/")
318
+ return s.lower() if _NAME_RE.match(s) else ""
@@ -0,0 +1,64 @@
1
+ """RSS 源(rss):通用订阅聚合——「全部」合并时间线 + 每订阅一个信息流。
2
+
3
+ 包内分层(全项目统一的源模板):
4
+ - fetch.py 取数层:httpx 条件请求(ETag/Last-Modified)+ 聚合流的并发编排
5
+ - parse.py 解析层:feedparser -> FeedItem / 详情 markdown,纯函数可测
6
+ - __init__.py 编排层:Source 能力声明与 open_* 分派(订阅型骨架走
7
+ core.SubscribedSource,本文件只剩 RSS 的差异)
8
+
9
+ 与 zhihu/x 的本质差异:feeds 不是代码写死的内置流,而是用户在 sources.toml
10
+ [rss.feeds] 里的订阅表(数据驱动的加法合成);无登录、无评论层;详情读
11
+ feed 自带正文(content 优先、summary 兜底),摘要型源正文短,Ctrl+O 跳浏
12
+ 览器补全。
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from typing import Any, ClassVar
18
+
19
+ from feedscope.core.config import Subscription
20
+ from feedscope.core.media import html_to_md
21
+ from feedscope.core.models import FeedItem
22
+ from feedscope.core.registry import register
23
+ from feedscope.core.source import FeedLoader, SubscribedSource
24
+ from feedscope.core.timefmt import fmt_unix
25
+ from feedscope.sources.rss.fetch import fetch_body, fetch_merged
26
+ from feedscope.sources.rss.parse import SOURCE_ID, parse_feed
27
+
28
+
29
+ @register
30
+ class RssSource(SubscribedSource):
31
+ source_id = SOURCE_ID
32
+ label = "RSS"
33
+ supports_detail = True
34
+ subscribable = True # feeds 来自用户订阅:F2 里 → 进订阅管理屏
35
+ detail_kinds = frozenset({""}) # 全部条目(kind 空)Enter 都进终端详情
36
+ detail_unit_label = "units.body" # 无楼层,详情屏尾行与 probe 用这个词
37
+
38
+ all_feed: ClassVar[str] = "all" # 聚合流名(「全部」),feeds 首位
39
+ sub_noun: ClassVar[str] = "units.subscription"
40
+ reserved_feed_keys = (all_feed,)
41
+ _meta_extra_key = "feed_label" # 详情元信息首段 = 订阅显示名
42
+
43
+ async def _fetch_merged(
44
+ self, feeds: list[Subscription]
45
+ ) -> tuple[list[FeedItem], list[str]]:
46
+ return await fetch_merged(feeds)
47
+
48
+ async def _open_one(self, feed: Subscription) -> FeedLoader:
49
+ items = parse_feed(
50
+ await fetch_body(feed),
51
+ feed_key=feed.key,
52
+ feed_label=feed.label,
53
+ all_view=False,
54
+ fallback_url=feed.url,
55
+ )
56
+ # 0 条 = 大概率源坏了(改版/地址失效),切片出口统一报;单个
57
+ # feed 也可能几百条,切片上屏防一次性挂载冻结
58
+ return self.sliced_feed(items, feed.label)
59
+
60
+ def _detail_time(self, extra: dict[str, Any]) -> str:
61
+ return fmt_unix(extra.get("published") or None)
62
+
63
+ def _detail_body(self, extra: dict[str, Any]) -> str:
64
+ return html_to_md(str(extra.get("content_html") or ""))
@@ -0,0 +1,94 @@
1
+ """取数层:订阅抓取(httpx 条件请求)+「全部」聚合流(共享编排)。
2
+
3
+ - 条件请求(ETag / Last-Modified):模块级缓存按 URL 记验证器与上次响应体,
4
+ 304 直接回放缓存正文——F5 反复刷新对源友好,也省流量。缓存只在进程内:
5
+ 重启后第一次仍是无条件 GET,语义最简单也够用(阅读器不是离线仓库)。
6
+ - 聚合编排(并发/失败隔离/倒序合并)见 core.source.merge_feeds;RSS 的差异
7
+ 是聚合视图解析、跨订阅 URL 去重、按发布时间(unix 秒)倒序。
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+
14
+ from feedscope.core import http
15
+ from feedscope.core.config import Subscription
16
+ from feedscope.core.models import FeedItem
17
+ from feedscope.core.source import SourceError, merge_feeds
18
+ from feedscope.core.i18n import t
19
+ from feedscope.sources.rss.parse import parse_feed
20
+
21
+ # RSS 专用 Accept:部分源对浏览器 UA + */* 的组合反而回 HTML 页,声明清楚要 feed
22
+ _ACCEPT = (
23
+ "application/rss+xml, application/atom+xml, application/feed+json, "
24
+ "application/xml, text/xml, */*"
25
+ )
26
+
27
+
28
+ @dataclass
29
+ class _CondEntry:
30
+ """一个订阅的条件请求缓存(验证器 + 上次响应体)。"""
31
+
32
+ etag: str | None
33
+ last_modified: str | None
34
+ body: bytes
35
+
36
+
37
+ _CACHE: dict[str, _CondEntry] = {}
38
+
39
+
40
+ def _conditional_headers(cached: _CondEntry | None) -> dict[str, str]:
41
+ headers = {"Accept": _ACCEPT}
42
+ if cached is not None:
43
+ if cached.etag:
44
+ headers["If-None-Match"] = cached.etag
45
+ if cached.last_modified:
46
+ headers["If-Modified-Since"] = cached.last_modified
47
+ return headers
48
+
49
+
50
+ async def fetch_body(feed: Subscription) -> bytes:
51
+ """抓一个订阅的 XML 字节;304 回放缓存体,200 记验证器并更新缓存。"""
52
+ cached = _CACHE.get(feed.url)
53
+ r = await http.get(
54
+ feed.url,
55
+ source=t("sources.rss.with_label", label=feed.label),
56
+ headers=_conditional_headers(cached),
57
+ allow_304=True,
58
+ )
59
+ if r.status_code == 304:
60
+ if cached is not None:
61
+ return cached.body
62
+ # 无缓存却 304(服务器不合规):去掉条件头再来一次,还 304 才是真没辙
63
+ r = await http.get(
64
+ feed.url,
65
+ source=t("sources.rss.with_label", label=feed.label),
66
+ headers={"Accept": _ACCEPT},
67
+ )
68
+ if r.status_code == 304:
69
+ raise SourceError(t("errors.rss_304_no_cache", label=feed.label))
70
+ _CACHE[feed.url] = _CondEntry(
71
+ etag=r.headers.get("ETag"), last_modified=r.headers.get("Last-Modified"), body=r.content
72
+ )
73
+ return r.content
74
+
75
+
76
+ async def fetch_merged(feeds: list[Subscription]) -> tuple[list[FeedItem], list[str]]:
77
+ """全部订阅并发拉取并聚合成一条时间线:(合并条目, 失败订阅的短消息列表)。
78
+
79
+ 编排(并发/失败隔离/倒序合并)见 core.source.merge_feeds;RSS 的差异:
80
+ 聚合视图解析(all_view,订阅名进元信息列)、跨订阅 URL 去重(同一 URL
81
+ 多源覆盖只留书写顺序靠前的一份)、按发布时间(unix 秒)倒序。
82
+ """
83
+
84
+ async def one(feed: Subscription) -> list[FeedItem]:
85
+ body = await fetch_body(feed)
86
+ return parse_feed(body, feed_key=feed.key, feed_label=feed.label, all_view=True)
87
+
88
+ return await merge_feeds(
89
+ feeds,
90
+ one,
91
+ error_label=lambda f: f.label,
92
+ sort_key=lambda it: float(it.extra.get("published") or 0.0),
93
+ dedup_urls=True,
94
+ )
@@ -0,0 +1,116 @@
1
+ """解析层:feedparser 结果 -> FeedItem / 详情 markdown,纯函数不碰网络。
2
+
3
+ RSS/Atom 野源格式千奇百怪,容错原则「能救则救、救不了就丢条不丢流」:
4
+ 标题带 HTML、缺 author、缺发布时间都只是字段级缺失;guid 与 link 双缺才
5
+ 整条跳过(无法寻址,进详情 / 跳浏览器都没有落点)。
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import calendar
11
+ import html as html_mod
12
+ import re
13
+ from time import struct_time
14
+ from urllib.parse import urljoin
15
+
16
+ import feedparser
17
+
18
+ from feedscope.core.models import FeedItem
19
+ from feedscope.core.i18n import t
20
+ from feedscope.core.text import strip_zwsp
21
+
22
+ SOURCE_ID = "rss"
23
+
24
+ # 标题/作者里的残余标签(<b>标题</b>、署名 <a> 之类),清理成纯文本
25
+ _TAGS = re.compile(r"<[^>]*>")
26
+ # 首页 HTML 里的 <link> 声明(自动发现 feed 用)
27
+ _LINK_TAG = re.compile(r"<link\b[^>]*>", re.IGNORECASE)
28
+ _HREF = re.compile(r"""href=(["'])(.*?)\1""", re.IGNORECASE)
29
+ _FEED_TYPES = ("rss+xml", "atom+xml", "feed+json", "feed+xml")
30
+
31
+
32
+ def _clean(text: object) -> str:
33
+ """字段级清洗:去标签、还原实体、去零宽;None / 非 str 回空串。"""
34
+ if not isinstance(text, str):
35
+ return ""
36
+ return strip_zwsp(html_mod.unescape(_TAGS.sub("", text)))
37
+
38
+
39
+ def _epoch(st: object) -> float:
40
+ """feedparser 的 *_parsed(struct_time,UTC)-> unix 秒;缺省 0(排序沉底)。"""
41
+ return float(calendar.timegm(st)) if isinstance(st, struct_time) and st.tm_year > 1 else 0.0
42
+
43
+
44
+ def _content_html(entry: dict) -> str:
45
+ """正文 HTML:优先 content(多为全文),退 summary(多为摘要)。"""
46
+ content = entry.get("content")
47
+ if isinstance(content, list) and content:
48
+ first = content[0]
49
+ if isinstance(first, dict) and isinstance(first.get("value"), str):
50
+ return first["value"]
51
+ summary = entry.get("summary")
52
+ return summary if isinstance(summary, str) else ""
53
+
54
+
55
+ def probe(body: bytes | str) -> tuple[str, int]:
56
+ """(feed 自述标题, 条目数):订阅管理判断「这是不是 feed」与取默认显示名。"""
57
+ parsed = feedparser.parse(body)
58
+ title = _clean(parsed.feed.get("title")) if getattr(parsed, "feed", None) else ""
59
+ return title, len(parsed.entries)
60
+
61
+
62
+ def discover_feed_link(html: str, base_url: str) -> str:
63
+ """首页 HTML -> feed 地址:<link rel="alternate" type="…rss/atom…">,
64
+ 相对 href 以 base_url 绝对化;找不到回空串。"""
65
+ for tag in _LINK_TAG.findall(html):
66
+ low = tag.lower()
67
+ if "alternate" not in low or not any(t in low for t in _FEED_TYPES):
68
+ continue
69
+ m = _HREF.search(tag)
70
+ if m and m.group(2):
71
+ return urljoin(base_url, m.group(2))
72
+ return ""
73
+
74
+
75
+ def parse_feed(
76
+ body: bytes | str,
77
+ *,
78
+ feed_key: str,
79
+ feed_label: str,
80
+ all_view: bool,
81
+ fallback_url: str = "",
82
+ ) -> list[FeedItem]:
83
+ """一个订阅的 XML -> FeedItem 列表(保持源内顺序,通常已是倒序)。
84
+
85
+ all_view 决定元信息列内容:聚合「全部」里必须能认出条目来自哪个订阅
86
+ (订阅名打头),单订阅视图里订阅名是噪音、只留作者。extra 带 detail
87
+ 所需的全部字段(feed 自带正文,进详情不重新抓)。
88
+ """
89
+ parsed = feedparser.parse(body)
90
+ items: list[FeedItem] = []
91
+ for entry in parsed.entries:
92
+ link = entry.get("link") or fallback_url
93
+ guid = entry.get("id")
94
+ if not guid and not link:
95
+ continue # 双缺无法寻址
96
+ author = _clean(entry.get("author"))
97
+ meta_parts = ([feed_label] if all_view else []) + ([author] if author else [])
98
+ items.append(
99
+ FeedItem(
100
+ source=SOURCE_ID,
101
+ id=f"{feed_key}/{guid or link}",
102
+ title=_clean(entry.get("title")) or t("words.no_title"),
103
+ url=link,
104
+ meta=" · ".join(meta_parts),
105
+ extra={
106
+ "feed_key": feed_key,
107
+ "feed_label": feed_label,
108
+ "author": author,
109
+ "published": _epoch(
110
+ entry.get("published_parsed") or entry.get("updated_parsed")
111
+ ),
112
+ "content_html": _content_html(entry),
113
+ },
114
+ )
115
+ )
116
+ return items
@@ -0,0 +1,61 @@
1
+ """Telegram 频道源(telegram):纯订阅聚合——「全部」合并时间线 + 每频道一流。
2
+
3
+ 包内分层(全项目统一的源模板):
4
+ - fetch.py 取数层:t.me/s 预览页(免登录 SSR HTML)+ 聚合流的并发编排
5
+ - parse.py 解析层:预览 HTML -> FeedItem / 正文 markdown,纯函数可测
6
+ - __init__.py 编排层:Source 能力声明与 open_* 分派(订阅型骨架走
7
+ core.SubscribedSource,本文件只剩 Telegram 的差异)
8
+
9
+ 与 RSS 完全同形(代码级近亲):feeds 来自 [telegram.feeds] 订阅表(加法
10
+ 合成),无登录、无评论层;详情读列表自带正文(extra 透传),进详情零请求,
11
+ 摘要外没有的部分 Ctrl+O 跳浏览器。频道帖的正文是行内 HTML,走 html_to_md
12
+ 共用管线;图片/视频换 [图片]/[视频] 记号。翻页走 ?before= 游标(单频道流);
13
+ 「全部」聚合流一次并发拉全、内存切片上屏(SliceFeedLoader)。
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from typing import Any, ClassVar
19
+
20
+ from feedscope.core.config import Subscription
21
+ from feedscope.core.i18n import t
22
+ from feedscope.core.models import FeedItem
23
+ from feedscope.core.registry import register
24
+ from feedscope.core.source import FeedLoader, SourceError, SubscribedSource
25
+ from feedscope.sources.telegram.fetch import TelegramFeedLoader, fetch_merged
26
+ from feedscope.sources.telegram.parse import SOURCE_ID, display_time
27
+
28
+
29
+ @register
30
+ class TelegramSource(SubscribedSource):
31
+ source_id = SOURCE_ID
32
+ label = "Telegram"
33
+ supports_detail = True
34
+ subscribable = True # feeds 来自用户订阅:F2 里 → 进订阅管理屏
35
+ supports_comments = False # 频道无评论层(讨论组在另一维度,预览页拿不到)
36
+ detail_kinds = frozenset({""}) # 全部条目(kind 空)Enter 都进终端详情
37
+ detail_unit_label = "units.body" # 无楼层,详情屏尾行与 probe 用这个词(同 RSS)
38
+
39
+ all_feed: ClassVar[str] = "all" # 聚合流名(「全部」),保留字:订阅 key 与它同名会遮住聚合流
40
+ sub_noun: ClassVar[str] = "units.channel"
41
+ reserved_feed_keys = (all_feed,)
42
+ _meta_extra_key = "channel" # 详情元信息首段 = 频道名
43
+
44
+ async def _fetch_merged(
45
+ self, feeds: list[Subscription]
46
+ ) -> tuple[list[FeedItem], list[str]]:
47
+ return await fetch_merged(feeds)
48
+
49
+ async def _open_one(self, feed: Subscription) -> FeedLoader:
50
+ loader = TelegramFeedLoader(feed.key, feed.label)
51
+ await loader.load_more() # 首批
52
+ # 0 条大概率是频道不存在/为空/关闭了 Web 预览,不是站点改版——给专属指引
53
+ if not loader.items:
54
+ raise SourceError(t("errors.telegram_no_items", key=feed.key))
55
+ return loader
56
+
57
+ def _detail_time(self, extra: dict[str, Any]) -> str:
58
+ return display_time(str(extra.get("published") or ""))
59
+
60
+ def _detail_body(self, extra: dict[str, Any]) -> str:
61
+ return str(extra.get("content_md") or "")