rss-mcp-server 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rss-mcp-server might be problematic. Click here for more details.
- rss_mcp/__init__.py +3 -0
- rss_mcp/__main__.py +4 -0
- rss_mcp/config.py +28 -0
- rss_mcp/db.py +74 -0
- rss_mcp/feed_client.py +192 -0
- rss_mcp/models.py +44 -0
- rss_mcp/opml.py +58 -0
- rss_mcp/readability_fetch.py +137 -0
- rss_mcp/server.py +361 -0
- rss_mcp/store.py +361 -0
- rss_mcp_server-0.1.0.dist-info/METADATA +141 -0
- rss_mcp_server-0.1.0.dist-info/RECORD +15 -0
- rss_mcp_server-0.1.0.dist-info/WHEEL +4 -0
- rss_mcp_server-0.1.0.dist-info/entry_points.txt +3 -0
- rss_mcp_server-0.1.0.dist-info/licenses/LICENSE +21 -0
rss_mcp/__init__.py
ADDED
rss_mcp/__main__.py
ADDED
rss_mcp/config.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
_DEFAULT_USER_AGENT = "rss-mcp/0.1 (+https://github.com/huangcheng/rss-mcp)"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Settings:
|
|
12
|
+
db_path: Path
|
|
13
|
+
user_agent: str
|
|
14
|
+
default_limit: int = 20
|
|
15
|
+
max_limit: int = 100
|
|
16
|
+
fetch_entry_cap: int = 50
|
|
17
|
+
auth_token: str | None = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_settings() -> Settings:
|
|
21
|
+
raw_db = os.environ.get("RSS_MCP_DB")
|
|
22
|
+
if raw_db and raw_db.strip():
|
|
23
|
+
db_path = Path(raw_db.strip()).expanduser()
|
|
24
|
+
else:
|
|
25
|
+
db_path = Path.home() / ".local" / "share" / "rss-mcp" / "rss.db"
|
|
26
|
+
user_agent = os.environ.get("RSS_MCP_USER_AGENT") or _DEFAULT_USER_AGENT
|
|
27
|
+
auth_token = (os.environ.get("RSS_MCP_AUTH_TOKEN") or "").strip() or None
|
|
28
|
+
return Settings(db_path=db_path, user_agent=user_agent, auth_token=auth_token)
|
rss_mcp/db.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
_SCHEMA = """
|
|
7
|
+
CREATE TABLE IF NOT EXISTS subscriptions (
|
|
8
|
+
id INTEGER PRIMARY KEY,
|
|
9
|
+
url TEXT NOT NULL UNIQUE,
|
|
10
|
+
title TEXT,
|
|
11
|
+
tags TEXT,
|
|
12
|
+
etag TEXT,
|
|
13
|
+
last_modified TEXT,
|
|
14
|
+
last_fetched_at TEXT,
|
|
15
|
+
last_error TEXT,
|
|
16
|
+
created_at TEXT NOT NULL
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
CREATE TABLE IF NOT EXISTS entries (
|
|
20
|
+
id INTEGER PRIMARY KEY,
|
|
21
|
+
subscription_id INTEGER NOT NULL REFERENCES subscriptions(id) ON DELETE CASCADE,
|
|
22
|
+
guid TEXT NOT NULL,
|
|
23
|
+
url TEXT,
|
|
24
|
+
title TEXT,
|
|
25
|
+
author TEXT,
|
|
26
|
+
published_at TEXT,
|
|
27
|
+
summary TEXT,
|
|
28
|
+
content TEXT,
|
|
29
|
+
fetched_at TEXT NOT NULL,
|
|
30
|
+
UNIQUE (subscription_id, guid)
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
CREATE TABLE IF NOT EXISTS entry_state (
|
|
34
|
+
entry_id INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
|
|
35
|
+
read_at TEXT,
|
|
36
|
+
starred INTEGER NOT NULL DEFAULT 0
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
|
|
40
|
+
title,
|
|
41
|
+
summary,
|
|
42
|
+
content,
|
|
43
|
+
content='entries',
|
|
44
|
+
content_rowid='id'
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
CREATE TRIGGER IF NOT EXISTS entries_fts_ai AFTER INSERT ON entries BEGIN
|
|
48
|
+
INSERT INTO entries_fts(rowid, title, summary, content)
|
|
49
|
+
VALUES (new.id, new.title, new.summary, new.content);
|
|
50
|
+
END;
|
|
51
|
+
|
|
52
|
+
CREATE TRIGGER IF NOT EXISTS entries_fts_ad AFTER DELETE ON entries BEGIN
|
|
53
|
+
INSERT INTO entries_fts(entries_fts, rowid, title, summary, content)
|
|
54
|
+
VALUES ('delete', old.id, old.title, old.summary, old.content);
|
|
55
|
+
END;
|
|
56
|
+
|
|
57
|
+
CREATE TRIGGER IF NOT EXISTS entries_fts_au AFTER UPDATE ON entries BEGIN
|
|
58
|
+
INSERT INTO entries_fts(entries_fts, rowid, title, summary, content)
|
|
59
|
+
VALUES ('delete', old.id, old.title, old.summary, old.content);
|
|
60
|
+
INSERT INTO entries_fts(rowid, title, summary, content)
|
|
61
|
+
VALUES (new.id, new.title, new.summary, new.content);
|
|
62
|
+
END;
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def connect(db_path: Path) -> sqlite3.Connection:
|
|
67
|
+
conn = sqlite3.connect(Path(db_path))
|
|
68
|
+
conn.row_factory = sqlite3.Row
|
|
69
|
+
conn.execute("PRAGMA foreign_keys = ON")
|
|
70
|
+
return conn
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def init_db(conn: sqlite3.Connection) -> None:
|
|
74
|
+
conn.executescript(_SCHEMA)
|
rss_mcp/feed_client.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import io
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from typing import Literal
|
|
8
|
+
from urllib.parse import urljoin
|
|
9
|
+
|
|
10
|
+
import feedparser
|
|
11
|
+
import httpx
|
|
12
|
+
|
|
13
|
+
from rss_mcp.models import EntryInput
|
|
14
|
+
|
|
15
|
+
MAX_FEED_BYTES = 5 * 1024 * 1024
|
|
16
|
+
_READ_CHUNK = 64 * 1024
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class FetchResult:
|
|
21
|
+
status: Literal["ok", "not_modified", "error"]
|
|
22
|
+
entries: list[EntryInput]
|
|
23
|
+
etag: str | None
|
|
24
|
+
last_modified: str | None
|
|
25
|
+
feed_title: str | None
|
|
26
|
+
error: str | None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _text(value: object) -> str | None:
|
|
30
|
+
if not isinstance(value, str):
|
|
31
|
+
return None
|
|
32
|
+
stripped = value.strip()
|
|
33
|
+
return stripped or None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _header(response: httpx.Response, name: str) -> str | None:
|
|
37
|
+
return _text(response.headers.get(name))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _error(message: str) -> FetchResult:
|
|
41
|
+
return FetchResult(
|
|
42
|
+
status="error",
|
|
43
|
+
entries=[],
|
|
44
|
+
etag=None,
|
|
45
|
+
last_modified=None,
|
|
46
|
+
feed_title=None,
|
|
47
|
+
error=message,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _published_at(entry: feedparser.FeedParserDict) -> str | None:
|
|
52
|
+
parsed = entry.get("published_parsed") or entry.get("updated_parsed")
|
|
53
|
+
if not parsed:
|
|
54
|
+
return None
|
|
55
|
+
try:
|
|
56
|
+
moment = datetime(*parsed[:6], tzinfo=timezone.utc)
|
|
57
|
+
except (TypeError, ValueError, OverflowError):
|
|
58
|
+
return None
|
|
59
|
+
return moment.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _content(entry: feedparser.FeedParserDict) -> str | None:
|
|
63
|
+
blocks = entry.get("content") or []
|
|
64
|
+
for block in blocks:
|
|
65
|
+
if not isinstance(block, dict):
|
|
66
|
+
continue
|
|
67
|
+
text = _text(block.get("value"))
|
|
68
|
+
if text is not None:
|
|
69
|
+
return text
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _resolve_url(value: object, base_url: str) -> str | None:
|
|
74
|
+
raw = _text(value)
|
|
75
|
+
if raw is None or not base_url:
|
|
76
|
+
return raw
|
|
77
|
+
return _text(urljoin(base_url, raw))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _entry(entry: feedparser.FeedParserDict, base_url: str) -> EntryInput:
|
|
81
|
+
return EntryInput(
|
|
82
|
+
guid=_text(entry.get("id")),
|
|
83
|
+
url=_resolve_url(entry.get("link"), base_url),
|
|
84
|
+
title=_text(entry.get("title")),
|
|
85
|
+
author=_text(entry.get("author")),
|
|
86
|
+
published_at=_published_at(entry),
|
|
87
|
+
summary=_text(entry.get("summary")),
|
|
88
|
+
content=_content(entry),
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _cap(entries: list[EntryInput], entry_cap: int) -> list[EntryInput]:
|
|
93
|
+
if entry_cap <= 0:
|
|
94
|
+
return []
|
|
95
|
+
return entries[:entry_cap]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _from_parsed(
|
|
99
|
+
parsed: feedparser.FeedParserDict,
|
|
100
|
+
*,
|
|
101
|
+
base_url: str,
|
|
102
|
+
etag: str | None,
|
|
103
|
+
last_modified: str | None,
|
|
104
|
+
entry_cap: int,
|
|
105
|
+
) -> FetchResult:
|
|
106
|
+
version = parsed.get("version") or ""
|
|
107
|
+
if not isinstance(version, str):
|
|
108
|
+
version = ""
|
|
109
|
+
raw_entries = list(parsed.entries)
|
|
110
|
+
if not raw_entries and not version.strip():
|
|
111
|
+
exc = parsed.get("bozo_exception")
|
|
112
|
+
detail = str(exc).strip() if exc else "unrecognized feed"
|
|
113
|
+
return _error(f"unable to parse feed: {detail}")
|
|
114
|
+
return FetchResult(
|
|
115
|
+
status="ok",
|
|
116
|
+
entries=_cap(
|
|
117
|
+
[_entry(entry, base_url) for entry in raw_entries],
|
|
118
|
+
entry_cap,
|
|
119
|
+
),
|
|
120
|
+
etag=etag,
|
|
121
|
+
last_modified=last_modified,
|
|
122
|
+
feed_title=_text(parsed.feed.get("title")),
|
|
123
|
+
error=None,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
async def _read_limited(response: httpx.Response) -> bytes | None:
|
|
128
|
+
chunks: list[bytes] = []
|
|
129
|
+
total = 0
|
|
130
|
+
async for chunk in response.aiter_bytes(chunk_size=_READ_CHUNK):
|
|
131
|
+
total += len(chunk)
|
|
132
|
+
if total > MAX_FEED_BYTES:
|
|
133
|
+
return None
|
|
134
|
+
chunks.append(chunk)
|
|
135
|
+
return b"".join(chunks)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def fetch_feed(
|
|
139
|
+
url: str,
|
|
140
|
+
*,
|
|
141
|
+
etag: str | None,
|
|
142
|
+
last_modified: str | None,
|
|
143
|
+
user_agent: str,
|
|
144
|
+
entry_cap: int,
|
|
145
|
+
timeout: float = 30.0,
|
|
146
|
+
) -> FetchResult:
|
|
147
|
+
sent_etag = _text(etag)
|
|
148
|
+
sent_modified = _text(last_modified)
|
|
149
|
+
headers: dict[str, str] = {}
|
|
150
|
+
if sent_etag is not None:
|
|
151
|
+
headers["If-None-Match"] = sent_etag
|
|
152
|
+
if sent_modified is not None:
|
|
153
|
+
headers["If-Modified-Since"] = sent_modified
|
|
154
|
+
try:
|
|
155
|
+
async with httpx.AsyncClient(
|
|
156
|
+
timeout=timeout,
|
|
157
|
+
follow_redirects=True,
|
|
158
|
+
headers={"User-Agent": user_agent},
|
|
159
|
+
) as client:
|
|
160
|
+
async with client.stream("GET", url, headers=headers) as response:
|
|
161
|
+
response_etag = _header(response, "etag")
|
|
162
|
+
response_modified = _header(response, "last-modified")
|
|
163
|
+
if response.status_code == 304:
|
|
164
|
+
return FetchResult(
|
|
165
|
+
status="not_modified",
|
|
166
|
+
entries=[],
|
|
167
|
+
etag=response_etag or sent_etag,
|
|
168
|
+
last_modified=response_modified or sent_modified,
|
|
169
|
+
feed_title=None,
|
|
170
|
+
error=None,
|
|
171
|
+
)
|
|
172
|
+
if response.status_code != 200:
|
|
173
|
+
return _error(f"HTTP {response.status_code}")
|
|
174
|
+
final_url = str(response.url)
|
|
175
|
+
body = await _read_limited(response)
|
|
176
|
+
except (httpx.HTTPError, httpx.InvalidURL, UnicodeEncodeError) as exc:
|
|
177
|
+
message = str(exc).strip() or exc.__class__.__name__
|
|
178
|
+
return _error(message)
|
|
179
|
+
if body is None:
|
|
180
|
+
return _error(f"response exceeds {MAX_FEED_BYTES} bytes")
|
|
181
|
+
try:
|
|
182
|
+
# BytesIO: feedparser.parse(bytes) opens a body that is an existing path.
|
|
183
|
+
parsed = await asyncio.to_thread(feedparser.parse, io.BytesIO(body))
|
|
184
|
+
except Exception as exc:
|
|
185
|
+
return _error(f"unable to parse feed: {exc}")
|
|
186
|
+
return _from_parsed(
|
|
187
|
+
parsed,
|
|
188
|
+
base_url=final_url,
|
|
189
|
+
etag=response_etag,
|
|
190
|
+
last_modified=response_modified,
|
|
191
|
+
entry_cap=entry_cap,
|
|
192
|
+
)
|
rss_mcp/models.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class Subscription:
|
|
8
|
+
id: int
|
|
9
|
+
url: str
|
|
10
|
+
title: str | None
|
|
11
|
+
tags: list[str]
|
|
12
|
+
etag: str | None
|
|
13
|
+
last_modified: str | None
|
|
14
|
+
last_fetched_at: str | None
|
|
15
|
+
last_error: str | None
|
|
16
|
+
created_at: str
|
|
17
|
+
unread_count: int = 0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class EntryInput:
|
|
22
|
+
guid: str | None = None
|
|
23
|
+
url: str | None = None
|
|
24
|
+
title: str | None = None
|
|
25
|
+
author: str | None = None
|
|
26
|
+
published_at: str | None = None
|
|
27
|
+
summary: str | None = None
|
|
28
|
+
content: str | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class Entry:
|
|
33
|
+
id: int
|
|
34
|
+
subscription_id: int
|
|
35
|
+
guid: str
|
|
36
|
+
url: str | None
|
|
37
|
+
title: str | None
|
|
38
|
+
author: str | None
|
|
39
|
+
published_at: str | None
|
|
40
|
+
summary: str | None
|
|
41
|
+
content: str | None
|
|
42
|
+
fetched_at: str
|
|
43
|
+
read_at: str | None
|
|
44
|
+
starred: bool
|
rss_mcp/opml.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import xml.etree.ElementTree as ET
|
|
4
|
+
|
|
5
|
+
from rss_mcp.models import Subscription
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _local(tag: str) -> str:
|
|
9
|
+
if tag.startswith("{"):
|
|
10
|
+
return tag.rsplit("}", 1)[-1]
|
|
11
|
+
return tag
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _attr(element: ET.Element, name: str) -> str | None:
|
|
15
|
+
value = element.get(name)
|
|
16
|
+
if not isinstance(value, str):
|
|
17
|
+
return None
|
|
18
|
+
stripped = value.strip()
|
|
19
|
+
return stripped or None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _title(element: ET.Element, url: str) -> str | None:
|
|
23
|
+
title = _attr(element, "title")
|
|
24
|
+
if title is not None:
|
|
25
|
+
return title
|
|
26
|
+
text = _attr(element, "text")
|
|
27
|
+
if text is None or text == url:
|
|
28
|
+
return None
|
|
29
|
+
return text
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def parse_opml(xml: str) -> list[tuple[str, str | None]]:
|
|
33
|
+
root = ET.fromstring(xml)
|
|
34
|
+
feeds: list[tuple[str, str | None]] = []
|
|
35
|
+
for element in root.iter():
|
|
36
|
+
if _local(element.tag) != "outline":
|
|
37
|
+
continue
|
|
38
|
+
url = _attr(element, "xmlUrl")
|
|
39
|
+
if url is None:
|
|
40
|
+
continue
|
|
41
|
+
feeds.append((url, _title(element, url)))
|
|
42
|
+
return feeds
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def build_opml(subs: list[Subscription]) -> str:
|
|
46
|
+
opml = ET.Element("opml", {"version": "2.0"})
|
|
47
|
+
head = ET.SubElement(opml, "head")
|
|
48
|
+
ET.SubElement(head, "title").text = "rss-mcp subscriptions"
|
|
49
|
+
body = ET.SubElement(opml, "body")
|
|
50
|
+
for sub in subs:
|
|
51
|
+
url = sub.url.strip()
|
|
52
|
+
title = (sub.title or "").strip() or None
|
|
53
|
+
attrs = {"type": "rss", "xmlUrl": url, "text": title or url}
|
|
54
|
+
if title is not None:
|
|
55
|
+
attrs["title"] = title
|
|
56
|
+
ET.SubElement(body, "outline", attrs)
|
|
57
|
+
ET.indent(opml)
|
|
58
|
+
return ET.tostring(opml, encoding="unicode", xml_declaration=True)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from lxml import html as lxml_html
|
|
7
|
+
from readability import Document
|
|
8
|
+
|
|
9
|
+
MAX_ARTICLE_BYTES = 5 * 1024 * 1024
|
|
10
|
+
_WHITESPACE = re.compile(r"[^\S\n]+")
|
|
11
|
+
_BLANK_LINES = re.compile(r"\n{3,}")
|
|
12
|
+
_BLOCK_TAGS = frozenset(
|
|
13
|
+
{
|
|
14
|
+
"address",
|
|
15
|
+
"article",
|
|
16
|
+
"aside",
|
|
17
|
+
"blockquote",
|
|
18
|
+
"br",
|
|
19
|
+
"caption",
|
|
20
|
+
"dd",
|
|
21
|
+
"div",
|
|
22
|
+
"dl",
|
|
23
|
+
"dt",
|
|
24
|
+
"figcaption",
|
|
25
|
+
"figure",
|
|
26
|
+
"footer",
|
|
27
|
+
"h1",
|
|
28
|
+
"h2",
|
|
29
|
+
"h3",
|
|
30
|
+
"h4",
|
|
31
|
+
"h5",
|
|
32
|
+
"h6",
|
|
33
|
+
"header",
|
|
34
|
+
"hr",
|
|
35
|
+
"li",
|
|
36
|
+
"main",
|
|
37
|
+
"ol",
|
|
38
|
+
"p",
|
|
39
|
+
"pre",
|
|
40
|
+
"section",
|
|
41
|
+
"table",
|
|
42
|
+
"tbody",
|
|
43
|
+
"td",
|
|
44
|
+
"tfoot",
|
|
45
|
+
"th",
|
|
46
|
+
"thead",
|
|
47
|
+
"tr",
|
|
48
|
+
"ul",
|
|
49
|
+
}
|
|
50
|
+
)
|
|
51
|
+
_HTML_TYPES = frozenset(
|
|
52
|
+
{
|
|
53
|
+
"text/html",
|
|
54
|
+
"text/plain",
|
|
55
|
+
"application/xhtml+xml",
|
|
56
|
+
"application/xml",
|
|
57
|
+
"text/xml",
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _media_type(response: httpx.Response) -> str | None:
|
|
63
|
+
raw = response.headers.get("content-type")
|
|
64
|
+
if not isinstance(raw, str) or not raw.strip():
|
|
65
|
+
return None
|
|
66
|
+
return raw.split(";", 1)[0].strip().lower() or None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _allows_extract(response: httpx.Response) -> bool:
|
|
70
|
+
media = _media_type(response)
|
|
71
|
+
if media is None:
|
|
72
|
+
return True
|
|
73
|
+
return media in _HTML_TYPES or media.endswith("+xml")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _looks_textual(body: bytes) -> bool:
|
|
77
|
+
sample = body[:4096]
|
|
78
|
+
if sample.startswith((b"\xff\xfe", b"\xfe\xff")):
|
|
79
|
+
return True
|
|
80
|
+
return b"\x00" not in sample
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def _read_limited(response: httpx.Response) -> bytes | None:
|
|
84
|
+
chunks: list[bytes] = []
|
|
85
|
+
total = 0
|
|
86
|
+
async for chunk in response.aiter_bytes():
|
|
87
|
+
total += len(chunk)
|
|
88
|
+
if total > MAX_ARTICLE_BYTES:
|
|
89
|
+
return None
|
|
90
|
+
chunks.append(chunk)
|
|
91
|
+
return b"".join(chunks)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _separate_blocks(root) -> None:
|
|
95
|
+
for element in root.iter():
|
|
96
|
+
tag = element.tag
|
|
97
|
+
if not isinstance(tag, str) or tag.lower() not in _BLOCK_TAGS:
|
|
98
|
+
continue
|
|
99
|
+
element.tail = "\n" + (element.tail or "")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _plain_text(summary_html: str) -> str | None:
|
|
103
|
+
root = lxml_html.fromstring(summary_html)
|
|
104
|
+
_separate_blocks(root)
|
|
105
|
+
text = root.text_content().replace("\r\n", "\n").replace("\r", "\n")
|
|
106
|
+
text = _WHITESPACE.sub(" ", text)
|
|
107
|
+
text = _BLANK_LINES.sub("\n\n", text)
|
|
108
|
+
lines = [line.strip() for line in text.split("\n")]
|
|
109
|
+
cleaned = "\n".join(line for line in lines if line).strip()
|
|
110
|
+
return cleaned or None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
async def fetch_article_text(
|
|
114
|
+
url: str,
|
|
115
|
+
*,
|
|
116
|
+
user_agent: str,
|
|
117
|
+
timeout: float = 30.0,
|
|
118
|
+
) -> str | None:
|
|
119
|
+
try:
|
|
120
|
+
async with httpx.AsyncClient(
|
|
121
|
+
timeout=timeout,
|
|
122
|
+
follow_redirects=True,
|
|
123
|
+
headers={"User-Agent": user_agent},
|
|
124
|
+
) as client:
|
|
125
|
+
async with client.stream("GET", url) as response:
|
|
126
|
+
if response.status_code != 200 or not _allows_extract(response):
|
|
127
|
+
return None
|
|
128
|
+
body = await _read_limited(response)
|
|
129
|
+
except (httpx.HTTPError, httpx.InvalidURL, UnicodeEncodeError):
|
|
130
|
+
return None
|
|
131
|
+
if not body or not body.strip() or not _looks_textual(body):
|
|
132
|
+
return None
|
|
133
|
+
try:
|
|
134
|
+
summary = Document(body, url=url).summary()
|
|
135
|
+
return _plain_text(summary)
|
|
136
|
+
except Exception:
|
|
137
|
+
return None
|