webpage-parser 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webpage_parser/__init__.py +11 -0
- webpage_parser/author.py +503 -0
- webpage_parser/content.py +915 -0
- webpage_parser/extract.py +63 -0
- webpage_parser/htmlprep.py +760 -0
- webpage_parser/pagegate.py +177 -0
- webpage_parser/pipeline.py +196 -0
- webpage_parser/pubtime.py +490 -0
- webpage_parser/siteconfig.py +110 -0
- webpage_parser/sitespecific.py +170 -0
- webpage_parser/timetext.py +365 -0
- webpage_parser/title.py +222 -0
- webpage_parser/util.py +95 -0
- webpage_parser-0.1.0.dist-info/METADATA +93 -0
- webpage_parser-0.1.0.dist-info/RECORD +18 -0
- webpage_parser-0.1.0.dist-info/WHEEL +5 -0
- webpage_parser-0.1.0.dist-info/entry_points.txt +2 -0
- webpage_parser-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""网页解析算子:从爬取到的 HTML 抽取 title/content/pub_time/author。
|
|
2
|
+
|
|
3
|
+
包内模块沿用「相对导入优先、扁平导入兜底」的双模式写法,因此既可以按包导入
|
|
4
|
+
(`from webpage_parser import process_row`),也可以把目录直接摊平当脚本跑
|
|
5
|
+
(`python3 extract.py`)。推荐按包导入。
|
|
6
|
+
"""
|
|
7
|
+
__version__ = "0.1.0"
|
|
8
|
+
|
|
9
|
+
from .pipeline import empty_result, process_row
|
|
10
|
+
|
|
11
|
+
__all__ = ["empty_result", "process_row", "__version__"]
|
webpage_parser/author.py
ADDED
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
"""author 抽取:候选枚举 + 强拒绝(V3 以否定式条款为主)。
|
|
2
|
+
|
|
3
|
+
候选链:JSON-LD author -> meta author 键 -> extra.anchor(AuthorList 系且可见文本命中)
|
|
4
|
+
-> 可见文本 byline 正则。
|
|
5
|
+
强拒绝:站点名 / 编辑角色 / 占位符 / @handle / 邮箱 / URL。
|
|
6
|
+
"""
|
|
7
|
+
import re
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
from .util import clean_text, norm_key, has_cjk
|
|
11
|
+
from .htmlprep import ARTICLE_TYPES, walk_json_keys
|
|
12
|
+
from .title import _site_names, _confirmed_site
|
|
13
|
+
except ImportError:
|
|
14
|
+
from util import clean_text, norm_key, has_cjk
|
|
15
|
+
from htmlprep import ARTICLE_TYPES, walk_json_keys
|
|
16
|
+
from title import _site_names, _confirmed_site
|
|
17
|
+
|
|
18
|
+
_AUTHOR_META_KEYS = ("author", "article:author", "byline", "byl", "dc.creator",
|
|
19
|
+
"og:author", "sogou_author", "og:article:author",
|
|
20
|
+
"sailthru.author", "parsely-author", "mrf:authors",
|
|
21
|
+
"authors", "dable:author", "cxenseparse:author",
|
|
22
|
+
"articleauthor", "dcterms.creator", "creator",
|
|
23
|
+
"mediaid") # mediaid:搜狐号署名 meta(验收反馈 case_08)
|
|
24
|
+
# twitter:creator 刻意排除:实测值全是 @站号句柄(调研05 §6.3)
|
|
25
|
+
# 标签剥离:多字标签冒号可选;单字「文」必须带冒号,否则会把「文娱阁」这类
|
|
26
|
+
# 以文开头的署名截掉首字(验收反馈 case_09:anchor/页面署名均为「文娱阁」被截成「娱阁」)
|
|
27
|
+
_LABEL_ACCEPT_RE = re.compile(
|
|
28
|
+
r"^(?:(?:\u4f5c\u8005|\u8bb0\u8005|\u64b0\u7a3f|\u64b0\u6587|\u4f5c\u8005\u540d|"
|
|
29
|
+
r"\u7279\u7ea6\u4f5c\u8005|\u901a\u8baf\u5458|by|author)\s*[::]?\s*|\u6587\s*[::]\s*)",
|
|
30
|
+
re.IGNORECASE,
|
|
31
|
+
)
|
|
32
|
+
_LABEL_REJECT_RE = re.compile(
|
|
33
|
+
r"^(?:\u7f16\u8f91|\u8d23\u7f16|\u8d23\u4efb\u7f16\u8f91|\u5ba1\u6838|\u6821\u5bf9|"
|
|
34
|
+
r"\u503c\u73ed\u7f16\u8f91|\u6765\u6e90|\u51fa\u5904|\u6587\u7ae0\u6765\u6e90|\u7a3f\u6e90|"
|
|
35
|
+
r"\u7533\u660e|\u58f0\u660e)\s*[::]?",
|
|
36
|
+
re.IGNORECASE,
|
|
37
|
+
)
|
|
38
|
+
_PLACEHOLDER_RE = re.compile(r"\{\{|%\w+%|\$\{|<\{|unknown|anonymous", re.IGNORECASE)
|
|
39
|
+
# 候选含域名 → 整串是站点口号/署名垃圾,拆分前整体拒绝(dm5 meta author 实测)
|
|
40
|
+
_DOMAIN_IN_NAME_RE = re.compile(
|
|
41
|
+
r"\b[\w-]+\.(?:com|net|org|cn|io|co|cc|tv|me|xyz|top|vip|site|info)\b",
|
|
42
|
+
re.IGNORECASE)
|
|
43
|
+
# 纯英文角色词整串:Assistant Editor / Managing Editor 等是角色不是姓名
|
|
44
|
+
_EN_ROLE_RE = re.compile(
|
|
45
|
+
r"^(?:(?:assistant|associate|managing|senior|deputy|contributing|executive|"
|
|
46
|
+
r"chief|staff|web|news|copy|digital|online|commissioning)\s+)*"
|
|
47
|
+
r"(?:editor|writer|correspondent|reporter|columnist|producer|publisher)s?\.?$",
|
|
48
|
+
re.IGNORECASE)
|
|
49
|
+
_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.]+")
|
|
50
|
+
_URL_RE = re.compile(r"https?://|www\.", re.IGNORECASE)
|
|
51
|
+
_HANDLE_RE = re.compile(r"^@[\w\u4e00-\u9fff]+$")
|
|
52
|
+
_ROLE_TAIL_RE = re.compile(r"(\u7f16\u8f91|\u8d23\u7f16|\u5ba1\u6838|\u6821\u5bf9)$")
|
|
53
|
+
_USERNAME_RE = re.compile(
|
|
54
|
+
r"^(admin|administrator|user\d*|guest\d*|test\d*|editor\d*|"
|
|
55
|
+
r"\u6e38\u5ba2|\u697c\u4e3b|\u533f\u540d\u7528\u6237)$", re.IGNORECASE)
|
|
56
|
+
_COPYRIGHT_RE = re.compile(r"\u7248\u6743|\u00a9|copyright", re.IGNORECASE)
|
|
57
|
+
_UI_NOISE_RE = re.compile(
|
|
58
|
+
r"\u8bc4\u8bba|\u56de\u590d|\u9605\u8bfb|\u70b9\u8d5e|\u5206\u4eab|\u5173\u6ce8|"
|
|
59
|
+
r"\u8ba2\u9605|\u6536\u85cf|\u8f6c\u53d1|\u767b\u5f55|\u6ce8\u518c|\u8bbf\u95ee|"
|
|
60
|
+
r"share|comment|subscribe|sign\s*in|log\s*in|preferred\s+on\s+google|add\s+as\s+preferred|\u4ec5\u4ee3\u8868\u4f5c\u8005|\u67e5\u770b\u4f5c\u8005\u7b80\u4ecb", re.IGNORECASE)
|
|
61
|
+
_DATE_FRAG_RE = re.compile(
|
|
62
|
+
r"(?:19|20)\d{2}[-/\u5e74.]\s*\d{1,2}[-/\u6708.]\s*\d{1,2}\s*\u65e5?"
|
|
63
|
+
r"(?:\s*\d{1,2}[::\u65f6]\d{2}(?:[::\u5206]\d{2}\u79d2?)?)?"
|
|
64
|
+
r"|\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(:\d{2})?"
|
|
65
|
+
r"|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+\d{1,2}(?:st|nd|rd|th)?(?:,?\s*\d{4})?"
|
|
66
|
+
r"|\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?(?:\s*\d{4})?"
|
|
67
|
+
r"|\d+\s*(?:seconds?|minutes?|hours?|days?|weeks?|months?)\s+ago"
|
|
68
|
+
r"|a\s+(?:day|week|month|year|few\s+(?:days|hours|minutes))\s+ago"
|
|
69
|
+
r"|just\s+now"
|
|
70
|
+
r"|\b\d{1,2}:\d{2}\s*(?:a\.?m\.?|p\.?m\.?)\b(?:\s*[A-Z]{2,4})?"
|
|
71
|
+
r"|\d+\s*\u4e2a?\s*(?:\u79d2|\u5206\u949f|\u5c0f\u65f6|\u5929|\u65e5|\u5468|\u661f\u671f)\u524d"
|
|
72
|
+
r"|\u6628\u5929|\u4eca\u5929|\u524d\u5929|\u521a\u521a"
|
|
73
|
+
r"|(?:published|updated|posted|added|by)\b",
|
|
74
|
+
re.IGNORECASE)
|
|
75
|
+
_CN_TAIL_SIGN_RE = re.compile(
|
|
76
|
+
r"[\uff08(]\s*(?:[\u4e00-\u9fff]{0,6}?\u8bb0\u8005)\s*"
|
|
77
|
+
r"([^\uff08\uff09()\uff0c\uff0c\u3002\uff1b;,]{2,20}?)\s*[\uff09)]"
|
|
78
|
+
r"|\u6587\s*[\uff5c|/]\s*([^\s\uff0c\u3002,;]{2,15})"
|
|
79
|
+
)
|
|
80
|
+
_FOLLOW_TAIL_RE = re.compile(
|
|
81
|
+
r"(\u5173\u6ce8|\u8ba2\u9605|\u52a0\u5173\u6ce8|follow|subscribe)$", re.IGNORECASE)
|
|
82
|
+
_BYLINE_RE = re.compile(
|
|
83
|
+
r"(?:\u4f5c\u8005|\u8bb0\u8005|\u6587|\u64b0\u7a3f|\u64b0\u6587)\s*[::]\s*"
|
|
84
|
+
r"([^\s\uff0c\u3002\uff1b\uff1a|,;:(\uff08)\uff09\u201c\u201d\u2018\u2019\"\x27+]{2,20})"
|
|
85
|
+
)
|
|
86
|
+
_BYLINE_EN_RE = re.compile(r"\bBy\s+([A-Z][A-Za-z.\-]+(?:\s+[A-Z][A-Za-z.\-]+){0,3})")
|
|
87
|
+
_WIRE_BYLINE_RE = re.compile(
|
|
88
|
+
r"(?<![A-Za-z,])([A-Z][A-Za-z.'\-]+(?:\s+[A-Z][A-Za-z.'\-]+){1,3}),\s+"
|
|
89
|
+
r"(?:The\s+)?[A-Z][A-Za-z.&' ]{1,40}?"
|
|
90
|
+
r"(?:Press|News|Media|Wire|Times|Post|Herald|Tribune|Reuters|AFP|AP)\.?(?:\s|$)")
|
|
91
|
+
_WIRE_ORG_IN_NAME_RE = re.compile(
|
|
92
|
+
r"\b(Press|News|Media|Wire|Times|Post|Herald|Tribune|Reuters|AFP|AP)\b")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _wire_byline_candidates(tail):
|
|
96
|
+
pos = 0
|
|
97
|
+
while pos < len(tail):
|
|
98
|
+
m = _WIRE_BYLINE_RE.search(tail, pos)
|
|
99
|
+
if not m:
|
|
100
|
+
return
|
|
101
|
+
name = m.group(1)
|
|
102
|
+
if _WIRE_ORG_IN_NAME_RE.search(name):
|
|
103
|
+
pos = m.start() + 1
|
|
104
|
+
continue
|
|
105
|
+
yield name
|
|
106
|
+
pos = m.end()
|
|
107
|
+
_TITLE_SEG_RE = re.compile(r"[_\-||–—»]+")
|
|
108
|
+
_SECTION_WORDS = {
|
|
109
|
+
"焦点", "时尚", "娱乐", "体育", "财经", "科技", "军事", "国际", "国内",
|
|
110
|
+
"社会", "专题", "汽车", "房产", "游戏", "教育", "健康", "核心提示", "要闻",
|
|
111
|
+
"租借",
|
|
112
|
+
}
|
|
113
|
+
_ORG_HINT_RE = re.compile(
|
|
114
|
+
r"\u793e|\u7f51|\u62a5|\u53f0|\u5a92\u4f53|\u4e2d\u5fc3|\u90e8|\u5c40|\u59d4\u5458\u4f1a|"
|
|
115
|
+
r"\u516c\u53f8|\u96c6\u56e2|\u5927\u5b66|\u7814\u7a76\u9662|news|media|agency|studio$",
|
|
116
|
+
re.IGNORECASE,
|
|
117
|
+
)
|
|
118
|
+
_SPLIT_RE = re.compile(r"[\u3001\uff0c,|/;]+|\s{2,}")
|
|
119
|
+
_MAX_AUTHORS = 8
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _norm_cmp(s):
|
|
123
|
+
return norm_key(s)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _reject(name, site_cfg, title_norm, site_names):
|
|
127
|
+
n = clean_text(name)
|
|
128
|
+
if not n:
|
|
129
|
+
return True
|
|
130
|
+
nn = _norm_cmp(n)
|
|
131
|
+
if not nn:
|
|
132
|
+
return True
|
|
133
|
+
if len(n) > 60:
|
|
134
|
+
return True
|
|
135
|
+
if n.isdigit():
|
|
136
|
+
return True
|
|
137
|
+
if "=" in n or "&" in n:
|
|
138
|
+
return True
|
|
139
|
+
if n in _SECTION_WORDS:
|
|
140
|
+
return True
|
|
141
|
+
if re.match(r"^\d", n):
|
|
142
|
+
return True
|
|
143
|
+
if re.match(r"^\u6765\u81ea", n):
|
|
144
|
+
return True
|
|
145
|
+
if re.match(r"^\u5173\u4e8e", n):
|
|
146
|
+
return True
|
|
147
|
+
if re.search(r"\uae30\uc790|\uae30\uc0ac|\ubaa8\uc74c|\uc785\ub825", n) and len(n) <= 6:
|
|
148
|
+
return True
|
|
149
|
+
if _PLACEHOLDER_RE.search(n):
|
|
150
|
+
return True
|
|
151
|
+
if _EMAIL_RE.search(n) or _URL_RE.search(n) or _HANDLE_RE.match(n):
|
|
152
|
+
return True
|
|
153
|
+
if _USERNAME_RE.match(n):
|
|
154
|
+
return True
|
|
155
|
+
if _COPYRIGHT_RE.search(n):
|
|
156
|
+
return True
|
|
157
|
+
if _UI_NOISE_RE.search(n):
|
|
158
|
+
return True
|
|
159
|
+
if re.search(r"\bstaff\b", n, re.IGNORECASE):
|
|
160
|
+
return True
|
|
161
|
+
if _LABEL_REJECT_RE.match(n):
|
|
162
|
+
return True
|
|
163
|
+
if _ROLE_TAIL_RE.search(n) and len(n) <= 6:
|
|
164
|
+
return True
|
|
165
|
+
if has_cjk(n):
|
|
166
|
+
if len(n) > 20 or re.search(r"[\u3002\uff01\uff1f]", n):
|
|
167
|
+
return True
|
|
168
|
+
else:
|
|
169
|
+
if len(n.split()) > 8: # 拉丁整句按词数判定(03 §5.3)
|
|
170
|
+
return True
|
|
171
|
+
for bad in site_cfg.reject_names:
|
|
172
|
+
if nn == _norm_cmp(bad):
|
|
173
|
+
return True
|
|
174
|
+
if site_names and _confirmed_site(n, site_names):
|
|
175
|
+
return True
|
|
176
|
+
if title_norm and nn == title_norm:
|
|
177
|
+
return True
|
|
178
|
+
if nn in ("null", "none", "n/a", "\u4f5a\u540d", "\u533f\u540d", "\u65e0",
|
|
179
|
+
"\u6682\u65e0", "\u5c0f\u7f16", "\u7efc\u5408", "\u7f51\u7edc",
|
|
180
|
+
"\u7f51\u53cb", "\u8f6c\u8f7d", "\u5176\u4ed6", "staff", "file",
|
|
181
|
+
"\u7528\u6237\u540d", "newsprovided",
|
|
182
|
+
"\u53d1\u5e03\u4e8e", "\u53d1\u8868\u4e8e",
|
|
183
|
+
"on", "by", "at", "in", "of", "the", "and", "posted"):
|
|
184
|
+
return True
|
|
185
|
+
return False
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _strip_labels(name):
|
|
189
|
+
"""循环剥标签前缀;若命中拒绝型标签(编辑/来源),返回 None。"""
|
|
190
|
+
cur = clean_text(name)
|
|
191
|
+
for _ in range(3):
|
|
192
|
+
if _LABEL_REJECT_RE.match(cur):
|
|
193
|
+
return None
|
|
194
|
+
new = _LABEL_ACCEPT_RE.sub("", cur)
|
|
195
|
+
if new == cur:
|
|
196
|
+
break
|
|
197
|
+
cur = new.strip()
|
|
198
|
+
return cur
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _split_multi(name):
|
|
202
|
+
if has_cjk(name):
|
|
203
|
+
parts = [p.strip() for p in re.split(r"[\u3001\uff0c,|/;]+|\s+", name) if p.strip()]
|
|
204
|
+
else:
|
|
205
|
+
parts = [p.strip() for p in _SPLIT_RE.split(name) if p.strip()]
|
|
206
|
+
return parts[:_MAX_AUTHORS] if parts else []
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _finalize(cand, site_cfg, title_norm, page_norms, source, site_names):
|
|
210
|
+
if not cand:
|
|
211
|
+
return None
|
|
212
|
+
stripped = _strip_labels(cand)
|
|
213
|
+
if stripped is None:
|
|
214
|
+
return None
|
|
215
|
+
if _URL_RE.search(stripped) or _EMAIL_RE.search(stripped):
|
|
216
|
+
return None
|
|
217
|
+
if _DOMAIN_IN_NAME_RE.search(stripped):
|
|
218
|
+
return None
|
|
219
|
+
if _EN_ROLE_RE.match(stripped.strip()):
|
|
220
|
+
return None
|
|
221
|
+
out = []
|
|
222
|
+
for p in _split_multi(stripped):
|
|
223
|
+
if _reject(p, site_cfg, title_norm, site_names):
|
|
224
|
+
continue
|
|
225
|
+
if _EN_ROLE_RE.match(p):
|
|
226
|
+
continue
|
|
227
|
+
np = _norm_cmp(p)
|
|
228
|
+
if page_norms and np not in page_norms[0] and np not in page_norms[1]:
|
|
229
|
+
continue
|
|
230
|
+
if p not in out:
|
|
231
|
+
out.append(p)
|
|
232
|
+
if not out:
|
|
233
|
+
return None
|
|
234
|
+
joined = "\u3001".join(out)
|
|
235
|
+
return joined, source, bool(_ORG_HINT_RE.search(joined))
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _jsonld_names(obj, out, depth=0):
|
|
239
|
+
if depth > 4:
|
|
240
|
+
return
|
|
241
|
+
if isinstance(obj, str):
|
|
242
|
+
if obj.strip():
|
|
243
|
+
out.append(obj.strip())
|
|
244
|
+
return
|
|
245
|
+
if isinstance(obj, dict):
|
|
246
|
+
nm = obj.get("name")
|
|
247
|
+
if isinstance(nm, str):
|
|
248
|
+
out.append(nm.strip())
|
|
249
|
+
return
|
|
250
|
+
if isinstance(nm, dict):
|
|
251
|
+
v = nm.get("@value") or nm.get("name")
|
|
252
|
+
if isinstance(v, str):
|
|
253
|
+
out.append(v.strip())
|
|
254
|
+
return
|
|
255
|
+
for k in ("author", "creator", "contributor"):
|
|
256
|
+
if k in obj:
|
|
257
|
+
_jsonld_names(obj[k], out, depth + 1)
|
|
258
|
+
elif isinstance(obj, list):
|
|
259
|
+
for item in obj[:10]:
|
|
260
|
+
_jsonld_names(item, out, depth + 1)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _visible_tail(signals, limit=4000):
|
|
264
|
+
"""从解析树取可见文本尾部(跳过 script/style,保留词边界)。"""
|
|
265
|
+
tree = getattr(signals, "tree", None)
|
|
266
|
+
if tree is None:
|
|
267
|
+
return ""
|
|
268
|
+
parts = []
|
|
269
|
+
|
|
270
|
+
def walk(el):
|
|
271
|
+
if not isinstance(el.tag, str):
|
|
272
|
+
return
|
|
273
|
+
if el.tag in ("script", "style", "noscript", "template"):
|
|
274
|
+
return
|
|
275
|
+
if el.text:
|
|
276
|
+
parts.append(el.text)
|
|
277
|
+
for ch in el:
|
|
278
|
+
walk(ch)
|
|
279
|
+
if ch.tail:
|
|
280
|
+
parts.append(ch.tail)
|
|
281
|
+
|
|
282
|
+
try:
|
|
283
|
+
body = tree.find("body")
|
|
284
|
+
walk(body if body is not None else tree)
|
|
285
|
+
except Exception:
|
|
286
|
+
return ""
|
|
287
|
+
return clean_text(" ".join(parts))[-limit:]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _l1b_author(signals, site_cfg):
|
|
291
|
+
"""L1b 内嵌状态 JSON 的作者名(仅 allow_l1b 站点族)。
|
|
292
|
+
|
|
293
|
+
百家号 bsdata...itemdata.name、小红书 note.user.nickname。
|
|
294
|
+
base 必须在白名单键内且路径含 hint,防止抓到文章标题的 name。
|
|
295
|
+
"""
|
|
296
|
+
hints = getattr(site_cfg, "l1b_author_hints", ()) or ()
|
|
297
|
+
if not hints:
|
|
298
|
+
return ""
|
|
299
|
+
base_ok = ("name", "nickname", "authorname", "media_name")
|
|
300
|
+
for obj in signals.state_objs:
|
|
301
|
+
for key, val in walk_json_keys(obj):
|
|
302
|
+
base = key.rsplit(".", 1)[-1]
|
|
303
|
+
if base not in base_ok or not isinstance(val, str):
|
|
304
|
+
continue
|
|
305
|
+
if not (2 <= len(val.strip()) <= 40):
|
|
306
|
+
continue
|
|
307
|
+
if any(h in key for h in hints):
|
|
308
|
+
return val.strip()
|
|
309
|
+
return ""
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _clean_byline_node(txt):
|
|
313
|
+
"""署名节点文本清洗:'By Allyson Castillo' / '小周侃球关注' /
|
|
314
|
+
'John Sicher a day ago' / '大河报·豫视频 2026-08-28 08:15' -> 干净名字或 None。"""
|
|
315
|
+
if not txt:
|
|
316
|
+
return None
|
|
317
|
+
t = clean_text(txt)
|
|
318
|
+
if not t or len(t) > 60:
|
|
319
|
+
return None
|
|
320
|
+
if re.search(r"[。!?]", t):
|
|
321
|
+
return None
|
|
322
|
+
t = _EMAIL_RE.sub("", t)
|
|
323
|
+
t = _DATE_FRAG_RE.sub(" ", t)
|
|
324
|
+
t = _FOLLOW_TAIL_RE.sub("", t).strip(" ·||,,-—–")
|
|
325
|
+
t = _LABEL_ACCEPT_RE.sub("", t).strip()
|
|
326
|
+
t = re.sub(r"\s{2,}", " ", t).strip(" ·||,,")
|
|
327
|
+
if not t or len(t) > 40:
|
|
328
|
+
return None
|
|
329
|
+
if _DATE_FRAG_RE.search(t): # 清完还剩日期 -> 本身就不是署名块
|
|
330
|
+
return None
|
|
331
|
+
if len(t) <= 12 and _UI_NOISE_RE.search(t):
|
|
332
|
+
return None
|
|
333
|
+
if not re.search(r"[A-Za-z\u4e00-\u9fff\uac00-\ud7a5\u3040-\u30ff\u0400-\u04ff\u0600-\u06ff]", t):
|
|
334
|
+
return None
|
|
335
|
+
return t
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
_MEDIA_ORG_RE = re.compile(
|
|
339
|
+
r"(\u65e5\u62a5|\u665a\u62a5|\u65f6\u62a5|\u5feb\u62a5|\u5468\u520a|\u6742\u5fd7|\u65b0\u95fb"
|
|
340
|
+
r"|\u7535\u89c6\u53f0|\u5e7f\u64ad|\u901a\u8baf\u793e|\u51fa\u7248|\u62a5\u4e1a|\u4f20\u5a92"
|
|
341
|
+
r"|\u5a92\u4f53|\u5ba2\u6237\u7aef|\u65b0\u6d6a|\u7f51\u6613|\u817e\u8baf|\u51e4\u51f0"
|
|
342
|
+
r"|sina|sohu|163|qq\.|ifeng|cctv|xinhua|people|chinanews"
|
|
343
|
+
r"|thepaper)|(\u7f51|\u62a5|\u53f0|\u793e)$", re.I)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _anchor_context_ok(anchor, signals):
|
|
347
|
+
"""anchor 的前文若是「来源:/转载自」且 anchor 本身是媒体机构名,则是内容来源
|
|
348
|
+
而非作者(03 §5.1b 反例 anchor=新浪财经);自媒体账号名放行(163 模板里
|
|
349
|
+
「来源: 王姐懒人家常菜」中的账号名就是网易号作者)。
|
|
350
|
+
|
|
351
|
+
返回 True=可以当作者候选。
|
|
352
|
+
"""
|
|
353
|
+
tree = getattr(signals, "tree", None)
|
|
354
|
+
if tree is None or not anchor:
|
|
355
|
+
return False
|
|
356
|
+
try:
|
|
357
|
+
body = tree.find("body")
|
|
358
|
+
root = body if body is not None else tree
|
|
359
|
+
parts = []
|
|
360
|
+
|
|
361
|
+
def walk(el):
|
|
362
|
+
if not isinstance(el.tag, str):
|
|
363
|
+
return
|
|
364
|
+
if el.tag in ("script", "style", "noscript", "template"):
|
|
365
|
+
return
|
|
366
|
+
if el.text:
|
|
367
|
+
parts.append(el.text)
|
|
368
|
+
for ch in el:
|
|
369
|
+
walk(ch)
|
|
370
|
+
if ch.tail:
|
|
371
|
+
parts.append(ch.tail)
|
|
372
|
+
|
|
373
|
+
walk(root)
|
|
374
|
+
text = clean_text(" ".join(parts))
|
|
375
|
+
except Exception:
|
|
376
|
+
return False
|
|
377
|
+
idx = text.find(anchor)
|
|
378
|
+
if idx < 0:
|
|
379
|
+
return False
|
|
380
|
+
pre = text[max(0, idx - 8):idx]
|
|
381
|
+
if re.search(r"(\u6765\u6e90|\u8f6c\u8f7d\u81ea|\u51fa\u5904)\s*[:\uff1a/]?\s*$", pre) \
|
|
382
|
+
and _MEDIA_ORG_RE.search(anchor):
|
|
383
|
+
return False
|
|
384
|
+
return True
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _cn_tail_sign(signals):
|
|
388
|
+
"""正文末尾中文记者署名:(记者 张三)/(本报记者 李四)/文|王五。"""
|
|
389
|
+
tail = _visible_tail(signals, limit=2500)
|
|
390
|
+
if not tail:
|
|
391
|
+
return ""
|
|
392
|
+
for m in _CN_TAIL_SIGN_RE.finditer(tail):
|
|
393
|
+
name = m.group(1) or m.group(2) or ""
|
|
394
|
+
name = re.sub(r"^[\u4e00-\u9fff]{0,8}?\u8bb0\u8005", "", name)
|
|
395
|
+
name = re.sub(r"\s*\u901a\u8baf\u5458\s*", "\u3001", name).strip()
|
|
396
|
+
if name:
|
|
397
|
+
return name
|
|
398
|
+
return ""
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def extract_author(signals, site_cfg, extra, visible_norm, html_norm, title=""):
|
|
402
|
+
"""返回 (author, author_source, author_is_org)。"""
|
|
403
|
+
page_norms = (visible_norm or "", html_norm or "")
|
|
404
|
+
title_norm = _norm_cmp(title)
|
|
405
|
+
|
|
406
|
+
site_names = _site_names(signals, site_cfg)
|
|
407
|
+
for seg in _TITLE_SEG_RE.split(signals.title_tag or ""):
|
|
408
|
+
seg = clean_text(seg)
|
|
409
|
+
if len(seg) >= 2:
|
|
410
|
+
site_names.add(_norm_cmp(seg))
|
|
411
|
+
|
|
412
|
+
# 1) JSON-LD author
|
|
413
|
+
for node in signals.jsonld_nodes:
|
|
414
|
+
t = node.get("@type")
|
|
415
|
+
types = set()
|
|
416
|
+
if isinstance(t, str):
|
|
417
|
+
types.add(t.lower())
|
|
418
|
+
elif isinstance(t, list):
|
|
419
|
+
types.update(str(x).lower() for x in t)
|
|
420
|
+
if types and not (types & ARTICLE_TYPES):
|
|
421
|
+
continue
|
|
422
|
+
if "author" not in node:
|
|
423
|
+
continue
|
|
424
|
+
names = []
|
|
425
|
+
_jsonld_names(node["author"], names)
|
|
426
|
+
for nm in names:
|
|
427
|
+
r = _finalize(nm, site_cfg, title_norm, page_norms, "jsonld", site_names)
|
|
428
|
+
if r:
|
|
429
|
+
return r
|
|
430
|
+
|
|
431
|
+
# 2) meta author 键
|
|
432
|
+
for key in _AUTHOR_META_KEYS:
|
|
433
|
+
for v in signals.meta.get(key, []):
|
|
434
|
+
r = _finalize(v, site_cfg, title_norm, page_norms, "meta:" + key, site_names)
|
|
435
|
+
if r:
|
|
436
|
+
return r
|
|
437
|
+
|
|
438
|
+
# 3) extra.anchor(AuthorList 系白名单 + 可见文本命中 + 非来源语境,调研05 §5.1b)
|
|
439
|
+
anchor = ""
|
|
440
|
+
try:
|
|
441
|
+
anchor = clean_text(str((extra or {}).get("anchor") or ""))
|
|
442
|
+
except Exception:
|
|
443
|
+
anchor = ""
|
|
444
|
+
if (anchor and site_cfg.allow_anchor_author
|
|
445
|
+
and _norm_cmp(anchor) in page_norms[0]
|
|
446
|
+
and _anchor_context_ok(anchor, signals)):
|
|
447
|
+
r = _finalize(anchor, site_cfg, title_norm, page_norms, "extra_anchor", site_names)
|
|
448
|
+
if r:
|
|
449
|
+
return r
|
|
450
|
+
|
|
451
|
+
# 3b) L1b 内嵌状态 JSON 的作者名(百家号 itemdata.name / 小红书 note.user.nickname)
|
|
452
|
+
if site_cfg.allow_l1b:
|
|
453
|
+
nm = _l1b_author(signals, site_cfg)
|
|
454
|
+
if nm:
|
|
455
|
+
r = _finalize(nm, site_cfg, title_norm, page_norms, "state_json", site_names)
|
|
456
|
+
if r:
|
|
457
|
+
return r
|
|
458
|
+
|
|
459
|
+
# 4) 署名节点(class/id/testid/itemprop/rel 命中),清洗后过强拒绝
|
|
460
|
+
node_cands = []
|
|
461
|
+
for txt, strong_bz, weak_bz, pos, tag in getattr(signals, "author_nodes", []):
|
|
462
|
+
if strong_bz:
|
|
463
|
+
continue
|
|
464
|
+
c = _clean_byline_node(txt)
|
|
465
|
+
if not c:
|
|
466
|
+
continue
|
|
467
|
+
node_cands.append((0 if pos < 0.5 else 1, len(c), c))
|
|
468
|
+
node_cands.sort()
|
|
469
|
+
for _score, _ln, cand in node_cands[:6]:
|
|
470
|
+
r = _finalize(cand, site_cfg, title_norm, page_norms, "byline_node", site_names)
|
|
471
|
+
if r:
|
|
472
|
+
return r
|
|
473
|
+
|
|
474
|
+
# 5) 可见文本 byline 正则(扫正文前部纯文本)
|
|
475
|
+
hay = signals.html[:120_000]
|
|
476
|
+
text_only = clean_text(re.sub(r"<[^>]+>", " ", hay))[:6000]
|
|
477
|
+
for m in _BYLINE_RE.finditer(text_only):
|
|
478
|
+
v = m.group(1)
|
|
479
|
+
if has_cjk(v) and len(v) > 8:
|
|
480
|
+
continue
|
|
481
|
+
r = _finalize(v, site_cfg, title_norm, page_norms, "byline_hint", site_names)
|
|
482
|
+
if r:
|
|
483
|
+
return r
|
|
484
|
+
for m in _BYLINE_EN_RE.finditer(text_only):
|
|
485
|
+
r = _finalize(m.group(1), site_cfg, title_norm, page_norms, "byline_en", site_names)
|
|
486
|
+
if r:
|
|
487
|
+
return r
|
|
488
|
+
|
|
489
|
+
# 6) 中文末尾记者署名((记者 张三)/文|张三)
|
|
490
|
+
nm = _cn_tail_sign(signals)
|
|
491
|
+
if nm:
|
|
492
|
+
r = _finalize(nm, site_cfg, title_norm, page_norms, "cn_tail_sign", site_names)
|
|
493
|
+
if r:
|
|
494
|
+
return r
|
|
495
|
+
|
|
496
|
+
# 7) 文章末尾通讯社署名(Elissa Mendes, The Canadian Press → 只取人名)
|
|
497
|
+
tail = _visible_tail(signals)
|
|
498
|
+
for nm in _wire_byline_candidates(tail):
|
|
499
|
+
r = _finalize(nm, site_cfg, title_norm, page_norms, "wire_byline", site_names)
|
|
500
|
+
if r:
|
|
501
|
+
return r
|
|
502
|
+
|
|
503
|
+
return "", "none", False
|