sherlock-mcp 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -18
- package/dist/cli.cjs +19 -19
- package/package.json +1 -1
- package/python/social_profiles.py +136 -8
package/package.json
CHANGED
|
@@ -23,10 +23,13 @@ WALL = re.compile(
|
|
|
23
23
|
r"verify (?:you are|you're) human|security check|unusual activity|access denied|"
|
|
24
24
|
r"temporarily blocked|登录后|登入後|安全验证|安全驗證|验证码|驗證碼", re.I)
|
|
25
25
|
MAX_BYTES = 1_000_000
|
|
26
|
+
BING_SEARCH_HOSTS = {"www.bing.com", "cn.bing.com"}
|
|
26
27
|
|
|
27
28
|
|
|
28
29
|
class DiscoveryError(Exception):
|
|
29
|
-
|
|
30
|
+
def __init__(self, code, http_status=None):
|
|
31
|
+
super().__init__(code)
|
|
32
|
+
self.http_status = http_status
|
|
30
33
|
|
|
31
34
|
|
|
32
35
|
def now():
|
|
@@ -100,7 +103,8 @@ def failure(error):
|
|
|
100
103
|
return "timeout" if "timeout" in type(error).__name__.lower() else "request_failed"
|
|
101
104
|
|
|
102
105
|
|
|
103
|
-
def get(session, url, config, deadline, *, params=None, allowed_redirect=None
|
|
106
|
+
def get(session, url, config, deadline, *, params=None, allowed_redirect=None,
|
|
107
|
+
return_metadata=False, allow_not_found=False):
|
|
104
108
|
"""Bound every read and check redirects BEFORE sending the next request."""
|
|
105
109
|
proxy = config.get("proxy")
|
|
106
110
|
for _ in range(4):
|
|
@@ -118,9 +122,9 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
|
|
|
118
122
|
url = target
|
|
119
123
|
continue
|
|
120
124
|
if response.status_code in {401, 403, 407, 429, 999}:
|
|
121
|
-
raise DiscoveryError("blocked")
|
|
122
|
-
if response.status_code >= 400:
|
|
123
|
-
raise DiscoveryError("http_error")
|
|
125
|
+
raise DiscoveryError("blocked", response.status_code)
|
|
126
|
+
if response.status_code >= 400 and not (allow_not_found and response.status_code in {404, 410}):
|
|
127
|
+
raise DiscoveryError("http_error", response.status_code)
|
|
124
128
|
chunks, size = [], 0
|
|
125
129
|
for chunk in response.iter_content(16384):
|
|
126
130
|
if time.monotonic() >= deadline:
|
|
@@ -129,10 +133,24 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
|
|
|
129
133
|
if size > MAX_BYTES:
|
|
130
134
|
raise DiscoveryError("response_too_large")
|
|
131
135
|
chunks.append(chunk)
|
|
132
|
-
|
|
136
|
+
result = b"".join(chunks), response.headers.get("Content-Type", ""), response.encoding or "utf-8"
|
|
137
|
+
return (*result, response.status_code, response.url) if return_metadata else result
|
|
133
138
|
raise DiscoveryError("redirect_restricted")
|
|
134
139
|
|
|
135
140
|
|
|
141
|
+
def bing_search_redirect(url, query):
|
|
142
|
+
"""Bing can select a regional endpoint; keep the original RSS query."""
|
|
143
|
+
try:
|
|
144
|
+
parts = urlsplit(url)
|
|
145
|
+
params = parse_qs(parts.query)
|
|
146
|
+
return (parts.scheme == "https" and parts.hostname in BING_SEARCH_HOSTS
|
|
147
|
+
and parts.port in {None, 443} and parts.username is None
|
|
148
|
+
and parts.password is None and parts.path == "/search" and not parts.fragment
|
|
149
|
+
and params.get("q") == [query] and params.get("format") == ["rss"])
|
|
150
|
+
except (ValueError, TypeError):
|
|
151
|
+
return False
|
|
152
|
+
|
|
153
|
+
|
|
136
154
|
def search_hits(session, query, config, deadline):
|
|
137
155
|
searx = config.get("searxng_url")
|
|
138
156
|
if searx:
|
|
@@ -152,7 +170,7 @@ def search_hits(session, query, config, deadline):
|
|
|
152
170
|
raw, content_type, _ = get(
|
|
153
171
|
session, "https://www.bing.com/search", config, deadline,
|
|
154
172
|
params={"q": query, "format": "rss", "count": 10},
|
|
155
|
-
allowed_redirect=lambda u:
|
|
173
|
+
allowed_redirect=lambda u: bing_search_redirect(u, query))
|
|
156
174
|
try:
|
|
157
175
|
root = ET.fromstring(raw)
|
|
158
176
|
except ET.ParseError:
|
|
@@ -163,7 +181,107 @@ def search_hits(session, query, config, deadline):
|
|
|
163
181
|
"snippet": item.findtext("description", "")} for item in root.findall("./channel/item")[:20]], False, "bing_rss"
|
|
164
182
|
|
|
165
183
|
|
|
166
|
-
|
|
184
|
+
class ProfileEvidence(HTMLParser):
|
|
185
|
+
"""Read explicit public profile metadata; HTTP 200 alone is not a hit."""
|
|
186
|
+
def __init__(self):
|
|
187
|
+
super().__init__(convert_charrefs=True)
|
|
188
|
+
self.meta = {}
|
|
189
|
+
self.canonical = []
|
|
190
|
+
|
|
191
|
+
def handle_starttag(self, tag, attrs):
|
|
192
|
+
attrs = dict(attrs)
|
|
193
|
+
if tag == "meta":
|
|
194
|
+
key = (attrs.get("property") or attrs.get("name") or "").lower()
|
|
195
|
+
self.meta[key] = attrs.get("content", "")
|
|
196
|
+
if tag == "link" and "canonical" in (attrs.get("rel") or "").lower().split():
|
|
197
|
+
self.canonical.append(attrs.get("href", ""))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def direct_profile(platform, config, deadline):
|
|
201
|
+
import requests
|
|
202
|
+
username = config["username"]
|
|
203
|
+
base = {"facebook": "https://www.facebook.com/", "twitter": "https://x.com/",
|
|
204
|
+
"linkedin": "https://www.linkedin.com/in/"}[platform]
|
|
205
|
+
valid = profile_url(base + username, platform)
|
|
206
|
+
row = {"platform": platform, "method": "direct_username", "username": username,
|
|
207
|
+
"url": valid[0] if valid else "", "checked_at": None, "status": "ILLEGAL" if not valid else "UNKNOWN",
|
|
208
|
+
"search_status": "not_applicable" if not valid else "failed", "search_provider": "none",
|
|
209
|
+
"http_status": None, "page_status": "not_requested", "identity_status": "unverified", "candidates": []}
|
|
210
|
+
if not valid:
|
|
211
|
+
row["reason"] = "invalid_for_site"
|
|
212
|
+
return row
|
|
213
|
+
row["checked_at"] = now()
|
|
214
|
+
|
|
215
|
+
def same_profile(target):
|
|
216
|
+
parts = urlsplit(target)
|
|
217
|
+
if parts.scheme != "https" or parts.port not in {None, 443}:
|
|
218
|
+
return False
|
|
219
|
+
candidate = profile_url(target, platform)
|
|
220
|
+
if candidate and candidate[0].casefold() == valid[0].casefold():
|
|
221
|
+
return True
|
|
222
|
+
# Do not follow login/checkpoint redirects or borrow a browser session.
|
|
223
|
+
if parts.hostname in HOSTS[platform] and re.match(r"^/(?:login|checkpoint|authwall|i/flow/login)(?:[/.?]|$)", parts.path):
|
|
224
|
+
raise DiscoveryError("login_or_challenge")
|
|
225
|
+
return False
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
with requests.Session() as session:
|
|
229
|
+
session.trust_env = False
|
|
230
|
+
session.headers.update({"User-Agent": "sherlock-mcp/2.0 public-profile-probe",
|
|
231
|
+
"Accept-Language": "en-US,en;q=0.8"})
|
|
232
|
+
raw, ctype, encoding, status, _final_url = get(
|
|
233
|
+
session, valid[0], config, deadline, allowed_redirect=same_profile,
|
|
234
|
+
return_metadata=True, allow_not_found=True)
|
|
235
|
+
row["http_status"] = status
|
|
236
|
+
if ctype and not any(t in ctype.lower() for t in ("text/html", "application/xhtml")):
|
|
237
|
+
raise DiscoveryError("unsupported_content_type")
|
|
238
|
+
html = raw.decode(encoding, errors="replace")
|
|
239
|
+
content = text_of(html)
|
|
240
|
+
if WALL.search(content):
|
|
241
|
+
raise DiscoveryError("login_or_challenge")
|
|
242
|
+
if status in {404, 410}:
|
|
243
|
+
row.update(status="AVAILABLE", search_status="no_matches", page_status="not_found", reason="http_not_found")
|
|
244
|
+
return row
|
|
245
|
+
evidence = ProfileEvidence()
|
|
246
|
+
evidence.feed(html)
|
|
247
|
+
meta = evidence.meta
|
|
248
|
+
canonical = any(value and value[0].casefold() == valid[0].casefold()
|
|
249
|
+
for value in (profile_url(u, platform) for u in [*evidence.canonical, meta.get("og:url", "")]))
|
|
250
|
+
title = meta.get("og:title", "").strip()
|
|
251
|
+
profile_type = meta.get("og:type", "").casefold() == "profile"
|
|
252
|
+
if platform == "facebook":
|
|
253
|
+
native = any(re.match(r"^fb://(?:profile|page)/\d+(?:[/?]|$)", meta.get(k, ""))
|
|
254
|
+
for k in ("al:ios:url", "al:android:url"))
|
|
255
|
+
elif platform == "twitter":
|
|
256
|
+
native = meta.get("twitter:creator", "").lstrip("@").casefold() == username.casefold()
|
|
257
|
+
else:
|
|
258
|
+
native = bool(meta.get("profile:first_name") and meta.get("profile:last_name"))
|
|
259
|
+
generic = title.casefold() in {"facebook", "linkedin", "twitter", "x", "log in", "sign in"}
|
|
260
|
+
if not (canonical and title and not generic and (profile_type or native)):
|
|
261
|
+
row.update(page_status="empty" if not content else "insufficient_profile_evidence",
|
|
262
|
+
reason="insufficient_profile_evidence")
|
|
263
|
+
return row
|
|
264
|
+
row.update(status="CLAIMED", search_status="candidates_found", page_status="fetched",
|
|
265
|
+
reason="public_profile_metadata", signals=["matching_profile_url", "public_profile_metadata"])
|
|
266
|
+
candidate = {"url": valid[0], "handle": valid[1], "discovery_method": "direct_username",
|
|
267
|
+
"title": title[:300], "exact_handle_match": True, "page_status": "fetched",
|
|
268
|
+
"http_status": status, "fetched_at": now(), "identity_status": "unverified"}
|
|
269
|
+
if config["fetch_profiles"]:
|
|
270
|
+
candidate["page_excerpt"] = content[:4000]
|
|
271
|
+
row["candidates"] = [candidate]
|
|
272
|
+
except Exception as exc:
|
|
273
|
+
reason = failure(exc)
|
|
274
|
+
status = getattr(exc, "http_status", None)
|
|
275
|
+
if status is not None:
|
|
276
|
+
row["http_status"] = status
|
|
277
|
+
if status == 407:
|
|
278
|
+
reason = "proxy_authentication_failed"
|
|
279
|
+
row.update(status="WAF" if reason in {"blocked", "login_or_challenge"} else "UNKNOWN",
|
|
280
|
+
page_status=reason, reason=reason)
|
|
281
|
+
return row
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def discover_search(platform, config, deadline):
|
|
167
285
|
import requests
|
|
168
286
|
site = {"facebook": "site:facebook.com", "twitter": "(site:x.com OR site:twitter.com)",
|
|
169
287
|
"linkedin": "site:linkedin.com/in/"}[platform]
|
|
@@ -217,11 +335,21 @@ def discover(platform, config, deadline):
|
|
|
217
335
|
return row
|
|
218
336
|
|
|
219
337
|
|
|
338
|
+
def discover(platform, config, deadline):
|
|
339
|
+
if config.get("mode", "username") == "username":
|
|
340
|
+
return direct_profile(platform, config, deadline)
|
|
341
|
+
row = discover_search(platform, config, deadline)
|
|
342
|
+
row["method"] = "web_search"
|
|
343
|
+
return row
|
|
344
|
+
|
|
345
|
+
|
|
220
346
|
def search(config, emit):
|
|
221
347
|
username = config["username"]
|
|
222
348
|
if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}", username):
|
|
223
349
|
raise ValueError("Invalid username")
|
|
224
350
|
platforms = config["platforms"]
|
|
351
|
+
if config.get("mode", "username") not in {"username", "web_search"}:
|
|
352
|
+
raise ValueError("Invalid search mode")
|
|
225
353
|
if (not platforms or len(platforms) > 3 or any(p not in PLATFORMS for p in platforms)
|
|
226
354
|
or not 1 <= config["limit_per_platform"] <= 3):
|
|
227
355
|
raise ValueError("Invalid platform selection")
|