sherlock-mcp 1.1.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -18
- package/dist/cli.cjs +19 -19
- package/package.json +1 -1
- package/python/social_profiles.py +121 -7
package/package.json
CHANGED
|
@@ -27,7 +27,9 @@ BING_SEARCH_HOSTS = {"www.bing.com", "cn.bing.com"}
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class DiscoveryError(Exception):
|
|
30
|
-
|
|
30
|
+
def __init__(self, code, http_status=None):
|
|
31
|
+
super().__init__(code)
|
|
32
|
+
self.http_status = http_status
|
|
31
33
|
|
|
32
34
|
|
|
33
35
|
def now():
|
|
@@ -101,7 +103,8 @@ def failure(error):
|
|
|
101
103
|
return "timeout" if "timeout" in type(error).__name__.lower() else "request_failed"
|
|
102
104
|
|
|
103
105
|
|
|
104
|
-
def get(session, url, config, deadline, *, params=None, allowed_redirect=None
|
|
106
|
+
def get(session, url, config, deadline, *, params=None, allowed_redirect=None,
|
|
107
|
+
return_metadata=False, allow_not_found=False):
|
|
105
108
|
"""Bound every read and check redirects BEFORE sending the next request."""
|
|
106
109
|
proxy = config.get("proxy")
|
|
107
110
|
for _ in range(4):
|
|
@@ -119,9 +122,9 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
|
|
|
119
122
|
url = target
|
|
120
123
|
continue
|
|
121
124
|
if response.status_code in {401, 403, 407, 429, 999}:
|
|
122
|
-
raise DiscoveryError("blocked")
|
|
123
|
-
if response.status_code >= 400:
|
|
124
|
-
raise DiscoveryError("http_error")
|
|
125
|
+
raise DiscoveryError("blocked", response.status_code)
|
|
126
|
+
if response.status_code >= 400 and not (allow_not_found and response.status_code in {404, 410}):
|
|
127
|
+
raise DiscoveryError("http_error", response.status_code)
|
|
125
128
|
chunks, size = [], 0
|
|
126
129
|
for chunk in response.iter_content(16384):
|
|
127
130
|
if time.monotonic() >= deadline:
|
|
@@ -130,7 +133,8 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
|
|
|
130
133
|
if size > MAX_BYTES:
|
|
131
134
|
raise DiscoveryError("response_too_large")
|
|
132
135
|
chunks.append(chunk)
|
|
133
|
-
|
|
136
|
+
result = b"".join(chunks), response.headers.get("Content-Type", ""), response.encoding or "utf-8"
|
|
137
|
+
return (*result, response.status_code, response.url) if return_metadata else result
|
|
134
138
|
raise DiscoveryError("redirect_restricted")
|
|
135
139
|
|
|
136
140
|
|
|
@@ -177,7 +181,107 @@ def search_hits(session, query, config, deadline):
|
|
|
177
181
|
"snippet": item.findtext("description", "")} for item in root.findall("./channel/item")[:20]], False, "bing_rss"
|
|
178
182
|
|
|
179
183
|
|
|
180
|
-
|
|
184
|
+
class ProfileEvidence(HTMLParser):
|
|
185
|
+
"""Read explicit public profile metadata; HTTP 200 alone is not a hit."""
|
|
186
|
+
def __init__(self):
|
|
187
|
+
super().__init__(convert_charrefs=True)
|
|
188
|
+
self.meta = {}
|
|
189
|
+
self.canonical = []
|
|
190
|
+
|
|
191
|
+
def handle_starttag(self, tag, attrs):
|
|
192
|
+
attrs = dict(attrs)
|
|
193
|
+
if tag == "meta":
|
|
194
|
+
key = (attrs.get("property") or attrs.get("name") or "").lower()
|
|
195
|
+
self.meta[key] = attrs.get("content", "")
|
|
196
|
+
if tag == "link" and "canonical" in (attrs.get("rel") or "").lower().split():
|
|
197
|
+
self.canonical.append(attrs.get("href", ""))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def direct_profile(platform, config, deadline):
|
|
201
|
+
import requests
|
|
202
|
+
username = config["username"]
|
|
203
|
+
base = {"facebook": "https://www.facebook.com/", "twitter": "https://x.com/",
|
|
204
|
+
"linkedin": "https://www.linkedin.com/in/"}[platform]
|
|
205
|
+
valid = profile_url(base + username, platform)
|
|
206
|
+
row = {"platform": platform, "method": "direct_username", "username": username,
|
|
207
|
+
"url": valid[0] if valid else "", "checked_at": None, "status": "ILLEGAL" if not valid else "UNKNOWN",
|
|
208
|
+
"search_status": "not_applicable" if not valid else "failed", "search_provider": "none",
|
|
209
|
+
"http_status": None, "page_status": "not_requested", "identity_status": "unverified", "candidates": []}
|
|
210
|
+
if not valid:
|
|
211
|
+
row["reason"] = "invalid_for_site"
|
|
212
|
+
return row
|
|
213
|
+
row["checked_at"] = now()
|
|
214
|
+
|
|
215
|
+
def same_profile(target):
|
|
216
|
+
parts = urlsplit(target)
|
|
217
|
+
if parts.scheme != "https" or parts.port not in {None, 443}:
|
|
218
|
+
return False
|
|
219
|
+
candidate = profile_url(target, platform)
|
|
220
|
+
if candidate and candidate[0].casefold() == valid[0].casefold():
|
|
221
|
+
return True
|
|
222
|
+
# Do not follow login/checkpoint redirects or borrow a browser session.
|
|
223
|
+
if parts.hostname in HOSTS[platform] and re.match(r"^/(?:login|checkpoint|authwall|i/flow/login)(?:[/.?]|$)", parts.path):
|
|
224
|
+
raise DiscoveryError("login_or_challenge")
|
|
225
|
+
return False
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
with requests.Session() as session:
|
|
229
|
+
session.trust_env = False
|
|
230
|
+
session.headers.update({"User-Agent": "sherlock-mcp/2.0 public-profile-probe",
|
|
231
|
+
"Accept-Language": "en-US,en;q=0.8"})
|
|
232
|
+
raw, ctype, encoding, status, _final_url = get(
|
|
233
|
+
session, valid[0], config, deadline, allowed_redirect=same_profile,
|
|
234
|
+
return_metadata=True, allow_not_found=True)
|
|
235
|
+
row["http_status"] = status
|
|
236
|
+
if ctype and not any(t in ctype.lower() for t in ("text/html", "application/xhtml")):
|
|
237
|
+
raise DiscoveryError("unsupported_content_type")
|
|
238
|
+
html = raw.decode(encoding, errors="replace")
|
|
239
|
+
content = text_of(html)
|
|
240
|
+
if WALL.search(content):
|
|
241
|
+
raise DiscoveryError("login_or_challenge")
|
|
242
|
+
if status in {404, 410}:
|
|
243
|
+
row.update(status="AVAILABLE", search_status="no_matches", page_status="not_found", reason="http_not_found")
|
|
244
|
+
return row
|
|
245
|
+
evidence = ProfileEvidence()
|
|
246
|
+
evidence.feed(html)
|
|
247
|
+
meta = evidence.meta
|
|
248
|
+
canonical = any(value and value[0].casefold() == valid[0].casefold()
|
|
249
|
+
for value in (profile_url(u, platform) for u in [*evidence.canonical, meta.get("og:url", "")]))
|
|
250
|
+
title = meta.get("og:title", "").strip()
|
|
251
|
+
profile_type = meta.get("og:type", "").casefold() == "profile"
|
|
252
|
+
if platform == "facebook":
|
|
253
|
+
native = any(re.match(r"^fb://(?:profile|page)/\d+(?:[/?]|$)", meta.get(k, ""))
|
|
254
|
+
for k in ("al:ios:url", "al:android:url"))
|
|
255
|
+
elif platform == "twitter":
|
|
256
|
+
native = meta.get("twitter:creator", "").lstrip("@").casefold() == username.casefold()
|
|
257
|
+
else:
|
|
258
|
+
native = bool(meta.get("profile:first_name") and meta.get("profile:last_name"))
|
|
259
|
+
generic = title.casefold() in {"facebook", "linkedin", "twitter", "x", "log in", "sign in"}
|
|
260
|
+
if not (canonical and title and not generic and (profile_type or native)):
|
|
261
|
+
row.update(page_status="empty" if not content else "insufficient_profile_evidence",
|
|
262
|
+
reason="insufficient_profile_evidence")
|
|
263
|
+
return row
|
|
264
|
+
row.update(status="CLAIMED", search_status="candidates_found", page_status="fetched",
|
|
265
|
+
reason="public_profile_metadata", signals=["matching_profile_url", "public_profile_metadata"])
|
|
266
|
+
candidate = {"url": valid[0], "handle": valid[1], "discovery_method": "direct_username",
|
|
267
|
+
"title": title[:300], "exact_handle_match": True, "page_status": "fetched",
|
|
268
|
+
"http_status": status, "fetched_at": now(), "identity_status": "unverified"}
|
|
269
|
+
if config["fetch_profiles"]:
|
|
270
|
+
candidate["page_excerpt"] = content[:4000]
|
|
271
|
+
row["candidates"] = [candidate]
|
|
272
|
+
except Exception as exc:
|
|
273
|
+
reason = failure(exc)
|
|
274
|
+
status = getattr(exc, "http_status", None)
|
|
275
|
+
if status is not None:
|
|
276
|
+
row["http_status"] = status
|
|
277
|
+
if status == 407:
|
|
278
|
+
reason = "proxy_authentication_failed"
|
|
279
|
+
row.update(status="WAF" if reason in {"blocked", "login_or_challenge"} else "UNKNOWN",
|
|
280
|
+
page_status=reason, reason=reason)
|
|
281
|
+
return row
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def discover_search(platform, config, deadline):
|
|
181
285
|
import requests
|
|
182
286
|
site = {"facebook": "site:facebook.com", "twitter": "(site:x.com OR site:twitter.com)",
|
|
183
287
|
"linkedin": "site:linkedin.com/in/"}[platform]
|
|
@@ -231,11 +335,21 @@ def discover(platform, config, deadline):
|
|
|
231
335
|
return row
|
|
232
336
|
|
|
233
337
|
|
|
338
|
+
def discover(platform, config, deadline):
|
|
339
|
+
if config.get("mode", "username") == "username":
|
|
340
|
+
return direct_profile(platform, config, deadline)
|
|
341
|
+
row = discover_search(platform, config, deadline)
|
|
342
|
+
row["method"] = "web_search"
|
|
343
|
+
return row
|
|
344
|
+
|
|
345
|
+
|
|
234
346
|
def search(config, emit):
|
|
235
347
|
username = config["username"]
|
|
236
348
|
if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}", username):
|
|
237
349
|
raise ValueError("Invalid username")
|
|
238
350
|
platforms = config["platforms"]
|
|
351
|
+
if config.get("mode", "username") not in {"username", "web_search"}:
|
|
352
|
+
raise ValueError("Invalid search mode")
|
|
239
353
|
if (not platforms or len(platforms) > 3 or any(p not in PLATFORMS for p in platforms)
|
|
240
354
|
or not 1 <= config["limit_per_platform"] <= 3):
|
|
241
355
|
raise ValueError("Invalid platform selection")
|