sherlock-mcp 1.1.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sherlock-mcp",
3
- "version": "1.1.1",
3
+ "version": "2.0.0",
4
4
  "description": "Sherlock username discovery over stdio MCP, with an isolated Python runtime and proxy support",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -27,7 +27,9 @@ BING_SEARCH_HOSTS = {"www.bing.com", "cn.bing.com"}
27
27
 
28
28
 
29
29
  class DiscoveryError(Exception):
30
- pass
30
+ def __init__(self, code, http_status=None):
31
+ super().__init__(code)
32
+ self.http_status = http_status
31
33
 
32
34
 
33
35
  def now():
@@ -101,7 +103,8 @@ def failure(error):
101
103
  return "timeout" if "timeout" in type(error).__name__.lower() else "request_failed"
102
104
 
103
105
 
104
- def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
106
+ def get(session, url, config, deadline, *, params=None, allowed_redirect=None,
107
+ return_metadata=False, allow_not_found=False):
105
108
  """Bound every read and check redirects BEFORE sending the next request."""
106
109
  proxy = config.get("proxy")
107
110
  for _ in range(4):
@@ -119,9 +122,9 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
119
122
  url = target
120
123
  continue
121
124
  if response.status_code in {401, 403, 407, 429, 999}:
122
- raise DiscoveryError("blocked")
123
- if response.status_code >= 400:
124
- raise DiscoveryError("http_error")
125
+ raise DiscoveryError("blocked", response.status_code)
126
+ if response.status_code >= 400 and not (allow_not_found and response.status_code in {404, 410}):
127
+ raise DiscoveryError("http_error", response.status_code)
125
128
  chunks, size = [], 0
126
129
  for chunk in response.iter_content(16384):
127
130
  if time.monotonic() >= deadline:
@@ -130,7 +133,8 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
130
133
  if size > MAX_BYTES:
131
134
  raise DiscoveryError("response_too_large")
132
135
  chunks.append(chunk)
133
- return b"".join(chunks), response.headers.get("Content-Type", ""), response.encoding or "utf-8"
136
+ result = b"".join(chunks), response.headers.get("Content-Type", ""), response.encoding or "utf-8"
137
+ return (*result, response.status_code, response.url) if return_metadata else result
134
138
  raise DiscoveryError("redirect_restricted")
135
139
 
136
140
 
@@ -177,7 +181,107 @@ def search_hits(session, query, config, deadline):
177
181
  "snippet": item.findtext("description", "")} for item in root.findall("./channel/item")[:20]], False, "bing_rss"
178
182
 
179
183
 
180
- def discover(platform, config, deadline):
184
+ class ProfileEvidence(HTMLParser):
185
+ """Read explicit public profile metadata; HTTP 200 alone is not a hit."""
186
+ def __init__(self):
187
+ super().__init__(convert_charrefs=True)
188
+ self.meta = {}
189
+ self.canonical = []
190
+
191
+ def handle_starttag(self, tag, attrs):
192
+ attrs = dict(attrs)
193
+ if tag == "meta":
194
+ key = (attrs.get("property") or attrs.get("name") or "").lower()
195
+ self.meta[key] = attrs.get("content", "")
196
+ if tag == "link" and "canonical" in (attrs.get("rel") or "").lower().split():
197
+ self.canonical.append(attrs.get("href", ""))
198
+
199
+
200
+ def direct_profile(platform, config, deadline):
201
+ import requests
202
+ username = config["username"]
203
+ base = {"facebook": "https://www.facebook.com/", "twitter": "https://x.com/",
204
+ "linkedin": "https://www.linkedin.com/in/"}[platform]
205
+ valid = profile_url(base + username, platform)
206
+ row = {"platform": platform, "method": "direct_username", "username": username,
207
+ "url": valid[0] if valid else "", "checked_at": None, "status": "ILLEGAL" if not valid else "UNKNOWN",
208
+ "search_status": "not_applicable" if not valid else "failed", "search_provider": "none",
209
+ "http_status": None, "page_status": "not_requested", "identity_status": "unverified", "candidates": []}
210
+ if not valid:
211
+ row["reason"] = "invalid_for_site"
212
+ return row
213
+ row["checked_at"] = now()
214
+
215
+ def same_profile(target):
216
+ parts = urlsplit(target)
217
+ if parts.scheme != "https" or parts.port not in {None, 443}:
218
+ return False
219
+ candidate = profile_url(target, platform)
220
+ if candidate and candidate[0].casefold() == valid[0].casefold():
221
+ return True
222
+ # Do not follow login/checkpoint redirects or borrow a browser session.
223
+ if parts.hostname in HOSTS[platform] and re.match(r"^/(?:login|checkpoint|authwall|i/flow/login)(?:[/.?]|$)", parts.path):
224
+ raise DiscoveryError("login_or_challenge")
225
+ return False
226
+
227
+ try:
228
+ with requests.Session() as session:
229
+ session.trust_env = False
230
+ session.headers.update({"User-Agent": "sherlock-mcp/2.0 public-profile-probe",
231
+ "Accept-Language": "en-US,en;q=0.8"})
232
+ raw, ctype, encoding, status, _final_url = get(
233
+ session, valid[0], config, deadline, allowed_redirect=same_profile,
234
+ return_metadata=True, allow_not_found=True)
235
+ row["http_status"] = status
236
+ if ctype and not any(t in ctype.lower() for t in ("text/html", "application/xhtml")):
237
+ raise DiscoveryError("unsupported_content_type")
238
+ html = raw.decode(encoding, errors="replace")
239
+ content = text_of(html)
240
+ if WALL.search(content):
241
+ raise DiscoveryError("login_or_challenge")
242
+ if status in {404, 410}:
243
+ row.update(status="AVAILABLE", search_status="no_matches", page_status="not_found", reason="http_not_found")
244
+ return row
245
+ evidence = ProfileEvidence()
246
+ evidence.feed(html)
247
+ meta = evidence.meta
248
+ canonical = any(value and value[0].casefold() == valid[0].casefold()
249
+ for value in (profile_url(u, platform) for u in [*evidence.canonical, meta.get("og:url", "")]))
250
+ title = meta.get("og:title", "").strip()
251
+ profile_type = meta.get("og:type", "").casefold() == "profile"
252
+ if platform == "facebook":
253
+ native = any(re.match(r"^fb://(?:profile|page)/\d+(?:[/?]|$)", meta.get(k, ""))
254
+ for k in ("al:ios:url", "al:android:url"))
255
+ elif platform == "twitter":
256
+ native = meta.get("twitter:creator", "").lstrip("@").casefold() == username.casefold()
257
+ else:
258
+ native = bool(meta.get("profile:first_name") and meta.get("profile:last_name"))
259
+ generic = title.casefold() in {"facebook", "linkedin", "twitter", "x", "log in", "sign in"}
260
+ if not (canonical and title and not generic and (profile_type or native)):
261
+ row.update(page_status="empty" if not content else "insufficient_profile_evidence",
262
+ reason="insufficient_profile_evidence")
263
+ return row
264
+ row.update(status="CLAIMED", search_status="candidates_found", page_status="fetched",
265
+ reason="public_profile_metadata", signals=["matching_profile_url", "public_profile_metadata"])
266
+ candidate = {"url": valid[0], "handle": valid[1], "discovery_method": "direct_username",
267
+ "title": title[:300], "exact_handle_match": True, "page_status": "fetched",
268
+ "http_status": status, "fetched_at": now(), "identity_status": "unverified"}
269
+ if config["fetch_profiles"]:
270
+ candidate["page_excerpt"] = content[:4000]
271
+ row["candidates"] = [candidate]
272
+ except Exception as exc:
273
+ reason = failure(exc)
274
+ status = getattr(exc, "http_status", None)
275
+ if status is not None:
276
+ row["http_status"] = status
277
+ if status == 407:
278
+ reason = "proxy_authentication_failed"
279
+ row.update(status="WAF" if reason in {"blocked", "login_or_challenge"} else "UNKNOWN",
280
+ page_status=reason, reason=reason)
281
+ return row
282
+
283
+
284
+ def discover_search(platform, config, deadline):
181
285
  import requests
182
286
  site = {"facebook": "site:facebook.com", "twitter": "(site:x.com OR site:twitter.com)",
183
287
  "linkedin": "site:linkedin.com/in/"}[platform]
@@ -231,11 +335,21 @@ def discover(platform, config, deadline):
231
335
  return row
232
336
 
233
337
 
338
+ def discover(platform, config, deadline):
339
+ if config.get("mode", "username") == "username":
340
+ return direct_profile(platform, config, deadline)
341
+ row = discover_search(platform, config, deadline)
342
+ row["method"] = "web_search"
343
+ return row
344
+
345
+
234
346
  def search(config, emit):
235
347
  username = config["username"]
236
348
  if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}", username):
237
349
  raise ValueError("Invalid username")
238
350
  platforms = config["platforms"]
351
+ if config.get("mode", "username") not in {"username", "web_search"}:
352
+ raise ValueError("Invalid search mode")
239
353
  if (not platforms or len(platforms) > 3 or any(p not in PLATFORMS for p in platforms)
240
354
  or not 1 <= config["limit_per_platform"] <= 3):
241
355
  raise ValueError("Invalid platform selection")