sherlock-mcp 1.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sherlock-mcp",
3
- "version": "1.1.0",
3
+ "version": "2.0.0",
4
4
  "description": "Sherlock username discovery over stdio MCP, with an isolated Python runtime and proxy support",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -23,10 +23,13 @@ WALL = re.compile(
23
23
  r"verify (?:you are|you're) human|security check|unusual activity|access denied|"
24
24
  r"temporarily blocked|登录后|登入後|安全验证|安全驗證|验证码|驗證碼", re.I)
25
25
  MAX_BYTES = 1_000_000
26
+ BING_SEARCH_HOSTS = {"www.bing.com", "cn.bing.com"}
26
27
 
27
28
 
28
29
  class DiscoveryError(Exception):
29
- pass
30
+ def __init__(self, code, http_status=None):
31
+ super().__init__(code)
32
+ self.http_status = http_status
30
33
 
31
34
 
32
35
  def now():
@@ -100,7 +103,8 @@ def failure(error):
100
103
  return "timeout" if "timeout" in type(error).__name__.lower() else "request_failed"
101
104
 
102
105
 
103
- def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
106
+ def get(session, url, config, deadline, *, params=None, allowed_redirect=None,
107
+ return_metadata=False, allow_not_found=False):
104
108
  """Bound every read and check redirects BEFORE sending the next request."""
105
109
  proxy = config.get("proxy")
106
110
  for _ in range(4):
@@ -118,9 +122,9 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
118
122
  url = target
119
123
  continue
120
124
  if response.status_code in {401, 403, 407, 429, 999}:
121
- raise DiscoveryError("blocked")
122
- if response.status_code >= 400:
123
- raise DiscoveryError("http_error")
125
+ raise DiscoveryError("blocked", response.status_code)
126
+ if response.status_code >= 400 and not (allow_not_found and response.status_code in {404, 410}):
127
+ raise DiscoveryError("http_error", response.status_code)
124
128
  chunks, size = [], 0
125
129
  for chunk in response.iter_content(16384):
126
130
  if time.monotonic() >= deadline:
@@ -129,10 +133,24 @@ def get(session, url, config, deadline, *, params=None, allowed_redirect=None):
129
133
  if size > MAX_BYTES:
130
134
  raise DiscoveryError("response_too_large")
131
135
  chunks.append(chunk)
132
- return b"".join(chunks), response.headers.get("Content-Type", ""), response.encoding or "utf-8"
136
+ result = b"".join(chunks), response.headers.get("Content-Type", ""), response.encoding or "utf-8"
137
+ return (*result, response.status_code, response.url) if return_metadata else result
133
138
  raise DiscoveryError("redirect_restricted")
134
139
 
135
140
 
141
+ def bing_search_redirect(url, query):
142
+ """Bing can select a regional endpoint; keep the original RSS query."""
143
+ try:
144
+ parts = urlsplit(url)
145
+ params = parse_qs(parts.query)
146
+ return (parts.scheme == "https" and parts.hostname in BING_SEARCH_HOSTS
147
+ and parts.port in {None, 443} and parts.username is None
148
+ and parts.password is None and parts.path == "/search" and not parts.fragment
149
+ and params.get("q") == [query] and params.get("format") == ["rss"])
150
+ except (ValueError, TypeError):
151
+ return False
152
+
153
+
136
154
  def search_hits(session, query, config, deadline):
137
155
  searx = config.get("searxng_url")
138
156
  if searx:
@@ -152,7 +170,7 @@ def search_hits(session, query, config, deadline):
152
170
  raw, content_type, _ = get(
153
171
  session, "https://www.bing.com/search", config, deadline,
154
172
  params={"q": query, "format": "rss", "count": 10},
155
- allowed_redirect=lambda u: urlsplit(u).scheme == "https" and urlsplit(u).hostname == "www.bing.com")
173
+ allowed_redirect=lambda u: bing_search_redirect(u, query))
156
174
  try:
157
175
  root = ET.fromstring(raw)
158
176
  except ET.ParseError:
@@ -163,7 +181,107 @@ def search_hits(session, query, config, deadline):
163
181
  "snippet": item.findtext("description", "")} for item in root.findall("./channel/item")[:20]], False, "bing_rss"
164
182
 
165
183
 
166
- def discover(platform, config, deadline):
184
+ class ProfileEvidence(HTMLParser):
185
+ """Read explicit public profile metadata; HTTP 200 alone is not a hit."""
186
+ def __init__(self):
187
+ super().__init__(convert_charrefs=True)
188
+ self.meta = {}
189
+ self.canonical = []
190
+
191
+ def handle_starttag(self, tag, attrs):
192
+ attrs = dict(attrs)
193
+ if tag == "meta":
194
+ key = (attrs.get("property") or attrs.get("name") or "").lower()
195
+ self.meta[key] = attrs.get("content", "")
196
+ if tag == "link" and "canonical" in (attrs.get("rel") or "").lower().split():
197
+ self.canonical.append(attrs.get("href", ""))
198
+
199
+
200
+ def direct_profile(platform, config, deadline):
201
+ import requests
202
+ username = config["username"]
203
+ base = {"facebook": "https://www.facebook.com/", "twitter": "https://x.com/",
204
+ "linkedin": "https://www.linkedin.com/in/"}[platform]
205
+ valid = profile_url(base + username, platform)
206
+ row = {"platform": platform, "method": "direct_username", "username": username,
207
+ "url": valid[0] if valid else "", "checked_at": None, "status": "ILLEGAL" if not valid else "UNKNOWN",
208
+ "search_status": "not_applicable" if not valid else "failed", "search_provider": "none",
209
+ "http_status": None, "page_status": "not_requested", "identity_status": "unverified", "candidates": []}
210
+ if not valid:
211
+ row["reason"] = "invalid_for_site"
212
+ return row
213
+ row["checked_at"] = now()
214
+
215
+ def same_profile(target):
216
+ parts = urlsplit(target)
217
+ if parts.scheme != "https" or parts.port not in {None, 443}:
218
+ return False
219
+ candidate = profile_url(target, platform)
220
+ if candidate and candidate[0].casefold() == valid[0].casefold():
221
+ return True
222
+ # Do not follow login/checkpoint redirects or borrow a browser session.
223
+ if parts.hostname in HOSTS[platform] and re.match(r"^/(?:login|checkpoint|authwall|i/flow/login)(?:[/.?]|$)", parts.path):
224
+ raise DiscoveryError("login_or_challenge")
225
+ return False
226
+
227
+ try:
228
+ with requests.Session() as session:
229
+ session.trust_env = False
230
+ session.headers.update({"User-Agent": "sherlock-mcp/2.0 public-profile-probe",
231
+ "Accept-Language": "en-US,en;q=0.8"})
232
+ raw, ctype, encoding, status, _final_url = get(
233
+ session, valid[0], config, deadline, allowed_redirect=same_profile,
234
+ return_metadata=True, allow_not_found=True)
235
+ row["http_status"] = status
236
+ if ctype and not any(t in ctype.lower() for t in ("text/html", "application/xhtml")):
237
+ raise DiscoveryError("unsupported_content_type")
238
+ html = raw.decode(encoding, errors="replace")
239
+ content = text_of(html)
240
+ if WALL.search(content):
241
+ raise DiscoveryError("login_or_challenge")
242
+ if status in {404, 410}:
243
+ row.update(status="AVAILABLE", search_status="no_matches", page_status="not_found", reason="http_not_found")
244
+ return row
245
+ evidence = ProfileEvidence()
246
+ evidence.feed(html)
247
+ meta = evidence.meta
248
+ canonical = any(value and value[0].casefold() == valid[0].casefold()
249
+ for value in (profile_url(u, platform) for u in [*evidence.canonical, meta.get("og:url", "")]))
250
+ title = meta.get("og:title", "").strip()
251
+ profile_type = meta.get("og:type", "").casefold() == "profile"
252
+ if platform == "facebook":
253
+ native = any(re.match(r"^fb://(?:profile|page)/\d+(?:[/?]|$)", meta.get(k, ""))
254
+ for k in ("al:ios:url", "al:android:url"))
255
+ elif platform == "twitter":
256
+ native = meta.get("twitter:creator", "").lstrip("@").casefold() == username.casefold()
257
+ else:
258
+ native = bool(meta.get("profile:first_name") and meta.get("profile:last_name"))
259
+ generic = title.casefold() in {"facebook", "linkedin", "twitter", "x", "log in", "sign in"}
260
+ if not (canonical and title and not generic and (profile_type or native)):
261
+ row.update(page_status="empty" if not content else "insufficient_profile_evidence",
262
+ reason="insufficient_profile_evidence")
263
+ return row
264
+ row.update(status="CLAIMED", search_status="candidates_found", page_status="fetched",
265
+ reason="public_profile_metadata", signals=["matching_profile_url", "public_profile_metadata"])
266
+ candidate = {"url": valid[0], "handle": valid[1], "discovery_method": "direct_username",
267
+ "title": title[:300], "exact_handle_match": True, "page_status": "fetched",
268
+ "http_status": status, "fetched_at": now(), "identity_status": "unverified"}
269
+ if config["fetch_profiles"]:
270
+ candidate["page_excerpt"] = content[:4000]
271
+ row["candidates"] = [candidate]
272
+ except Exception as exc:
273
+ reason = failure(exc)
274
+ status = getattr(exc, "http_status", None)
275
+ if status is not None:
276
+ row["http_status"] = status
277
+ if status == 407:
278
+ reason = "proxy_authentication_failed"
279
+ row.update(status="WAF" if reason in {"blocked", "login_or_challenge"} else "UNKNOWN",
280
+ page_status=reason, reason=reason)
281
+ return row
282
+
283
+
284
+ def discover_search(platform, config, deadline):
167
285
  import requests
168
286
  site = {"facebook": "site:facebook.com", "twitter": "(site:x.com OR site:twitter.com)",
169
287
  "linkedin": "site:linkedin.com/in/"}[platform]
@@ -217,11 +335,21 @@ def discover(platform, config, deadline):
217
335
  return row
218
336
 
219
337
 
338
+ def discover(platform, config, deadline):
339
+ if config.get("mode", "username") == "username":
340
+ return direct_profile(platform, config, deadline)
341
+ row = discover_search(platform, config, deadline)
342
+ row["method"] = "web_search"
343
+ return row
344
+
345
+
220
346
  def search(config, emit):
221
347
  username = config["username"]
222
348
  if not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}", username):
223
349
  raise ValueError("Invalid username")
224
350
  platforms = config["platforms"]
351
+ if config.get("mode", "username") not in {"username", "web_search"}:
352
+ raise ValueError("Invalid search mode")
225
353
  if (not platforms or len(platforms) > 3 or any(p not in PLATFORMS for p in platforms)
226
354
  or not 1 <= config["limit_per_platform"] <= 3):
227
355
  raise ValueError("Invalid platform selection")