osintengine 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osintengine-1.0.2.dist-info/METADATA +311 -0
- osintengine-1.0.2.dist-info/RECORD +103 -0
- osintengine-1.0.2.dist-info/WHEEL +5 -0
- osintengine-1.0.2.dist-info/entry_points.txt +2 -0
- osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
- osintengine-1.0.2.dist-info/top_level.txt +2 -0
- src/watson/__init__.py +10 -0
- src/watson/agent/__init__.py +96 -0
- src/watson/agents/__init__.py +101 -0
- src/watson/agents/protocol.py +196 -0
- src/watson/auth/__init__.py +68 -0
- src/watson/auth/store.py +145 -0
- src/watson/conversation.py +68 -0
- src/watson/core/__init__.py +0 -0
- src/watson/core/models.py +96 -0
- src/watson/ethics.py +205 -0
- src/watson/exports.py +182 -0
- src/watson/graph/__init__.py +69 -0
- src/watson/graph/entities.py +207 -0
- src/watson/graph/graph.py +489 -0
- src/watson/graph/osint_framework.py +266 -0
- src/watson/graph/relationships.py +116 -0
- src/watson/graph/transforms.py +787 -0
- src/watson/infra/__init__.py +43 -0
- src/watson/infra/cache.py +171 -0
- src/watson/infra/ratelimit.py +125 -0
- src/watson/infra/resilience.py +239 -0
- src/watson/infra/retry.py +186 -0
- src/watson/metrics.py +128 -0
- src/watson/opsec/__init__.py +290 -0
- src/watson/orchestration/__init__.py +23 -0
- src/watson/orchestration/engine.py +5587 -0
- src/watson/orchestration/executor.py +96 -0
- src/watson/orchestration/intent.py +139 -0
- src/watson/orchestration/intent_classifier.py +45 -0
- src/watson/orchestration/llm_config.py +160 -0
- src/watson/orchestration/resolution.py +555 -0
- src/watson/orchestration/scraper.py +94 -0
- src/watson/orchestration/synthesis.py +726 -0
- src/watson/orchestration/target_profile.py +748 -0
- src/watson/persistence/__init__.py +18 -0
- src/watson/persistence/models.py +161 -0
- src/watson/persistence/store.py +230 -0
- src/watson/pipeline/__init__.py +16 -0
- src/watson/pipeline/pre_synthesis.py +621 -0
- src/watson/search.py +103 -0
- src/watson/serializers/stix.py +451 -0
- src/watson/tools/__init__.py +31 -0
- src/watson/tools/base.py +97 -0
- src/watson/tools/blockchain.py +359 -0
- src/watson/tools/captcha.py +276 -0
- src/watson/tools/conflict.py +107 -0
- src/watson/tools/corporate.py +287 -0
- src/watson/tools/darkweb.py +144 -0
- src/watson/tools/geolocation.py +347 -0
- src/watson/tools/image_video.py +108 -0
- src/watson/tools/marinetraffic.py +142 -0
- src/watson/tools/people.py +476 -0
- src/watson/tools/registry.py +56 -0
- src/watson/tools/satellite.py +118 -0
- src/watson/tools/scraper.py +745 -0
- src/watson/tools/shodan.py +129 -0
- src/watson/tools/social_media.py +160 -0
- src/watson/tools/websites.py +291 -0
- src/watson/tools/wikidata.py +398 -0
- src/watson/utils/__init__.py +1 -0
- src/watson/utils/helpers.py +49 -0
- src/watson/utils/http.py +119 -0
- src/watson/verification/__init__.py +245 -0
- watson/__init__.py +6 -0
- watson/agents/__init__.py +20 -0
- watson/agents/base.py +103 -0
- watson/agents/direct.py +225 -0
- watson/agents/hermes.py +275 -0
- watson/agents/hermes_mcp.py +292 -0
- watson/api_keys.py +176 -0
- watson/browser_scraper.py +481 -0
- watson/cli.py +836 -0
- watson/crossref.py +175 -0
- watson/ethics.py +20 -0
- watson/graph.py +406 -0
- watson/mcp_server.py +601 -0
- watson/memory.py +552 -0
- watson/neo4j_graph.py +124 -0
- watson/opsec/__init__.py +307 -0
- watson/reporter.py +537 -0
- watson/scheduler.py +486 -0
- watson/serializers/stix.py +451 -0
- watson/terminal.py +410 -0
- watson/toolkit.py +252 -0
- watson/toolkit_api.py +347 -0
- watson/toolkit_automation.py +95 -0
- watson/toolkit_registry.py +345 -0
- watson/verification/__init__.py +251 -0
- watson/web/__init__.py +1 -0
- watson/web/app.py +1650 -0
- watson/web/middleware.py +301 -0
- watson/web/static/assets/index-B7hPOc0z.js +278 -0
- watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
- watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
- watson/web/static/index.html +14 -0
- watson/web/templates/chat.html +2 -0
- watson/web/templates/investigation-map.html +429 -0
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
"""People search tool — username enumeration, email lookup, breach data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import logging
|
|
8
|
+
import urllib.parse
|
|
9
|
+
|
|
10
|
+
from .base import OSINTTool
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger("watson.people")
|
|
13
|
+
from .registry import registry
|
|
14
|
+
from ..core.models import Finding, FindingSource
|
|
15
|
+
from ..utils.http import get_client
|
|
16
|
+
from ..utils.helpers import is_email, clean_username
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PeopleTool(OSINTTool):
|
|
20
|
+
"""Search for individuals — username enumeration, email breach check, name search."""
|
|
21
|
+
|
|
22
|
+
category = FindingSource.PEOPLE
|
|
23
|
+
name = "people-search"
|
|
24
|
+
description = "Username enumeration, Have I Been Pwned check, email/name investigation"
|
|
25
|
+
free_tier_available = True
|
|
26
|
+
rate_limit_rps = 1.5
|
|
27
|
+
|
|
28
|
+
HIBP_API = "https://haveibeenpwned.com/api/v3/breachedaccount/{email}"
|
|
29
|
+
MAILCHECK_API = "https://api.mailcheck.ai/email/{email}"
|
|
30
|
+
|
|
31
|
+
async def investigate(self, query: str, context: str = "") -> list[Finding]:
|
|
32
|
+
findings: list[Finding] = []
|
|
33
|
+
client = get_client(rate_limit=self.rate_limit_rps)
|
|
34
|
+
|
|
35
|
+
emails = self._extract_emails(query)
|
|
36
|
+
|
|
37
|
+
for email in emails[:3]: # Max 3 emails
|
|
38
|
+
# Full email analysis — type, provider, risk profile
|
|
39
|
+
analysis = self._analyze_email(email)
|
|
40
|
+
if analysis:
|
|
41
|
+
findings.append(analysis)
|
|
42
|
+
|
|
43
|
+
# Check HIBP (fast, 5s timeout)
|
|
44
|
+
hibp_result = await self._check_hibp(client, email)
|
|
45
|
+
if hibp_result:
|
|
46
|
+
findings.append(hibp_result)
|
|
47
|
+
|
|
48
|
+
# Check if email is disposable (fast, 5s timeout)
|
|
49
|
+
disp_result = await self._check_disposable(client, email)
|
|
50
|
+
if disp_result:
|
|
51
|
+
findings.append(disp_result)
|
|
52
|
+
|
|
53
|
+
# Username enumeration — actually search platforms, don't just give guidance
|
|
54
|
+
usernames = self._extract_usernames(query)
|
|
55
|
+
if usernames:
|
|
56
|
+
uname = clean_username(usernames[0])
|
|
57
|
+
# Skip username search for role-based accounts — "info", "admin", etc. are useless
|
|
58
|
+
_ROLE_USERNAMES = {"info", "admin", "support", "sales", "contact", "hello",
|
|
59
|
+
"help", "noreply", "no-reply", "postmaster", "abuse",
|
|
60
|
+
"security", "webmaster", "hostmaster", "billing", "jobs",
|
|
61
|
+
"careers", "hr", "press", "media", "marketing", "office",
|
|
62
|
+
"service", "team", "newsletter", "notifications"}
|
|
63
|
+
if uname.lower() not in _ROLE_USERNAMES:
|
|
64
|
+
findings.append(
|
|
65
|
+
self._make_finding(
|
|
66
|
+
title=f"👤 Username search: '{uname}' across platforms",
|
|
67
|
+
description=(
|
|
68
|
+
f"Searching for username '{uname}' across platforms — "
|
|
69
|
+
f"real people leave digital footprints. Use the links below "
|
|
70
|
+
f"to manually verify any results."
|
|
71
|
+
),
|
|
72
|
+
evidence=[
|
|
73
|
+
f"https://www.google.com/search?q=%22{uname}%22+site%3Agithub.com+OR+site%3Alinkedin.com+OR+site%3Atwitter.com+OR+site%3Areddit.com",
|
|
74
|
+
f"https://whatsmyname.app/?q={uname}",
|
|
75
|
+
],
|
|
76
|
+
confidence=0.6,
|
|
77
|
+
username=uname,
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
# Actually search the platforms via DDG — don't just give links
|
|
82
|
+
try:
|
|
83
|
+
from ddgs import DDGS
|
|
84
|
+
# Search both concatenated username AND full name with spaces
|
|
85
|
+
person_name = self._extract_person_name(query)
|
|
86
|
+
full_name_str = person_name if person_name else uname
|
|
87
|
+
platform_queries = []
|
|
88
|
+
# If we have a full name, search for it across platforms
|
|
89
|
+
if person_name and person_name.lower() != uname.lower():
|
|
90
|
+
platform_queries = [
|
|
91
|
+
f'"{person_name}" linkedin',
|
|
92
|
+
f'"{person_name}" github OR twitter',
|
|
93
|
+
f'"{person_name}" journalist OR analyst OR author OR editor',
|
|
94
|
+
]
|
|
95
|
+
else:
|
|
96
|
+
# Fallback: search by username
|
|
97
|
+
platform_queries = [
|
|
98
|
+
f'"{uname}" linkedin',
|
|
99
|
+
f'"{uname}" github OR twitter',
|
|
100
|
+
f'"{uname}" journalist OR analyst OR author OR editor',
|
|
101
|
+
]
|
|
102
|
+
def _ddg_search(q):
|
|
103
|
+
try:
|
|
104
|
+
with DDGS() as ddgs:
|
|
105
|
+
return list(ddgs.text(q, max_results=3))
|
|
106
|
+
except Exception:
|
|
107
|
+
return []
|
|
108
|
+
all_raw = await asyncio.gather(*[
|
|
109
|
+
asyncio.to_thread(_ddg_search, q) for q in platform_queries
|
|
110
|
+
])
|
|
111
|
+
seen = set()
|
|
112
|
+
# URL patterns to EXCLUDE (login pages, help pages, homepages)
|
|
113
|
+
_NOISE_PATTERNS = [
|
|
114
|
+
'/login', '/signin', '/signup', '/help/', '/answer/',
|
|
115
|
+
'/company/', '/school/', '/jobs', '/pulse/',
|
|
116
|
+
]
|
|
117
|
+
def _is_noise_url(url: str) -> bool:
|
|
118
|
+
url_lower = url.lower()
|
|
119
|
+
# Root domains with no path
|
|
120
|
+
if url_lower.rstrip('/') in (
|
|
121
|
+
'https://www.linkedin.com', 'https://linkedin.com',
|
|
122
|
+
'https://github.com', 'https://www.github.com',
|
|
123
|
+
):
|
|
124
|
+
return True
|
|
125
|
+
for pat in _NOISE_PATTERNS:
|
|
126
|
+
if pat in url_lower:
|
|
127
|
+
return True
|
|
128
|
+
return False
|
|
129
|
+
|
|
130
|
+
for raw_list in all_raw:
|
|
131
|
+
for r in raw_list:
|
|
132
|
+
href = r.get("href", "")
|
|
133
|
+
title = r.get("title", "")[:150]
|
|
134
|
+
body = r.get("body", "")[:200]
|
|
135
|
+
if not href or href in seen:
|
|
136
|
+
continue
|
|
137
|
+
# Filter noise URLs
|
|
138
|
+
if _is_noise_url(href):
|
|
139
|
+
continue
|
|
140
|
+
# Filter: result must mention the person's name in title or body
|
|
141
|
+
name_parts = person_name.lower().split() if person_name else [uname.lower()]
|
|
142
|
+
combined = (title + " " + body).lower()
|
|
143
|
+
name_mentioned = any(part in combined for part in name_parts)
|
|
144
|
+
if not name_mentioned:
|
|
145
|
+
continue
|
|
146
|
+
seen.add(href)
|
|
147
|
+
findings.append(self._make_finding(
|
|
148
|
+
title=f"🔍 {title}",
|
|
149
|
+
description=body,
|
|
150
|
+
evidence=[href],
|
|
151
|
+
confidence=0.7,
|
|
152
|
+
username=uname,
|
|
153
|
+
))
|
|
154
|
+
if seen:
|
|
155
|
+
logger.info("username_search_found: %s → %d results", uname, len(seen))
|
|
156
|
+
except Exception as e:
|
|
157
|
+
logger.warning("username_search_failed: %s", e)
|
|
158
|
+
|
|
159
|
+
# Fallback: bare name lookup (e.g., "John Smith")
|
|
160
|
+
if not findings:
|
|
161
|
+
person_name = self._extract_person_name(query)
|
|
162
|
+
if person_name:
|
|
163
|
+
encoded = urllib.parse.quote(person_name)
|
|
164
|
+
findings.append(
|
|
165
|
+
self._make_finding(
|
|
166
|
+
title=f"👤 Person: {person_name}",
|
|
167
|
+
description=(
|
|
168
|
+
f"Searching for **{person_name}** across public records and platforms:\n"
|
|
169
|
+
f"- [Google search](https://www.google.com/search?q=%22{encoded}%22)\n"
|
|
170
|
+
f"- [LinkedIn](https://www.linkedin.com/search/results/people/?keywords={encoded})\n"
|
|
171
|
+
f"- [OpenSanctions](https://opensanctions.org/search/?q={encoded})\n"
|
|
172
|
+
f"- [Wikipedia](https://en.wikipedia.org/wiki/Special:Search?search={encoded})\n\n"
|
|
173
|
+
f"To investigate further, try:\n"
|
|
174
|
+
f"`watson investigate \"{person_name} email\"` for email breach check\n"
|
|
175
|
+
f"`watson investigate \"{person_name} company\"` for corporate ties"
|
|
176
|
+
),
|
|
177
|
+
evidence=[
|
|
178
|
+
f"https://www.google.com/search?q=%22{encoded}%22",
|
|
179
|
+
f"https://www.linkedin.com/search/results/people/?keywords={encoded}",
|
|
180
|
+
f"https://opensanctions.org/search/?q={encoded}",
|
|
181
|
+
],
|
|
182
|
+
confidence=0.5,
|
|
183
|
+
person_name=person_name,
|
|
184
|
+
)
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
return findings
|
|
188
|
+
|
|
189
|
+
async def _check_hibp(self, client, email: str) -> Finding | None:
|
|
190
|
+
"""Check email against Have I Been Pwned — async with 5s timeout."""
|
|
191
|
+
import httpx
|
|
192
|
+
try:
|
|
193
|
+
async with httpx.AsyncClient(timeout=5.0) as raw:
|
|
194
|
+
resp = await raw.get(
|
|
195
|
+
self.HIBP_API.format(email=email),
|
|
196
|
+
headers={
|
|
197
|
+
"hibp-api-key": "",
|
|
198
|
+
"User-Agent": "WatsonOSINT/0.3",
|
|
199
|
+
},
|
|
200
|
+
)
|
|
201
|
+
if resp.status_code == 404:
|
|
202
|
+
return self._make_finding(
|
|
203
|
+
title=f"✅ No breaches found: {email}",
|
|
204
|
+
description="This email was not found in any known data breaches (HIBP).",
|
|
205
|
+
confidence=0.7,
|
|
206
|
+
email=email,
|
|
207
|
+
)
|
|
208
|
+
resp.raise_for_status()
|
|
209
|
+
breaches = resp.json()
|
|
210
|
+
if isinstance(breaches, list) and breaches:
|
|
211
|
+
breach_names = [b.get("Name", "Unknown") for b in breaches[:5]]
|
|
212
|
+
return self._make_finding(
|
|
213
|
+
title=f"⚠️ Breach alert: {email}",
|
|
214
|
+
description=(
|
|
215
|
+
f"Found in {len(breaches)} known data breaches: "
|
|
216
|
+
+ ", ".join(breach_names)
|
|
217
|
+
),
|
|
218
|
+
evidence=[f"https://haveibeenpwned.com/account/{email}"],
|
|
219
|
+
confidence=0.95,
|
|
220
|
+
email=email,
|
|
221
|
+
breach_count=len(breaches),
|
|
222
|
+
)
|
|
223
|
+
else:
|
|
224
|
+
return self._make_finding(
|
|
225
|
+
title=f"✅ No breaches found: {email}",
|
|
226
|
+
description="This email was not found in any known data breaches (HIBP).",
|
|
227
|
+
confidence=0.7,
|
|
228
|
+
email=email,
|
|
229
|
+
)
|
|
230
|
+
except Exception:
|
|
231
|
+
return None
|
|
232
|
+
|
|
233
|
+
async def _check_disposable(self, client, email: str) -> Finding | None:
|
|
234
|
+
"""Check if email is from a disposable provider — 5s timeout."""
|
|
235
|
+
import httpx
|
|
236
|
+
try:
|
|
237
|
+
async with httpx.AsyncClient(timeout=5.0) as raw:
|
|
238
|
+
resp = await raw.get(
|
|
239
|
+
self.MAILCHECK_API.format(email=email),
|
|
240
|
+
headers={"User-Agent": "WatsonOSINT/0.3"},
|
|
241
|
+
)
|
|
242
|
+
if resp.status_code != 200:
|
|
243
|
+
return None
|
|
244
|
+
data = resp.json()
|
|
245
|
+
if isinstance(data, dict) and data.get("disposable"):
|
|
246
|
+
return self._make_finding(
|
|
247
|
+
title=f"📧 Disposable email: {email}",
|
|
248
|
+
description="This appears to be a disposable/temporary email address.",
|
|
249
|
+
confidence=0.8,
|
|
250
|
+
email=email,
|
|
251
|
+
)
|
|
252
|
+
except Exception:
|
|
253
|
+
pass
|
|
254
|
+
return None
|
|
255
|
+
|
|
256
|
+
# ── Email analysis ──────────────────────────────────────
|
|
257
|
+
|
|
258
|
+
# Known email provider domains and their categories
|
|
259
|
+
_EMAIL_PROVIDERS = {
|
|
260
|
+
# Personal / free email providers
|
|
261
|
+
"gmail.com": ("Personal", "Google Gmail"),
|
|
262
|
+
"googlemail.com": ("Personal", "Google Gmail"),
|
|
263
|
+
"yahoo.com": ("Personal", "Yahoo Mail"),
|
|
264
|
+
"yahoo.fr": ("Personal", "Yahoo Mail (France)"),
|
|
265
|
+
"yahoo.co.uk": ("Personal", "Yahoo Mail (UK)"),
|
|
266
|
+
"outlook.com": ("Personal", "Microsoft Outlook"),
|
|
267
|
+
"hotmail.com": ("Personal", "Microsoft Hotmail"),
|
|
268
|
+
"live.com": ("Personal", "Microsoft Live"),
|
|
269
|
+
"msn.com": ("Personal", "Microsoft MSN"),
|
|
270
|
+
"icloud.com": ("Personal", "Apple iCloud"),
|
|
271
|
+
"me.com": ("Personal", "Apple iCloud"),
|
|
272
|
+
"mac.com": ("Personal", "Apple iCloud"),
|
|
273
|
+
"protonmail.com": ("Personal", "ProtonMail (encrypted)"),
|
|
274
|
+
"proton.me": ("Personal", "Proton (encrypted)"),
|
|
275
|
+
"tutanota.com": ("Personal", "Tutanota (encrypted)"),
|
|
276
|
+
"tuta.io": ("Personal", "Tutanota (encrypted)"),
|
|
277
|
+
"mail.ru": ("Personal", "Mail.ru (Russia)"),
|
|
278
|
+
"yandex.ru": ("Personal", "Yandex (Russia)"),
|
|
279
|
+
"yandex.com": ("Personal", "Yandex"),
|
|
280
|
+
"qq.com": ("Personal", "QQ Mail (China)"),
|
|
281
|
+
"163.com": ("Personal", "163 Mail (China)"),
|
|
282
|
+
"126.com": ("Personal", "126 Mail (China)"),
|
|
283
|
+
"naver.com": ("Personal", "Naver (Korea)"),
|
|
284
|
+
"daum.net": ("Personal", "Daum (Korea)"),
|
|
285
|
+
"rambler.ru": ("Personal", "Rambler (Russia)"),
|
|
286
|
+
"aol.com": ("Personal", "AOL"),
|
|
287
|
+
"fastmail.com": ("Personal", "Fastmail"),
|
|
288
|
+
"zoho.com": ("Personal", "Zoho Mail"),
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
_ROLE_PREFIXES = {
|
|
292
|
+
"admin", "info", "support", "sales", "contact", "hello", "help",
|
|
293
|
+
"noreply", "no-reply", "noreply", "donotreply", "postmaster",
|
|
294
|
+
"abuse", "security", "webmaster", "hostmaster", "billing",
|
|
295
|
+
"jobs", "careers", "hr", "press", "media", "marketing",
|
|
296
|
+
"office", "service", "team", "newsletter", "notifications",
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
def _analyze_email(self, email: str) -> Finding | None:
|
|
300
|
+
"""Categorize an email address — type, provider, risk indicators.
|
|
301
|
+
|
|
302
|
+
Pure local analysis (no API calls). Detects:
|
|
303
|
+
- Type: Personal / Corporate / Government / Educational / Role-based
|
|
304
|
+
- Provider (for personal emails: Gmail, Yahoo, ProtonMail, etc.)
|
|
305
|
+
- Plus addressing (user+tag@domain.com)
|
|
306
|
+
- Role account detection (admin@, info@, noreply@)
|
|
307
|
+
- Custom domain indicators
|
|
308
|
+
"""
|
|
309
|
+
import re
|
|
310
|
+
|
|
311
|
+
local, _, domain = email.partition("@")
|
|
312
|
+
if not domain:
|
|
313
|
+
return None
|
|
314
|
+
|
|
315
|
+
domain_lower = domain.lower().strip()
|
|
316
|
+
local_lower = local.lower().strip()
|
|
317
|
+
|
|
318
|
+
# ── Determine type ──
|
|
319
|
+
provider_info = self._EMAIL_PROVIDERS.get(domain_lower)
|
|
320
|
+
is_role = local_lower in self._ROLE_PREFIXES or any(
|
|
321
|
+
local_lower.startswith(p + ".") or local_lower.startswith(p + "-")
|
|
322
|
+
for p in self._ROLE_PREFIXES
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
# Plus addressing: user+tag@domain (Gmail, Fastmail, ProtonMail)
|
|
326
|
+
has_plus = "+" in local and not is_role
|
|
327
|
+
|
|
328
|
+
# TLD-based classification
|
|
329
|
+
tld = domain_lower.rsplit(".", 1)[-1] if "." in domain_lower else ""
|
|
330
|
+
|
|
331
|
+
email_type = "Unknown"
|
|
332
|
+
provider = "Custom domain"
|
|
333
|
+
details: list[str] = []
|
|
334
|
+
|
|
335
|
+
if provider_info:
|
|
336
|
+
email_type, provider = provider_info
|
|
337
|
+
elif tld in ("gov", "mil"):
|
|
338
|
+
email_type = "Government"
|
|
339
|
+
provider = f"Government domain (.{tld})"
|
|
340
|
+
elif tld == "edu":
|
|
341
|
+
email_type = "Educational"
|
|
342
|
+
provider = "Educational institution (.edu)"
|
|
343
|
+
elif tld in ("org", "ngo"):
|
|
344
|
+
email_type = "Organization"
|
|
345
|
+
provider = f"Organization domain (.{tld})"
|
|
346
|
+
elif is_role:
|
|
347
|
+
email_type = "Role-based"
|
|
348
|
+
provider = "Organizational role account"
|
|
349
|
+
else:
|
|
350
|
+
email_type = "Corporate"
|
|
351
|
+
provider = f"Custom corporate domain ({domain_lower})"
|
|
352
|
+
|
|
353
|
+
# ── Build risk indicators ──
|
|
354
|
+
risk_indicators: list[str] = []
|
|
355
|
+
if is_role:
|
|
356
|
+
risk_indicators.append("🔶 Role account — likely shared inbox, not a person")
|
|
357
|
+
if has_plus:
|
|
358
|
+
risk_indicators.append("🔹 Plus addressing — may have aliases (e.g., user+netflix@, user+bank@)")
|
|
359
|
+
if provider_info and provider_info[0] == "Personal":
|
|
360
|
+
risk_indicators.append("🟢 Personal email — common for individuals, harder to trace")
|
|
361
|
+
if email_type == "Corporate":
|
|
362
|
+
risk_indicators.append("🔵 Corporate domain — belongs to an organization, check WHOIS")
|
|
363
|
+
if email_type == "Government":
|
|
364
|
+
risk_indicators.append("🔴 Government email — official capacity")
|
|
365
|
+
if "encrypted" in provider.lower():
|
|
366
|
+
risk_indicators.append("🛡 Encrypted provider — enhanced privacy")
|
|
367
|
+
if provider_info and "russia" in provider.lower():
|
|
368
|
+
risk_indicators.append("⚠ Russian provider — subject to data localization laws")
|
|
369
|
+
if provider_info and "china" in provider.lower():
|
|
370
|
+
risk_indicators.append("⚠ Chinese provider — subject to data localization laws")
|
|
371
|
+
|
|
372
|
+
return self._make_finding(
|
|
373
|
+
title=f"📧 {email_type} email — {provider}",
|
|
374
|
+
description=f"**{email}**\nType: {email_type}\nProvider: {provider}\n"
|
|
375
|
+
+ (f"Domain: {domain_lower}\n" if email_type == "Corporate" else "")
|
|
376
|
+
+ ("\n".join(f"{r}" for r in risk_indicators) if risk_indicators else ""),
|
|
377
|
+
confidence=0.95,
|
|
378
|
+
email=email,
|
|
379
|
+
email_type=email_type,
|
|
380
|
+
provider=provider,
|
|
381
|
+
is_role=is_role,
|
|
382
|
+
has_plus_addressing=has_plus,
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
def _extract_emails(self, text: str) -> list[str]:
|
|
386
|
+
"""Extract email addresses from text."""
|
|
387
|
+
import re
|
|
388
|
+
|
|
389
|
+
pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
|
|
390
|
+
return list(dict.fromkeys(re.findall(pattern, text)))
|
|
391
|
+
|
|
392
|
+
def _extract_usernames(self, text: str) -> list[str]:
|
|
393
|
+
"""Extract potential usernames from text."""
|
|
394
|
+
import re
|
|
395
|
+
|
|
396
|
+
usernames = []
|
|
397
|
+
|
|
398
|
+
# If there's an email, extract the LOCAL PART as the primary username
|
|
399
|
+
# (e.g., "baron.lorenzo99" from "baron.lorenzo99@gmail.com")
|
|
400
|
+
email_pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
|
|
401
|
+
emails = re.findall(email_pattern, text)
|
|
402
|
+
for email in emails:
|
|
403
|
+
local = email.split("@")[0]
|
|
404
|
+
if local and len(local) >= 3:
|
|
405
|
+
usernames.append(local)
|
|
406
|
+
|
|
407
|
+
# @handles (filter out email domains)
|
|
408
|
+
KNOWN_DOMAINS = {"gmail", "yahoo", "hotmail", "outlook", "icloud", "protonmail",
|
|
409
|
+
"mail", "aol", "live", "msn", "yandex", "qq", "163"}
|
|
410
|
+
for match in re.finditer(r"@(\w{3,30})", text):
|
|
411
|
+
handle = match.group(1)
|
|
412
|
+
if handle.lower() not in KNOWN_DOMAINS:
|
|
413
|
+
usernames.append(handle)
|
|
414
|
+
|
|
415
|
+
# "username/handle X"
|
|
416
|
+
match = re.search(r"(?:username|handle|alias)\s+(?:is\s+)?['\"]?(\w{3,30})['\"]?", text, re.IGNORECASE)
|
|
417
|
+
if match:
|
|
418
|
+
usernames.append(match.group(1))
|
|
419
|
+
|
|
420
|
+
# Fallback: use full name (not just first word) as potential username
|
|
421
|
+
if not usernames:
|
|
422
|
+
# Try full name as a compound username (e.g., "PaoloTrecate", "paolotrecate")
|
|
423
|
+
words = re.findall(r'[A-Za-z][a-z]+', text)
|
|
424
|
+
if len(words) >= 2:
|
|
425
|
+
# FullName, firstlast, first_last, first-last
|
|
426
|
+
full_name = ''.join(words)
|
|
427
|
+
usernames.append(full_name)
|
|
428
|
+
usernames.append(full_name.lower())
|
|
429
|
+
usernames.append('_'.join(words).lower())
|
|
430
|
+
usernames.append('-'.join(words).lower())
|
|
431
|
+
else:
|
|
432
|
+
# Single word — only use if it's distinctive (≥6 chars)
|
|
433
|
+
word_match = re.search(r'\b([A-Za-z][A-Za-z0-9_]{5,30})\b', text)
|
|
434
|
+
if word_match:
|
|
435
|
+
word = word_match.group(1)
|
|
436
|
+
if word.lower() not in ("who", "what", "where", "when", "why", "how",
|
|
437
|
+
"the", "and", "for", "with", "company", "person", "domain",
|
|
438
|
+
"investigate", "research", "search", "find", "look", "check"):
|
|
439
|
+
usernames.append(word)
|
|
440
|
+
|
|
441
|
+
return list(dict.fromkeys(usernames))
|
|
442
|
+
|
|
443
|
+
def _extract_person_name(self, text: str) -> str | None:
|
|
444
|
+
"""Extract a person's name from query text. Handles bare names like 'John Smith'."""
|
|
445
|
+
import re
|
|
446
|
+
|
|
447
|
+
# Pattern: two capitalized words (e.g., "Lorenzo Baron", "John Smith")
|
|
448
|
+
# Match anywhere in the text, handling quoted nicknames
|
|
449
|
+
match = re.search(
|
|
450
|
+
r"\b([A-Z][a-z]+(?:\s+(?:\"[A-Z][a-z]+\"\s+)?[A-Z][a-z]+){1,3})\b",
|
|
451
|
+
text
|
|
452
|
+
)
|
|
453
|
+
if match:
|
|
454
|
+
return match.group(1)
|
|
455
|
+
|
|
456
|
+
# Fallback: strip quotes and try again
|
|
457
|
+
clean = re.sub(r'["\']', '', text)
|
|
458
|
+
match = re.search(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})\b", clean)
|
|
459
|
+
if match:
|
|
460
|
+
return match.group(1)
|
|
461
|
+
|
|
462
|
+
# Also match "person named X" or "who is X"
|
|
463
|
+
for pattern in [
|
|
464
|
+
r"(?:person|individual|guy|man|woman)\s+(?:named|called|known as)\s+['\"]?([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})['\"]?",
|
|
465
|
+
r"who\s+is\s+['\"]?([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})['\"]?",
|
|
466
|
+
]:
|
|
467
|
+
match = re.search(pattern, text, re.IGNORECASE)
|
|
468
|
+
if match:
|
|
469
|
+
return match.group(1)
|
|
470
|
+
|
|
471
|
+
return None
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# Register
|
|
475
|
+
people_tool = PeopleTool()
|
|
476
|
+
registry.register(people_tool)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Tool registry — discovers and manages OSINT tools."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from ..core.models import FindingSource
|
|
8
|
+
from .base import OSINTTool
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ToolRegistry:
|
|
12
|
+
"""Manages available OSINT tools and maps categories to tools."""
|
|
13
|
+
|
|
14
|
+
def __init__(self):
|
|
15
|
+
self._tools: dict[str, OSINTTool] = {}
|
|
16
|
+
self._by_category: dict[FindingSource, list[OSINTTool]] = {s: [] for s in FindingSource}
|
|
17
|
+
|
|
18
|
+
def register(self, tool: OSINTTool) -> None:
|
|
19
|
+
"""Register a tool instance."""
|
|
20
|
+
self._tools[tool.name] = tool
|
|
21
|
+
self._by_category[tool.category].append(tool)
|
|
22
|
+
|
|
23
|
+
def get(self, name: str) -> Optional[OSINTTool]:
|
|
24
|
+
"""Get a tool by name."""
|
|
25
|
+
return self._tools.get(name)
|
|
26
|
+
|
|
27
|
+
def get_for_category(self, category: FindingSource) -> list[OSINTTool]:
|
|
28
|
+
"""Get all tools for a given category."""
|
|
29
|
+
return self._by_category.get(category, [])
|
|
30
|
+
|
|
31
|
+
def list_all(self) -> list[OSINTTool]:
|
|
32
|
+
"""Return all registered tools."""
|
|
33
|
+
return list(self._tools.values())
|
|
34
|
+
|
|
35
|
+
def list_categories(self) -> list[dict]:
|
|
36
|
+
"""Return categories with tool counts and descriptions."""
|
|
37
|
+
return [
|
|
38
|
+
{
|
|
39
|
+
"category": cat.value,
|
|
40
|
+
"tool_count": len(tools),
|
|
41
|
+
"tools": [t.name for t in tools],
|
|
42
|
+
}
|
|
43
|
+
for cat, tools in self._by_category.items()
|
|
44
|
+
if tools
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def tool_count(self) -> int:
|
|
49
|
+
return len(self._tools)
|
|
50
|
+
|
|
51
|
+
def __repr__(self) -> str:
|
|
52
|
+
return f"<ToolRegistry {self.tool_count} tools>"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# Singleton registry for the application
|
|
56
|
+
registry = ToolRegistry()
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Satellite & Maps tool — satellite imagery, terrain, coordinates."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from datetime import datetime, timedelta
|
|
7
|
+
|
|
8
|
+
from .base import OSINTTool
|
|
9
|
+
from .registry import registry
|
|
10
|
+
from ..core.models import Finding, FindingSource
|
|
11
|
+
from ..utils.http import get_client
|
|
12
|
+
from ..utils.helpers import extract_domain
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SatelliteTool(OSINTTool):
|
|
16
|
+
"""Investigate locations via satellite imagery, maps, and geospatial data."""
|
|
17
|
+
|
|
18
|
+
category = FindingSource.SATELLITE
|
|
19
|
+
name = "satellite-maps"
|
|
20
|
+
description = "Satellite imagery, terrain data, coordinate lookup, and geospatial analysis"
|
|
21
|
+
free_tier_available = True
|
|
22
|
+
rate_limit_rps = 0.5
|
|
23
|
+
|
|
24
|
+
GOOGLE_EARTH_HISTORICAL = "https://earth.google.com/web/search/{location}"
|
|
25
|
+
OPENSTREETMAP_NOMINATIM = "https://nominatim.openstreetmap.org/search"
|
|
26
|
+
OPENSTREETMAP_REVERSE = "https://nominatim.openstreetmap.org/reverse"
|
|
27
|
+
|
|
28
|
+
async def investigate(self, query: str, context: str = "") -> list[Finding]:
|
|
29
|
+
findings: list[Finding] = []
|
|
30
|
+
|
|
31
|
+
# Try to extract a location from the query
|
|
32
|
+
location = await self._extract_location(query)
|
|
33
|
+
if not location:
|
|
34
|
+
return findings
|
|
35
|
+
|
|
36
|
+
client = get_client(rate_limit=self.rate_limit_rps)
|
|
37
|
+
|
|
38
|
+
# 1. Geocode the location
|
|
39
|
+
try:
|
|
40
|
+
params = {"q": location, "format": "json", "limit": 3}
|
|
41
|
+
data = await client.get_json(self.OPENSTREETMAP_NOMINATIM, params=params)
|
|
42
|
+
|
|
43
|
+
if data and isinstance(data, list) and len(data) > 0:
|
|
44
|
+
first = data[0]
|
|
45
|
+
lat = first.get("lat")
|
|
46
|
+
lon = first.get("lon")
|
|
47
|
+
display_name = first.get("display_name", location)
|
|
48
|
+
|
|
49
|
+
findings.append(
|
|
50
|
+
self._make_finding(
|
|
51
|
+
title=f"📍 Location: {display_name[:80]}",
|
|
52
|
+
description=(
|
|
53
|
+
f"Coordinates: {lat}, {lon}. "
|
|
54
|
+
f"Type: {first.get('type', 'unknown')}. "
|
|
55
|
+
f"Category: {first.get('category', 'unknown')}."
|
|
56
|
+
),
|
|
57
|
+
evidence=[
|
|
58
|
+
f"https://www.openstreetmap.org/?mlat={lat}&mlon={lon}&zoom=14",
|
|
59
|
+
self.GOOGLE_EARTH_HISTORICAL.format(location=location),
|
|
60
|
+
],
|
|
61
|
+
confidence=0.9,
|
|
62
|
+
)
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# 2. Find nearby features from additional results
|
|
66
|
+
if len(data) > 1:
|
|
67
|
+
nearby = ", ".join(
|
|
68
|
+
d.get("display_name", "").split(",")[0].strip()
|
|
69
|
+
for d in data[1:3]
|
|
70
|
+
)
|
|
71
|
+
findings.append(
|
|
72
|
+
self._make_finding(
|
|
73
|
+
title=f"Nearby: {nearby}",
|
|
74
|
+
description=f"Related locations found near {display_name.split(',')[0]}.",
|
|
75
|
+
confidence=0.7,
|
|
76
|
+
lat=lat,
|
|
77
|
+
lon=lon,
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
except Exception as e:
|
|
82
|
+
findings.append(
|
|
83
|
+
self._make_finding(
|
|
84
|
+
title="Satellite lookup limited",
|
|
85
|
+
description=f"Could not retrieve full satellite data: {str(e)}. Try the Google Earth link.",
|
|
86
|
+
evidence=[self.GOOGLE_EARTH_HISTORICAL.format(location=location)],
|
|
87
|
+
confidence=0.3,
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
return findings
|
|
92
|
+
|
|
93
|
+
async def _extract_location(self, text: str) -> str | None:
|
|
94
|
+
"""Extract a location string from the query text."""
|
|
95
|
+
# Simple heuristic: look for location patterns
|
|
96
|
+
import re
|
|
97
|
+
|
|
98
|
+
# Match "in X" or "near X" or "at X" patterns
|
|
99
|
+
patterns = [
|
|
100
|
+
r"(?:in|near|at|around|over)\s+([A-Z][a-zA-Z\s,]+?)(?:\s+(?:and|or|\.|$|,))",
|
|
101
|
+
r"location[s]?\s+(?:is|are|:)?\s+([A-Z][a-zA-Z\s,]+?)(?:\s+(?:and|or|\.|$))",
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
for pattern in patterns:
|
|
105
|
+
match = re.search(pattern, text)
|
|
106
|
+
if match:
|
|
107
|
+
return match.group(1).strip().rstrip(".,")
|
|
108
|
+
|
|
109
|
+
# If no pattern matched, check if the whole thing might be a location
|
|
110
|
+
if len(text) < 80 and any(c.isupper() for c in text):
|
|
111
|
+
return text.strip()
|
|
112
|
+
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Register
|
|
117
|
+
satellite_tool = SatelliteTool()
|
|
118
|
+
registry.register(satellite_tool)
|