osintengine 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osintengine-1.0.2.dist-info/METADATA +311 -0
- osintengine-1.0.2.dist-info/RECORD +103 -0
- osintengine-1.0.2.dist-info/WHEEL +5 -0
- osintengine-1.0.2.dist-info/entry_points.txt +2 -0
- osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
- osintengine-1.0.2.dist-info/top_level.txt +2 -0
- src/watson/__init__.py +10 -0
- src/watson/agent/__init__.py +96 -0
- src/watson/agents/__init__.py +101 -0
- src/watson/agents/protocol.py +196 -0
- src/watson/auth/__init__.py +68 -0
- src/watson/auth/store.py +145 -0
- src/watson/conversation.py +68 -0
- src/watson/core/__init__.py +0 -0
- src/watson/core/models.py +96 -0
- src/watson/ethics.py +205 -0
- src/watson/exports.py +182 -0
- src/watson/graph/__init__.py +69 -0
- src/watson/graph/entities.py +207 -0
- src/watson/graph/graph.py +489 -0
- src/watson/graph/osint_framework.py +266 -0
- src/watson/graph/relationships.py +116 -0
- src/watson/graph/transforms.py +787 -0
- src/watson/infra/__init__.py +43 -0
- src/watson/infra/cache.py +171 -0
- src/watson/infra/ratelimit.py +125 -0
- src/watson/infra/resilience.py +239 -0
- src/watson/infra/retry.py +186 -0
- src/watson/metrics.py +128 -0
- src/watson/opsec/__init__.py +290 -0
- src/watson/orchestration/__init__.py +23 -0
- src/watson/orchestration/engine.py +5587 -0
- src/watson/orchestration/executor.py +96 -0
- src/watson/orchestration/intent.py +139 -0
- src/watson/orchestration/intent_classifier.py +45 -0
- src/watson/orchestration/llm_config.py +160 -0
- src/watson/orchestration/resolution.py +555 -0
- src/watson/orchestration/scraper.py +94 -0
- src/watson/orchestration/synthesis.py +726 -0
- src/watson/orchestration/target_profile.py +748 -0
- src/watson/persistence/__init__.py +18 -0
- src/watson/persistence/models.py +161 -0
- src/watson/persistence/store.py +230 -0
- src/watson/pipeline/__init__.py +16 -0
- src/watson/pipeline/pre_synthesis.py +621 -0
- src/watson/search.py +103 -0
- src/watson/serializers/stix.py +451 -0
- src/watson/tools/__init__.py +31 -0
- src/watson/tools/base.py +97 -0
- src/watson/tools/blockchain.py +359 -0
- src/watson/tools/captcha.py +276 -0
- src/watson/tools/conflict.py +107 -0
- src/watson/tools/corporate.py +287 -0
- src/watson/tools/darkweb.py +144 -0
- src/watson/tools/geolocation.py +347 -0
- src/watson/tools/image_video.py +108 -0
- src/watson/tools/marinetraffic.py +142 -0
- src/watson/tools/people.py +476 -0
- src/watson/tools/registry.py +56 -0
- src/watson/tools/satellite.py +118 -0
- src/watson/tools/scraper.py +745 -0
- src/watson/tools/shodan.py +129 -0
- src/watson/tools/social_media.py +160 -0
- src/watson/tools/websites.py +291 -0
- src/watson/tools/wikidata.py +398 -0
- src/watson/utils/__init__.py +1 -0
- src/watson/utils/helpers.py +49 -0
- src/watson/utils/http.py +119 -0
- src/watson/verification/__init__.py +245 -0
- watson/__init__.py +6 -0
- watson/agents/__init__.py +20 -0
- watson/agents/base.py +103 -0
- watson/agents/direct.py +225 -0
- watson/agents/hermes.py +275 -0
- watson/agents/hermes_mcp.py +292 -0
- watson/api_keys.py +176 -0
- watson/browser_scraper.py +481 -0
- watson/cli.py +836 -0
- watson/crossref.py +175 -0
- watson/ethics.py +20 -0
- watson/graph.py +406 -0
- watson/mcp_server.py +601 -0
- watson/memory.py +552 -0
- watson/neo4j_graph.py +124 -0
- watson/opsec/__init__.py +307 -0
- watson/reporter.py +537 -0
- watson/scheduler.py +486 -0
- watson/serializers/stix.py +451 -0
- watson/terminal.py +410 -0
- watson/toolkit.py +252 -0
- watson/toolkit_api.py +347 -0
- watson/toolkit_automation.py +95 -0
- watson/toolkit_registry.py +345 -0
- watson/verification/__init__.py +251 -0
- watson/web/__init__.py +1 -0
- watson/web/app.py +1650 -0
- watson/web/middleware.py +301 -0
- watson/web/static/assets/index-B7hPOc0z.js +278 -0
- watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
- watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
- watson/web/static/index.html +14 -0
- watson/web/templates/chat.html +2 -0
- watson/web/templates/investigation-map.html +429 -0
|
@@ -0,0 +1,748 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Target profiler — fast, accurate classification + rich decomposition.
|
|
3
|
+
|
|
4
|
+
Layered approach so the orchestrator knows not just WHAT is being
|
|
5
|
+
investigated but WHERE to start looking first.
|
|
6
|
+
|
|
7
|
+
Elon Musk → person + @elonmusk + Tesla/SpaceX + wikidata:Q317521
|
|
8
|
+
Stripe → company + stripe.com + founders + opencorp lookup
|
|
9
|
+
0xdead... → wallet + ETH chain + etherscan link
|
|
10
|
+
|
|
11
|
+
Architecture:
|
|
12
|
+
Layer 1: Regex (instant) — email, IP, domain, phone, crypto wallets
|
|
13
|
+
Layer 2: GLiNER (CPU, ~50ms) — person vs organization disambiguation
|
|
14
|
+
Layer 3: Wikidata API (REST) — QID, occupation, employer, social handles
|
|
15
|
+
Layer 4: LLM (fallback only) — truly ambiguous cases
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import json
|
|
22
|
+
import logging
|
|
23
|
+
import re
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from typing import Optional
|
|
26
|
+
from urllib.parse import quote
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# ── Data model ────────────────────────────────────────────────────
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class TargetProfile:
|
|
36
|
+
"""Rich decomposition of an investigation target.
|
|
37
|
+
|
|
38
|
+
The orchestrator uses this to decide which sources to hit first.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
target_type: str # person | company | organization | domain | wallet | ip | email
|
|
42
|
+
primary_name: str # cleaned canonical form
|
|
43
|
+
confidence: float # 0.0 — 1.0
|
|
44
|
+
raw_query: str = ""
|
|
45
|
+
|
|
46
|
+
# ── Associated entities ──
|
|
47
|
+
associated_orgs: list[str] = field(default_factory=list) # Tesla, SpaceX
|
|
48
|
+
associated_domains: list[str] = field(default_factory=list) # tesla.com
|
|
49
|
+
social_handles: list[str] = field(default_factory=list) # @elonmusk
|
|
50
|
+
known_aliases: list[str] = field(default_factory=list)
|
|
51
|
+
locations: list[str] = field(default_factory=list)
|
|
52
|
+
|
|
53
|
+
# ── Wikidata enrichment ──
|
|
54
|
+
wikidata_qid: Optional[str] = None # Q317521
|
|
55
|
+
wikidata_label: Optional[str] = None
|
|
56
|
+
wikidata_description: Optional[str] = None
|
|
57
|
+
|
|
58
|
+
# ── Orchestrator hints ──
|
|
59
|
+
suggested_sources: list[str] = field(default_factory=list)
|
|
60
|
+
# e.g. ["wikidata", "web_search", "twitter", "opencorporates"]
|
|
61
|
+
investigation_angles: list[str] = field(default_factory=list)
|
|
62
|
+
# e.g. ["social_media", "corporate_affiliations", "domain_infra"]
|
|
63
|
+
|
|
64
|
+
# ── Source-specific IDs ──
|
|
65
|
+
source_ids: dict[str, str] = field(default_factory=dict)
|
|
66
|
+
# e.g. {"opencorporates": "https://opencorporates.com/companies/us_de/..."}
|
|
67
|
+
|
|
68
|
+
# ── Classification path (for debugging) ──
|
|
69
|
+
classified_by: str = "" # "regex", "gliner", "wikidata", "llm"
|
|
70
|
+
|
|
71
|
+
def __post_init__(self):
|
|
72
|
+
if not self.suggested_sources:
|
|
73
|
+
self.suggested_sources = _DEFAULT_SOURCES.get(self.target_type, ["web_search"])
|
|
74
|
+
if not self.investigation_angles:
|
|
75
|
+
self.investigation_angles = _DEFAULT_ANGLES.get(self.target_type, ["general"])
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# ── Default source mappings per target type ───────────────────────
|
|
79
|
+
|
|
80
|
+
_DEFAULT_SOURCES: dict[str, list[str]] = {
|
|
81
|
+
"person": ["wikidata", "web_search", "social_media", "news_search", "sanctions"],
|
|
82
|
+
"company": ["opencorporates", "crtsh", "web_search", "wikidata", "sanctions", "secedgar"],
|
|
83
|
+
"organization": ["opencorporates", "crtsh", "web_search", "wikidata", "sanctions"],
|
|
84
|
+
"domain": ["crtsh", "wayback", "dns", "web_search", "shodan"],
|
|
85
|
+
"wallet": ["blockchain_explorer", "web_search", "dark_web"],
|
|
86
|
+
"ip": ["shodan", "geolocation", "abuseipdb", "web_search"],
|
|
87
|
+
"email": ["hibp", "social_enumeration", "web_search", "dark_web"],
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
_DEFAULT_ANGLES: dict[str, list[str]] = {
|
|
91
|
+
"person": ["identity", "social_media", "professional", "legal", "web_presence"],
|
|
92
|
+
"company": ["corporate_structure", "domain_infra", "legal_regulatory", "leadership", "web_presence"],
|
|
93
|
+
"organization": ["structure", "domain_infra", "legal", "leadership", "web_presence"],
|
|
94
|
+
"domain": ["ssl_certs", "dns_records", "wayback_history", "associated_ips", "subdomains"],
|
|
95
|
+
"wallet": ["transaction_graph", "exchange_links", "entity_attribution", "dark_web_mentions"],
|
|
96
|
+
"ip": ["geolocation", "hosting", "abuse_history", "associated_domains", "ports"],
|
|
97
|
+
"email": ["breaches", "social_accounts", "dark_web", "domain_owner"],
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# ── Layer 1: Deterministic regex ──────────────────────────────────
|
|
102
|
+
# Extended from SpiderFoot's battle-tested regexes, plus modern types
|
|
103
|
+
|
|
104
|
+
_REGEX_RULES: list[tuple[re.Pattern, str, float]] = [
|
|
105
|
+
# ── Email (must come before domain to catch user@host) ──
|
|
106
|
+
(re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"), "email", 0.98),
|
|
107
|
+
|
|
108
|
+
# ── Ethereum / EVM wallet ──
|
|
109
|
+
(re.compile(r"^0x[a-fA-F0-9]{40}$"), "wallet", 0.98),
|
|
110
|
+
|
|
111
|
+
# ── Solana wallet ──
|
|
112
|
+
(re.compile(r"^[1-9A-HJ-NP-Za-km-z]{32,44}$"), "wallet", 0.85),
|
|
113
|
+
|
|
114
|
+
# ── Bitcoin address (legacy P2PKH) ──
|
|
115
|
+
(re.compile(r"^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$"), "wallet", 0.85),
|
|
116
|
+
|
|
117
|
+
# ── IPv4 ──
|
|
118
|
+
(re.compile(
|
|
119
|
+
r"^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$"
|
|
120
|
+
), "ip", 0.95),
|
|
121
|
+
|
|
122
|
+
# ── IPv4 CIDR ──
|
|
123
|
+
(re.compile(
|
|
124
|
+
r"^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)/\d{1,2}$"
|
|
125
|
+
), "ip", 0.95),
|
|
126
|
+
|
|
127
|
+
# ── IPv6 (simplified) ──
|
|
128
|
+
(re.compile(r"^[0-9a-fA-F:]+$"), "ip", 0.80),
|
|
129
|
+
|
|
130
|
+
# ── Phone (E.164) ──
|
|
131
|
+
(re.compile(r"^\+[1-9]\d{6,14}$"), "email", 0.90), # will be overridden by phone type
|
|
132
|
+
|
|
133
|
+
# ── Domain with known TLD ──
|
|
134
|
+
(re.compile(
|
|
135
|
+
r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+"
|
|
136
|
+
r"(?:com|org|net|io|gov|edu|mil|ru|cn|de|uk|fr|br|in|jp|kr|"
|
|
137
|
+
r"au|ca|ch|nl|se|no|dk|fi|pl|ua|ir|ng|sg|hk|tw|"
|
|
138
|
+
r"info|biz|co|me|tv|ai|app|dev|xyz|site|online|shop|"
|
|
139
|
+
r"tech|blog|news|media|wiki|finance|law|health|"
|
|
140
|
+
r"design|agency|consulting|ventures|capital|partners|"
|
|
141
|
+
r"foundation|institute|international)$",
|
|
142
|
+
re.IGNORECASE,
|
|
143
|
+
), "domain", 0.90),
|
|
144
|
+
|
|
145
|
+
# ── URL ──
|
|
146
|
+
(re.compile(r"^https?://[^\s]+$", re.IGNORECASE), "domain", 0.70),
|
|
147
|
+
]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _regex_classify(query: str) -> Optional[TargetProfile]:
|
|
151
|
+
"""Try deterministic regex classification. Fastest path."""
|
|
152
|
+
q = query.strip()
|
|
153
|
+
|
|
154
|
+
for pattern, ttype, conf in _REGEX_RULES:
|
|
155
|
+
if pattern.match(q):
|
|
156
|
+
# ── Enrich email ──
|
|
157
|
+
if ttype == "email" and "@" in q:
|
|
158
|
+
domain = q.split("@")[-1].lower()
|
|
159
|
+
profile = TargetProfile(
|
|
160
|
+
target_type="email",
|
|
161
|
+
primary_name=q.lower(),
|
|
162
|
+
confidence=conf,
|
|
163
|
+
raw_query=query,
|
|
164
|
+
associated_domains=[domain],
|
|
165
|
+
classified_by="regex",
|
|
166
|
+
)
|
|
167
|
+
return profile
|
|
168
|
+
|
|
169
|
+
# ── Enrich wallet ──
|
|
170
|
+
if ttype == "wallet":
|
|
171
|
+
chain = "eth" if q.startswith("0x") else "btc" if q[0] in "13" else "sol"
|
|
172
|
+
explorers = {
|
|
173
|
+
"eth": f"https://etherscan.io/address/{q}",
|
|
174
|
+
"btc": f"https://www.blockchain.com/explorer/addresses/btc/{q}",
|
|
175
|
+
"sol": f"https://solscan.io/account/{q}",
|
|
176
|
+
}
|
|
177
|
+
return TargetProfile(
|
|
178
|
+
target_type="wallet",
|
|
179
|
+
primary_name=q,
|
|
180
|
+
confidence=conf,
|
|
181
|
+
raw_query=query,
|
|
182
|
+
source_ids={"explorer": explorers.get(chain, "")},
|
|
183
|
+
investigation_angles=["transaction_graph", "exchange_links", "entity_attribution"],
|
|
184
|
+
classified_by="regex",
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# ── Enrich IP ──
|
|
188
|
+
if ttype == "ip":
|
|
189
|
+
return TargetProfile(
|
|
190
|
+
target_type="ip",
|
|
191
|
+
primary_name=q,
|
|
192
|
+
confidence=conf,
|
|
193
|
+
raw_query=query,
|
|
194
|
+
source_ids={"shodan": f"https://www.shodan.io/host/{q}"},
|
|
195
|
+
classified_by="regex",
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
# ── Enrich domain ──
|
|
199
|
+
if ttype == "domain":
|
|
200
|
+
clean = q.replace("https://", "").replace("http://", "").rstrip("/")
|
|
201
|
+
# Extract base domain
|
|
202
|
+
parts = clean.split(".")
|
|
203
|
+
if len(parts) >= 2:
|
|
204
|
+
base = ".".join(parts[-2:]) if len(parts) > 2 else clean
|
|
205
|
+
else:
|
|
206
|
+
base = clean
|
|
207
|
+
return TargetProfile(
|
|
208
|
+
target_type="domain",
|
|
209
|
+
primary_name=clean,
|
|
210
|
+
confidence=conf,
|
|
211
|
+
raw_query=query,
|
|
212
|
+
associated_domains=[clean],
|
|
213
|
+
source_ids={
|
|
214
|
+
"crtsh": f"https://crt.sh/?q=%25.{base}",
|
|
215
|
+
"wayback": f"https://web.archive.org/web/*/{clean}",
|
|
216
|
+
},
|
|
217
|
+
classified_by="regex",
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
return None
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# ── Layer 2: GLiNER NER ──────────────────────────────────────────
|
|
224
|
+
# Person vs organization disambiguation, runs locally on CPU
|
|
225
|
+
|
|
226
|
+
_GLINER_MODEL = None
|
|
227
|
+
_GLINER_LOADED = False
|
|
228
|
+
|
|
229
|
+
# Entity labels GLiNER extracts
|
|
230
|
+
_ENTITY_LABELS = [
|
|
231
|
+
"person",
|
|
232
|
+
"organization",
|
|
233
|
+
"company",
|
|
234
|
+
"email",
|
|
235
|
+
"phone number",
|
|
236
|
+
"cryptocurrency wallet",
|
|
237
|
+
"domain",
|
|
238
|
+
"location",
|
|
239
|
+
"username",
|
|
240
|
+
"product",
|
|
241
|
+
"hacker group",
|
|
242
|
+
"criminal organization",
|
|
243
|
+
]
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _load_gliner():
|
|
247
|
+
"""Lazy-load GLiNER model — only loaded once, on first use."""
|
|
248
|
+
global _GLINER_MODEL, _GLINER_LOADED
|
|
249
|
+
if _GLINER_LOADED:
|
|
250
|
+
return _GLINER_MODEL
|
|
251
|
+
_GLINER_LOADED = True
|
|
252
|
+
try:
|
|
253
|
+
from gliner import GLiNER
|
|
254
|
+
_GLINER_MODEL = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
|
|
255
|
+
logger.info("GLiNER model loaded (medium-v2.1, CPU)")
|
|
256
|
+
except Exception as e:
|
|
257
|
+
logger.warning(f"GLiNER failed to load: {e}. Falling back to regex + LLM.")
|
|
258
|
+
_GLINER_MODEL = None
|
|
259
|
+
return _GLINER_MODEL
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _gliner_classify(query: str) -> Optional[TargetProfile]:
|
|
263
|
+
"""Use GLiNER to extract entities and determine target type.
|
|
264
|
+
|
|
265
|
+
Calibrated thresholds (empirically validated):
|
|
266
|
+
- person: 0.5 (Elon Musk 0.93, Dmitry Khoroshev 0.92)
|
|
267
|
+
- organization/company: 0.2 (Stripe 0.22, Binance 0.30, OpenAI 0.45)
|
|
268
|
+
- domain/email: 0.4
|
|
269
|
+
"""
|
|
270
|
+
model = _load_gliner()
|
|
271
|
+
if model is None:
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
try:
|
|
275
|
+
# Use lower threshold for org detection to catch single-word company names
|
|
276
|
+
entities = model.predict_entities(query, _ENTITY_LABELS, threshold=0.2)
|
|
277
|
+
except Exception as e:
|
|
278
|
+
logger.warning(f"GLiNER prediction failed: {e}")
|
|
279
|
+
return None
|
|
280
|
+
|
|
281
|
+
if not entities:
|
|
282
|
+
return None
|
|
283
|
+
|
|
284
|
+
# Separate entities by type
|
|
285
|
+
persons = []
|
|
286
|
+
orgs = []
|
|
287
|
+
domains_found = []
|
|
288
|
+
emails = []
|
|
289
|
+
locations = []
|
|
290
|
+
wallets = []
|
|
291
|
+
|
|
292
|
+
for ent in entities:
|
|
293
|
+
label = ent.get("label", "").lower()
|
|
294
|
+
text = ent.get("text", "").strip()
|
|
295
|
+
score = ent.get("score", 0.0)
|
|
296
|
+
|
|
297
|
+
# Person: high precision (0.5+) to avoid false positives
|
|
298
|
+
if label == "person" and score > 0.5:
|
|
299
|
+
persons.append(text)
|
|
300
|
+
# Organization/company: calibrated at 0.2 for single-word detection
|
|
301
|
+
elif label in ("organization", "company") and score > 0.25:
|
|
302
|
+
orgs.append(text)
|
|
303
|
+
elif label == "domain" and score > 0.4:
|
|
304
|
+
domains_found.append(text)
|
|
305
|
+
elif label == "email" and score > 0.5:
|
|
306
|
+
emails.append(text)
|
|
307
|
+
elif label == "location" and score > 0.4:
|
|
308
|
+
locations.append(text)
|
|
309
|
+
elif label in ("cryptocurrency wallet",) and score > 0.5:
|
|
310
|
+
wallets.append(text)
|
|
311
|
+
|
|
312
|
+
# ── Determine primary type ──
|
|
313
|
+
# Priority: wallet > email > domain > person > organization
|
|
314
|
+
if wallets:
|
|
315
|
+
return TargetProfile(
|
|
316
|
+
target_type="wallet",
|
|
317
|
+
primary_name=wallets[0],
|
|
318
|
+
confidence=0.85,
|
|
319
|
+
raw_query=query,
|
|
320
|
+
classified_by="gliner",
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
if emails:
|
|
324
|
+
domain = emails[0].split("@")[-1] if "@" in emails[0] else ""
|
|
325
|
+
return TargetProfile(
|
|
326
|
+
target_type="email",
|
|
327
|
+
primary_name=emails[0],
|
|
328
|
+
confidence=0.85,
|
|
329
|
+
associated_domains=[domain] if domain else [],
|
|
330
|
+
classified_by="gliner",
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
if domains_found:
|
|
334
|
+
return TargetProfile(
|
|
335
|
+
target_type="domain",
|
|
336
|
+
primary_name=domains_found[0],
|
|
337
|
+
confidence=0.80,
|
|
338
|
+
associated_domains=domains_found,
|
|
339
|
+
classified_by="gliner",
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
if persons:
|
|
343
|
+
profile = TargetProfile(
|
|
344
|
+
target_type="person",
|
|
345
|
+
primary_name=persons[0],
|
|
346
|
+
confidence=0.80,
|
|
347
|
+
raw_query=query,
|
|
348
|
+
known_aliases=persons[1:],
|
|
349
|
+
locations=locations,
|
|
350
|
+
classified_by="gliner",
|
|
351
|
+
)
|
|
352
|
+
if orgs:
|
|
353
|
+
profile.associated_orgs = orgs
|
|
354
|
+
return profile
|
|
355
|
+
|
|
356
|
+
if orgs:
|
|
357
|
+
return TargetProfile(
|
|
358
|
+
target_type="organization",
|
|
359
|
+
primary_name=orgs[0],
|
|
360
|
+
confidence=0.75,
|
|
361
|
+
raw_query=query,
|
|
362
|
+
associated_orgs=orgs[1:],
|
|
363
|
+
locations=locations,
|
|
364
|
+
classified_by="gliner",
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
return None
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
# ── Layer 3: Wikidata enrichment ──────────────────────────────────
|
|
371
|
+
|
|
372
|
+
async def _wikidata_enrich(profile: TargetProfile) -> TargetProfile:
|
|
373
|
+
"""Enrich a profile with Wikidata data. No API key needed."""
|
|
374
|
+
search_term = profile.primary_name
|
|
375
|
+
if profile.raw_query and profile.raw_query != profile.primary_name:
|
|
376
|
+
search_term = profile.raw_query
|
|
377
|
+
|
|
378
|
+
try:
|
|
379
|
+
# Search for entity
|
|
380
|
+
url = (
|
|
381
|
+
"https://www.wikidata.org/w/api.php"
|
|
382
|
+
f"?action=wbsearchentities&search={quote(search_term)}"
|
|
383
|
+
"&language=en&format=json&limit=3"
|
|
384
|
+
)
|
|
385
|
+
import aiohttp
|
|
386
|
+
async with aiohttp.ClientSession() as session:
|
|
387
|
+
async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as resp:
|
|
388
|
+
data = await resp.json()
|
|
389
|
+
except Exception as e:
|
|
390
|
+
logger.debug(f"Wikidata search failed: {e}")
|
|
391
|
+
return profile
|
|
392
|
+
|
|
393
|
+
results = data.get("search", [])
|
|
394
|
+
if not results:
|
|
395
|
+
return profile
|
|
396
|
+
|
|
397
|
+
best = results[0]
|
|
398
|
+
profile.wikidata_qid = best.get("id")
|
|
399
|
+
profile.wikidata_label = best.get("label")
|
|
400
|
+
profile.wikidata_description = best.get("description", "")
|
|
401
|
+
|
|
402
|
+
# ── Fetch detailed entity data ──
|
|
403
|
+
if profile.wikidata_qid:
|
|
404
|
+
try:
|
|
405
|
+
detail_url = (
|
|
406
|
+
f"https://www.wikidata.org/wiki/Special:EntityData/"
|
|
407
|
+
f"{profile.wikidata_qid}.json"
|
|
408
|
+
)
|
|
409
|
+
import aiohttp
|
|
410
|
+
async with aiohttp.ClientSession() as session:
|
|
411
|
+
async with session.get(detail_url, timeout=aiohttp.ClientTimeout(total=10)) as resp:
|
|
412
|
+
entity_data = await resp.json()
|
|
413
|
+
except Exception:
|
|
414
|
+
return profile
|
|
415
|
+
|
|
416
|
+
claims = (
|
|
417
|
+
entity_data.get("entities", {})
|
|
418
|
+
.get(profile.wikidata_qid, {})
|
|
419
|
+
.get("claims", {})
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
# ── Extract social handles ──
|
|
423
|
+
# P2002 = Twitter username, P4264 = LinkedIn, P2013 = Facebook
|
|
424
|
+
twitter = _extract_wikidata_prop(claims, "P2002")
|
|
425
|
+
if twitter:
|
|
426
|
+
profile.social_handles.append(f"@{twitter}")
|
|
427
|
+
|
|
428
|
+
linkedin = _extract_wikidata_prop(claims, "P4264")
|
|
429
|
+
if linkedin:
|
|
430
|
+
profile.social_handles.append(f"linkedin:{linkedin}")
|
|
431
|
+
|
|
432
|
+
# ── Extract associated domains ──
|
|
433
|
+
# P856 = official website
|
|
434
|
+
website = _extract_wikidata_prop(claims, "P856")
|
|
435
|
+
if website:
|
|
436
|
+
domain = website.replace("https://", "").replace("http://", "").rstrip("/")
|
|
437
|
+
profile.associated_domains.append(domain)
|
|
438
|
+
|
|
439
|
+
# ── Extract employer / founder relationships ──
|
|
440
|
+
# P108 = employer, P112 = founded by
|
|
441
|
+
if profile.target_type == "person":
|
|
442
|
+
employer = _extract_wikidata_prop(claims, "P108")
|
|
443
|
+
if employer:
|
|
444
|
+
profile.associated_orgs.append(employer)
|
|
445
|
+
|
|
446
|
+
if profile.target_type in ("company", "organization"):
|
|
447
|
+
founder = _extract_wikidata_prop(claims, "P112")
|
|
448
|
+
if founder:
|
|
449
|
+
profile.known_aliases.append(founder)
|
|
450
|
+
|
|
451
|
+
# ── Extract location ──
|
|
452
|
+
# P17 = country, P159 = HQ location, P19 = place of birth
|
|
453
|
+
country = _extract_wikidata_prop(claims, "P17")
|
|
454
|
+
if country and country not in profile.locations:
|
|
455
|
+
profile.locations.append(country)
|
|
456
|
+
|
|
457
|
+
hq = _extract_wikidata_prop(claims, "P159")
|
|
458
|
+
if hq and hq not in profile.locations:
|
|
459
|
+
profile.locations.append(hq)
|
|
460
|
+
|
|
461
|
+
birthplace = _extract_wikidata_prop(claims, "P19")
|
|
462
|
+
if birthplace and birthplace not in profile.locations:
|
|
463
|
+
profile.locations.append(birthplace)
|
|
464
|
+
|
|
465
|
+
# ── Source-specific IDs ──
|
|
466
|
+
if profile.target_type in ("company", "organization"):
|
|
467
|
+
# OpenCorporates ID
|
|
468
|
+
oc_id = _extract_wikidata_prop(claims, "P1320")
|
|
469
|
+
if oc_id:
|
|
470
|
+
profile.source_ids["opencorporates"] = (
|
|
471
|
+
f"https://opencorporates.com/companies/{oc_id}"
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
# ── Correct misclassified type using Wikidata ──
|
|
475
|
+
# GLiNER/LLM can misclassify orgs as persons (e.g., "Aviloop" looks like a name).
|
|
476
|
+
# Wikidata is authoritative — use its instance_of and description to fix.
|
|
477
|
+
if profile.wikidata_qid and profile.wikidata_description:
|
|
478
|
+
desc_lower = profile.wikidata_description.lower()
|
|
479
|
+
org_keywords = [
|
|
480
|
+
"company", "corporation", "organization", "organisation",
|
|
481
|
+
"business", "website", "enterprise", "aviation", "airline",
|
|
482
|
+
"startup", "brand", "service", "platform", "firm", "agency",
|
|
483
|
+
"studio", "foundation", "institute", "association",
|
|
484
|
+
]
|
|
485
|
+
is_org_by_desc = any(kw in desc_lower for kw in org_keywords)
|
|
486
|
+
|
|
487
|
+
# Check instance_of (P31) for business/org Wikidata types
|
|
488
|
+
instance_of = _extract_wikidata_prop(claims, "P31")
|
|
489
|
+
is_org_by_p31 = instance_of in (
|
|
490
|
+
"Q4830453", # business
|
|
491
|
+
"Q6881511", # enterprise
|
|
492
|
+
"Q43229", # organization
|
|
493
|
+
"Q35127", # website
|
|
494
|
+
"Q783794", # company
|
|
495
|
+
"Q167270", # trademark
|
|
496
|
+
"Q431289", # brand
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
if profile.target_type == "person" and (is_org_by_desc or is_org_by_p31):
|
|
500
|
+
logger.info(
|
|
501
|
+
f"wikidata_type_correction: '{profile.primary_name}' "
|
|
502
|
+
f"was classified as PERSON but Wikidata says '{profile.wikidata_description}' "
|
|
503
|
+
f"— switching to organization"
|
|
504
|
+
)
|
|
505
|
+
profile.target_type = "organization"
|
|
506
|
+
profile.investigation_angles = _DEFAULT_ANGLES.get("organization", ["general"])
|
|
507
|
+
profile.suggested_sources = _DEFAULT_SOURCES.get("organization", ["web_search"])
|
|
508
|
+
profile.classified_by = f"{profile.classified_by}+type_corrected"
|
|
509
|
+
|
|
510
|
+
# ── Update suggested sources based on what Wikidata told us ──
|
|
511
|
+
profile.suggested_sources = _DEFAULT_SOURCES.get(profile.target_type, ["web_search"])
|
|
512
|
+
profile.investigation_angles = _DEFAULT_ANGLES.get(profile.target_type, ["general"])
|
|
513
|
+
|
|
514
|
+
profile.classified_by = f"{profile.classified_by}+wikidata"
|
|
515
|
+
profile.confidence = min(profile.confidence + 0.1, 0.99)
|
|
516
|
+
|
|
517
|
+
return profile
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _extract_wikidata_prop(claims: dict, prop_id: str) -> Optional[str]:
|
|
521
|
+
"""Extract a simple string value from Wikidata claims."""
|
|
522
|
+
prop = claims.get(prop_id, [])
|
|
523
|
+
if not prop:
|
|
524
|
+
return None
|
|
525
|
+
mainsnak = prop[0].get("mainsnak", {})
|
|
526
|
+
if mainsnak.get("snaktype") != "value":
|
|
527
|
+
return None
|
|
528
|
+
datavalue = mainsnak.get("datavalue", {}).get("value", {})
|
|
529
|
+
|
|
530
|
+
# String
|
|
531
|
+
if isinstance(datavalue, str):
|
|
532
|
+
return datavalue
|
|
533
|
+
# Entity reference (Q-item)
|
|
534
|
+
if isinstance(datavalue, dict):
|
|
535
|
+
item = datavalue.get("id", "")
|
|
536
|
+
# For Q-items, we return the ID — caller must resolve labels
|
|
537
|
+
return item
|
|
538
|
+
return None
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
# ── Layer 4: LLM fallback ────────────────────────────────────────
|
|
542
|
+
|
|
543
|
+
_LLM_FALLBACK_PROMPT = """You are an OSINT target classifier. Analyze this query and return STRICT JSON.
|
|
544
|
+
|
|
545
|
+
QUERY: {query}
|
|
546
|
+
|
|
547
|
+
Return:
|
|
548
|
+
{{
|
|
549
|
+
"target_type": "person|company|organization|domain|wallet|ip|email",
|
|
550
|
+
"primary_name": "canonical name",
|
|
551
|
+
"confidence": 0.0-1.0,
|
|
552
|
+
"associated_orgs": ["any orgs mentioned or implied"],
|
|
553
|
+
"associated_domains": ["any domains mentioned or implied"],
|
|
554
|
+
"social_handles": ["any social handles mentioned"],
|
|
555
|
+
"known_aliases": ["aliases"],
|
|
556
|
+
"locations": ["locations"],
|
|
557
|
+
"investigation_angles": ["angles to investigate"]
|
|
558
|
+
}}
|
|
559
|
+
|
|
560
|
+
RULES:
|
|
561
|
+
- "Stripe" is a company, never a topic
|
|
562
|
+
- "Elon Musk" is a person associated with Tesla, SpaceX, xAI
|
|
563
|
+
- "LockBit 3.0" is a criminal organization / hacker group
|
|
564
|
+
- Email addresses with @ → email type
|
|
565
|
+
- 0x... addresses → wallet type
|
|
566
|
+
- IP addresses → ip type
|
|
567
|
+
- Domains with TLD → domain type
|
|
568
|
+
|
|
569
|
+
SINGLE-WORD DISAMBIGUATION (CRITICAL):
|
|
570
|
+
- A SINGLE made-up word (not a dictionary word, not a real first/last name)
|
|
571
|
+
is almost always a COMPANY, BRAND, or ORGANIZATION — NOT a person.
|
|
572
|
+
Examples: "Aviloop", "Spotify", "Canva", "Figma", "Stripe", "Notion", "Shopify"
|
|
573
|
+
→ target_type: "organization"
|
|
574
|
+
- A word that IS a real first name or last name (e.g., "Michael", "Chen", "Maria")
|
|
575
|
+
→ target_type: "person"
|
|
576
|
+
- Two words that look like First Last (e.g., "Satoshi Nakamoto", "Jane Smith")
|
|
577
|
+
→ target_type: "person"
|
|
578
|
+
- If the word contains company-like suffixes or patterns
|
|
579
|
+
(loop, ify, hub, ly, io, ai, oo, up, ify, box, pad, folio, base, flow, sync)
|
|
580
|
+
it is almost certainly a company → target_type: "organization"
|
|
581
|
+
- If the word sounds like a brand or product name, not a human name
|
|
582
|
+
→ target_type: "organization"
|
|
583
|
+
- When in doubt between person and organization for a single word,
|
|
584
|
+
PREFER organization — companies are far more common single-word OSINT targets.
|
|
585
|
+
|
|
586
|
+
JSON:"""
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
async def _llm_classify(query: str, call_llm) -> TargetProfile:
|
|
590
|
+
"""LLM fallback — only called when regex + GLiNER + Wikidata can't resolve."""
|
|
591
|
+
prompt = _LLM_FALLBACK_PROMPT.format(query=query[:500])
|
|
592
|
+
|
|
593
|
+
try:
|
|
594
|
+
raw = await call_llm(prompt, timeout=20)
|
|
595
|
+
except Exception:
|
|
596
|
+
return TargetProfile(
|
|
597
|
+
target_type="topic",
|
|
598
|
+
primary_name=query.strip(),
|
|
599
|
+
confidence=0.3,
|
|
600
|
+
raw_query=query,
|
|
601
|
+
classified_by="llm_timeout",
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
if not raw:
|
|
605
|
+
return TargetProfile(
|
|
606
|
+
target_type="topic",
|
|
607
|
+
primary_name=query.strip(),
|
|
608
|
+
confidence=0.3,
|
|
609
|
+
raw_query=query,
|
|
610
|
+
classified_by="llm_empty",
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
# Parse JSON
|
|
614
|
+
raw = raw.strip()
|
|
615
|
+
raw = re.sub(r"^```(?:json)?\s*", "", raw)
|
|
616
|
+
raw = re.sub(r"\s*```$", "", raw)
|
|
617
|
+
|
|
618
|
+
try:
|
|
619
|
+
parsed = json.loads(raw)
|
|
620
|
+
except json.JSONDecodeError:
|
|
621
|
+
m = re.search(r"\{.*\}", raw, re.DOTALL)
|
|
622
|
+
if m:
|
|
623
|
+
try:
|
|
624
|
+
parsed = json.loads(m.group(0))
|
|
625
|
+
except json.JSONDecodeError:
|
|
626
|
+
parsed = {}
|
|
627
|
+
else:
|
|
628
|
+
parsed = {}
|
|
629
|
+
|
|
630
|
+
target_type = str(parsed.get("target_type", "topic")).lower()
|
|
631
|
+
valid_types = {"person", "company", "organization", "domain", "wallet", "ip", "email"}
|
|
632
|
+
if target_type not in valid_types:
|
|
633
|
+
target_type = "topic"
|
|
634
|
+
|
|
635
|
+
# ── Post-LLM heuristic: single-word non-names are companies ──
|
|
636
|
+
# The LLM sometimes calls made-up words "person" (e.g., "Aviloop").
|
|
637
|
+
# If the target has no spaces and doesn't look like a human name, it's an org.
|
|
638
|
+
if target_type == "person" and " " not in query.strip():
|
|
639
|
+
query_lower = query.strip().lower()
|
|
640
|
+
# Real human first names (common ones — not exhaustive, just a safety check)
|
|
641
|
+
common_names = {
|
|
642
|
+
"john", "jane", "michael", "maria", "james", "sarah", "david",
|
|
643
|
+
"anna", "robert", "lisa", "william", "emma", "thomas", "olivia",
|
|
644
|
+
"daniel", "sophia", "matthew", "isabella", "alexander", "mia",
|
|
645
|
+
"joseph", "charlotte", "andrew", "amelia", "ryan", "harper",
|
|
646
|
+
"joshua", "evelyn", "ethan", "abigail", "christopher", "emily",
|
|
647
|
+
"nicholas", "elizabeth", "anthony", "sofia", "benjamin", "avery",
|
|
648
|
+
"samuel", "ella", "jacob", "scarlett", "logan", "grace",
|
|
649
|
+
"mohammed", "wei", "yuki", "satoshi", "filippo", "giulia",
|
|
650
|
+
"paolo", "lorenzo", "francesca", "marco", "elena", "andrea",
|
|
651
|
+
}
|
|
652
|
+
if query_lower not in common_names:
|
|
653
|
+
logger.info(
|
|
654
|
+
f"llm_type_correction: '{query}' classified as PERSON by LLM "
|
|
655
|
+
f"but is a single non-name word — switching to organization"
|
|
656
|
+
)
|
|
657
|
+
target_type = "organization"
|
|
658
|
+
|
|
659
|
+
return TargetProfile(
|
|
660
|
+
target_type=target_type,
|
|
661
|
+
primary_name=str(parsed.get("primary_name", query.strip())),
|
|
662
|
+
confidence=float(parsed.get("confidence", 0.5)),
|
|
663
|
+
raw_query=query,
|
|
664
|
+
associated_orgs=_list_str(parsed.get("associated_orgs", [])),
|
|
665
|
+
associated_domains=_list_str(parsed.get("associated_domains", [])),
|
|
666
|
+
social_handles=_list_str(parsed.get("social_handles", [])),
|
|
667
|
+
known_aliases=_list_str(parsed.get("known_aliases", [])),
|
|
668
|
+
locations=_list_str(parsed.get("locations", [])),
|
|
669
|
+
investigation_angles=_list_str(parsed.get("investigation_angles", [])),
|
|
670
|
+
classified_by="llm",
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def _list_str(val) -> list[str]:
|
|
675
|
+
if isinstance(val, list):
|
|
676
|
+
return [str(v) for v in val if str(v).strip()]
|
|
677
|
+
return []
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# ── Main entry point ─────────────────────────────────────────────
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
async def profile_target(
|
|
684
|
+
query: str,
|
|
685
|
+
call_llm=None,
|
|
686
|
+
skip_wikidata: bool = False,
|
|
687
|
+
) -> TargetProfile:
|
|
688
|
+
"""Profile an investigation target — fast, layered classification.
|
|
689
|
+
|
|
690
|
+
Steps:
|
|
691
|
+
1. Regex for unambiguous types (instant)
|
|
692
|
+
2. GLiNER NER for person vs organization (CPU, ~50ms)
|
|
693
|
+
3. Wikidata API for enrichment (REST, ~200ms)
|
|
694
|
+
4. LLM fallback only when all above fail (API call)
|
|
695
|
+
|
|
696
|
+
Returns a rich TargetProfile the orchestrator can use to drive
|
|
697
|
+
investigation phases.
|
|
698
|
+
"""
|
|
699
|
+
if not query or not query.strip():
|
|
700
|
+
return TargetProfile(
|
|
701
|
+
target_type="topic",
|
|
702
|
+
primary_name=query or "",
|
|
703
|
+
confidence=0.0,
|
|
704
|
+
classified_by="empty",
|
|
705
|
+
)
|
|
706
|
+
|
|
707
|
+
# ── 1. Regex ──
|
|
708
|
+
profile = _regex_classify(query)
|
|
709
|
+
if profile and profile.confidence >= 0.90:
|
|
710
|
+
if not skip_wikidata:
|
|
711
|
+
profile = await _wikidata_enrich(profile)
|
|
712
|
+
return profile
|
|
713
|
+
|
|
714
|
+
# ── 2. GLiNER ──
|
|
715
|
+
gliner_profile = _gliner_classify(query)
|
|
716
|
+
if gliner_profile and gliner_profile.confidence >= 0.70:
|
|
717
|
+
if not skip_wikidata:
|
|
718
|
+
gliner_profile = await _wikidata_enrich(gliner_profile)
|
|
719
|
+
return gliner_profile
|
|
720
|
+
|
|
721
|
+
# If regex found something low-confidence and GLiNER also found something,
|
|
722
|
+
# prefer the higher-confidence result
|
|
723
|
+
if profile and gliner_profile:
|
|
724
|
+
best = profile if profile.confidence >= gliner_profile.confidence else gliner_profile
|
|
725
|
+
if not skip_wikidata:
|
|
726
|
+
best = await _wikidata_enrich(best)
|
|
727
|
+
return best
|
|
728
|
+
|
|
729
|
+
if profile:
|
|
730
|
+
if not skip_wikidata:
|
|
731
|
+
profile = await _wikidata_enrich(profile)
|
|
732
|
+
return profile
|
|
733
|
+
|
|
734
|
+
# ── 3. LLM fallback ──
|
|
735
|
+
if call_llm:
|
|
736
|
+
llm_profile = await _llm_classify(query, call_llm)
|
|
737
|
+
if not skip_wikidata and llm_profile.target_type in ("person", "company", "organization"):
|
|
738
|
+
llm_profile = await _wikidata_enrich(llm_profile)
|
|
739
|
+
return llm_profile
|
|
740
|
+
|
|
741
|
+
# ── 4. Nothing worked ──
|
|
742
|
+
return TargetProfile(
|
|
743
|
+
target_type="topic",
|
|
744
|
+
primary_name=query.strip(),
|
|
745
|
+
confidence=0.2,
|
|
746
|
+
raw_query=query,
|
|
747
|
+
classified_by="none",
|
|
748
|
+
)
|