osintengine 1.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. osintengine-1.0.2.dist-info/METADATA +311 -0
  2. osintengine-1.0.2.dist-info/RECORD +103 -0
  3. osintengine-1.0.2.dist-info/WHEEL +5 -0
  4. osintengine-1.0.2.dist-info/entry_points.txt +2 -0
  5. osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
  6. osintengine-1.0.2.dist-info/top_level.txt +2 -0
  7. src/watson/__init__.py +10 -0
  8. src/watson/agent/__init__.py +96 -0
  9. src/watson/agents/__init__.py +101 -0
  10. src/watson/agents/protocol.py +196 -0
  11. src/watson/auth/__init__.py +68 -0
  12. src/watson/auth/store.py +145 -0
  13. src/watson/conversation.py +68 -0
  14. src/watson/core/__init__.py +0 -0
  15. src/watson/core/models.py +96 -0
  16. src/watson/ethics.py +205 -0
  17. src/watson/exports.py +182 -0
  18. src/watson/graph/__init__.py +69 -0
  19. src/watson/graph/entities.py +207 -0
  20. src/watson/graph/graph.py +489 -0
  21. src/watson/graph/osint_framework.py +266 -0
  22. src/watson/graph/relationships.py +116 -0
  23. src/watson/graph/transforms.py +787 -0
  24. src/watson/infra/__init__.py +43 -0
  25. src/watson/infra/cache.py +171 -0
  26. src/watson/infra/ratelimit.py +125 -0
  27. src/watson/infra/resilience.py +239 -0
  28. src/watson/infra/retry.py +186 -0
  29. src/watson/metrics.py +128 -0
  30. src/watson/opsec/__init__.py +290 -0
  31. src/watson/orchestration/__init__.py +23 -0
  32. src/watson/orchestration/engine.py +5587 -0
  33. src/watson/orchestration/executor.py +96 -0
  34. src/watson/orchestration/intent.py +139 -0
  35. src/watson/orchestration/intent_classifier.py +45 -0
  36. src/watson/orchestration/llm_config.py +160 -0
  37. src/watson/orchestration/resolution.py +555 -0
  38. src/watson/orchestration/scraper.py +94 -0
  39. src/watson/orchestration/synthesis.py +726 -0
  40. src/watson/orchestration/target_profile.py +748 -0
  41. src/watson/persistence/__init__.py +18 -0
  42. src/watson/persistence/models.py +161 -0
  43. src/watson/persistence/store.py +230 -0
  44. src/watson/pipeline/__init__.py +16 -0
  45. src/watson/pipeline/pre_synthesis.py +621 -0
  46. src/watson/search.py +103 -0
  47. src/watson/serializers/stix.py +451 -0
  48. src/watson/tools/__init__.py +31 -0
  49. src/watson/tools/base.py +97 -0
  50. src/watson/tools/blockchain.py +359 -0
  51. src/watson/tools/captcha.py +276 -0
  52. src/watson/tools/conflict.py +107 -0
  53. src/watson/tools/corporate.py +287 -0
  54. src/watson/tools/darkweb.py +144 -0
  55. src/watson/tools/geolocation.py +347 -0
  56. src/watson/tools/image_video.py +108 -0
  57. src/watson/tools/marinetraffic.py +142 -0
  58. src/watson/tools/people.py +476 -0
  59. src/watson/tools/registry.py +56 -0
  60. src/watson/tools/satellite.py +118 -0
  61. src/watson/tools/scraper.py +745 -0
  62. src/watson/tools/shodan.py +129 -0
  63. src/watson/tools/social_media.py +160 -0
  64. src/watson/tools/websites.py +291 -0
  65. src/watson/tools/wikidata.py +398 -0
  66. src/watson/utils/__init__.py +1 -0
  67. src/watson/utils/helpers.py +49 -0
  68. src/watson/utils/http.py +119 -0
  69. src/watson/verification/__init__.py +245 -0
  70. watson/__init__.py +6 -0
  71. watson/agents/__init__.py +20 -0
  72. watson/agents/base.py +103 -0
  73. watson/agents/direct.py +225 -0
  74. watson/agents/hermes.py +275 -0
  75. watson/agents/hermes_mcp.py +292 -0
  76. watson/api_keys.py +176 -0
  77. watson/browser_scraper.py +481 -0
  78. watson/cli.py +836 -0
  79. watson/crossref.py +175 -0
  80. watson/ethics.py +20 -0
  81. watson/graph.py +406 -0
  82. watson/mcp_server.py +601 -0
  83. watson/memory.py +552 -0
  84. watson/neo4j_graph.py +124 -0
  85. watson/opsec/__init__.py +307 -0
  86. watson/reporter.py +537 -0
  87. watson/scheduler.py +486 -0
  88. watson/serializers/stix.py +451 -0
  89. watson/terminal.py +410 -0
  90. watson/toolkit.py +252 -0
  91. watson/toolkit_api.py +347 -0
  92. watson/toolkit_automation.py +95 -0
  93. watson/toolkit_registry.py +345 -0
  94. watson/verification/__init__.py +251 -0
  95. watson/web/__init__.py +1 -0
  96. watson/web/app.py +1650 -0
  97. watson/web/middleware.py +301 -0
  98. watson/web/static/assets/index-B7hPOc0z.js +278 -0
  99. watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
  100. watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
  101. watson/web/static/index.html +14 -0
  102. watson/web/templates/chat.html +2 -0
  103. watson/web/templates/investigation-map.html +429 -0
@@ -0,0 +1,748 @@
1
+ """
2
+ Target profiler — fast, accurate classification + rich decomposition.
3
+
4
+ Layered approach so the orchestrator knows not just WHAT is being
5
+ investigated but WHERE to start looking first.
6
+
7
+ Elon Musk → person + @elonmusk + Tesla/SpaceX + wikidata:Q317521
8
+ Stripe → company + stripe.com + founders + opencorp lookup
9
+ 0xdead... → wallet + ETH chain + etherscan link
10
+
11
+ Architecture:
12
+ Layer 1: Regex (instant) — email, IP, domain, phone, crypto wallets
13
+ Layer 2: GLiNER (CPU, ~50ms) — person vs organization disambiguation
14
+ Layer 3: Wikidata API (REST) — QID, occupation, employer, social handles
15
+ Layer 4: LLM (fallback only) — truly ambiguous cases
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import asyncio
21
+ import json
22
+ import logging
23
+ import re
24
+ from dataclasses import dataclass, field
25
+ from typing import Optional
26
+ from urllib.parse import quote
27
+
28
+ logger = logging.getLogger(__name__)
29
+
30
+
31
+ # ── Data model ────────────────────────────────────────────────────
32
+
33
+
34
+ @dataclass
35
+ class TargetProfile:
36
+ """Rich decomposition of an investigation target.
37
+
38
+ The orchestrator uses this to decide which sources to hit first.
39
+ """
40
+
41
+ target_type: str # person | company | organization | domain | wallet | ip | email
42
+ primary_name: str # cleaned canonical form
43
+ confidence: float # 0.0 — 1.0
44
+ raw_query: str = ""
45
+
46
+ # ── Associated entities ──
47
+ associated_orgs: list[str] = field(default_factory=list) # Tesla, SpaceX
48
+ associated_domains: list[str] = field(default_factory=list) # tesla.com
49
+ social_handles: list[str] = field(default_factory=list) # @elonmusk
50
+ known_aliases: list[str] = field(default_factory=list)
51
+ locations: list[str] = field(default_factory=list)
52
+
53
+ # ── Wikidata enrichment ──
54
+ wikidata_qid: Optional[str] = None # Q317521
55
+ wikidata_label: Optional[str] = None
56
+ wikidata_description: Optional[str] = None
57
+
58
+ # ── Orchestrator hints ──
59
+ suggested_sources: list[str] = field(default_factory=list)
60
+ # e.g. ["wikidata", "web_search", "twitter", "opencorporates"]
61
+ investigation_angles: list[str] = field(default_factory=list)
62
+ # e.g. ["social_media", "corporate_affiliations", "domain_infra"]
63
+
64
+ # ── Source-specific IDs ──
65
+ source_ids: dict[str, str] = field(default_factory=dict)
66
+ # e.g. {"opencorporates": "https://opencorporates.com/companies/us_de/..."}
67
+
68
+ # ── Classification path (for debugging) ──
69
+ classified_by: str = "" # "regex", "gliner", "wikidata", "llm"
70
+
71
+ def __post_init__(self):
72
+ if not self.suggested_sources:
73
+ self.suggested_sources = _DEFAULT_SOURCES.get(self.target_type, ["web_search"])
74
+ if not self.investigation_angles:
75
+ self.investigation_angles = _DEFAULT_ANGLES.get(self.target_type, ["general"])
76
+
77
+
78
+ # ── Default source mappings per target type ───────────────────────
79
+
80
+ _DEFAULT_SOURCES: dict[str, list[str]] = {
81
+ "person": ["wikidata", "web_search", "social_media", "news_search", "sanctions"],
82
+ "company": ["opencorporates", "crtsh", "web_search", "wikidata", "sanctions", "secedgar"],
83
+ "organization": ["opencorporates", "crtsh", "web_search", "wikidata", "sanctions"],
84
+ "domain": ["crtsh", "wayback", "dns", "web_search", "shodan"],
85
+ "wallet": ["blockchain_explorer", "web_search", "dark_web"],
86
+ "ip": ["shodan", "geolocation", "abuseipdb", "web_search"],
87
+ "email": ["hibp", "social_enumeration", "web_search", "dark_web"],
88
+ }
89
+
90
+ _DEFAULT_ANGLES: dict[str, list[str]] = {
91
+ "person": ["identity", "social_media", "professional", "legal", "web_presence"],
92
+ "company": ["corporate_structure", "domain_infra", "legal_regulatory", "leadership", "web_presence"],
93
+ "organization": ["structure", "domain_infra", "legal", "leadership", "web_presence"],
94
+ "domain": ["ssl_certs", "dns_records", "wayback_history", "associated_ips", "subdomains"],
95
+ "wallet": ["transaction_graph", "exchange_links", "entity_attribution", "dark_web_mentions"],
96
+ "ip": ["geolocation", "hosting", "abuse_history", "associated_domains", "ports"],
97
+ "email": ["breaches", "social_accounts", "dark_web", "domain_owner"],
98
+ }
99
+
100
+
101
+ # ── Layer 1: Deterministic regex ──────────────────────────────────
102
+ # Extended from SpiderFoot's battle-tested regexes, plus modern types
103
+
104
+ _REGEX_RULES: list[tuple[re.Pattern, str, float]] = [
105
+ # ── Email (must come before domain to catch user@host) ──
106
+ (re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$"), "email", 0.98),
107
+
108
+ # ── Ethereum / EVM wallet ──
109
+ (re.compile(r"^0x[a-fA-F0-9]{40}$"), "wallet", 0.98),
110
+
111
+ # ── Solana wallet ──
112
+ (re.compile(r"^[1-9A-HJ-NP-Za-km-z]{32,44}$"), "wallet", 0.85),
113
+
114
+ # ── Bitcoin address (legacy P2PKH) ──
115
+ (re.compile(r"^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$"), "wallet", 0.85),
116
+
117
+ # ── IPv4 ──
118
+ (re.compile(
119
+ r"^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$"
120
+ ), "ip", 0.95),
121
+
122
+ # ── IPv4 CIDR ──
123
+ (re.compile(
124
+ r"^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)/\d{1,2}$"
125
+ ), "ip", 0.95),
126
+
127
+ # ── IPv6 (simplified) ──
128
+ (re.compile(r"^[0-9a-fA-F:]+$"), "ip", 0.80),
129
+
130
+ # ── Phone (E.164) ──
131
+ (re.compile(r"^\+[1-9]\d{6,14}$"), "email", 0.90), # will be overridden by phone type
132
+
133
+ # ── Domain with known TLD ──
134
+ (re.compile(
135
+ r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+"
136
+ r"(?:com|org|net|io|gov|edu|mil|ru|cn|de|uk|fr|br|in|jp|kr|"
137
+ r"au|ca|ch|nl|se|no|dk|fi|pl|ua|ir|ng|sg|hk|tw|"
138
+ r"info|biz|co|me|tv|ai|app|dev|xyz|site|online|shop|"
139
+ r"tech|blog|news|media|wiki|finance|law|health|"
140
+ r"design|agency|consulting|ventures|capital|partners|"
141
+ r"foundation|institute|international)$",
142
+ re.IGNORECASE,
143
+ ), "domain", 0.90),
144
+
145
+ # ── URL ──
146
+ (re.compile(r"^https?://[^\s]+$", re.IGNORECASE), "domain", 0.70),
147
+ ]
148
+
149
+
150
+ def _regex_classify(query: str) -> Optional[TargetProfile]:
151
+ """Try deterministic regex classification. Fastest path."""
152
+ q = query.strip()
153
+
154
+ for pattern, ttype, conf in _REGEX_RULES:
155
+ if pattern.match(q):
156
+ # ── Enrich email ──
157
+ if ttype == "email" and "@" in q:
158
+ domain = q.split("@")[-1].lower()
159
+ profile = TargetProfile(
160
+ target_type="email",
161
+ primary_name=q.lower(),
162
+ confidence=conf,
163
+ raw_query=query,
164
+ associated_domains=[domain],
165
+ classified_by="regex",
166
+ )
167
+ return profile
168
+
169
+ # ── Enrich wallet ──
170
+ if ttype == "wallet":
171
+ chain = "eth" if q.startswith("0x") else "btc" if q[0] in "13" else "sol"
172
+ explorers = {
173
+ "eth": f"https://etherscan.io/address/{q}",
174
+ "btc": f"https://www.blockchain.com/explorer/addresses/btc/{q}",
175
+ "sol": f"https://solscan.io/account/{q}",
176
+ }
177
+ return TargetProfile(
178
+ target_type="wallet",
179
+ primary_name=q,
180
+ confidence=conf,
181
+ raw_query=query,
182
+ source_ids={"explorer": explorers.get(chain, "")},
183
+ investigation_angles=["transaction_graph", "exchange_links", "entity_attribution"],
184
+ classified_by="regex",
185
+ )
186
+
187
+ # ── Enrich IP ──
188
+ if ttype == "ip":
189
+ return TargetProfile(
190
+ target_type="ip",
191
+ primary_name=q,
192
+ confidence=conf,
193
+ raw_query=query,
194
+ source_ids={"shodan": f"https://www.shodan.io/host/{q}"},
195
+ classified_by="regex",
196
+ )
197
+
198
+ # ── Enrich domain ──
199
+ if ttype == "domain":
200
+ clean = q.replace("https://", "").replace("http://", "").rstrip("/")
201
+ # Extract base domain
202
+ parts = clean.split(".")
203
+ if len(parts) >= 2:
204
+ base = ".".join(parts[-2:]) if len(parts) > 2 else clean
205
+ else:
206
+ base = clean
207
+ return TargetProfile(
208
+ target_type="domain",
209
+ primary_name=clean,
210
+ confidence=conf,
211
+ raw_query=query,
212
+ associated_domains=[clean],
213
+ source_ids={
214
+ "crtsh": f"https://crt.sh/?q=%25.{base}",
215
+ "wayback": f"https://web.archive.org/web/*/{clean}",
216
+ },
217
+ classified_by="regex",
218
+ )
219
+
220
+ return None
221
+
222
+
223
+ # ── Layer 2: GLiNER NER ──────────────────────────────────────────
224
+ # Person vs organization disambiguation, runs locally on CPU
225
+
226
+ _GLINER_MODEL = None
227
+ _GLINER_LOADED = False
228
+
229
+ # Entity labels GLiNER extracts
230
+ _ENTITY_LABELS = [
231
+ "person",
232
+ "organization",
233
+ "company",
234
+ "email",
235
+ "phone number",
236
+ "cryptocurrency wallet",
237
+ "domain",
238
+ "location",
239
+ "username",
240
+ "product",
241
+ "hacker group",
242
+ "criminal organization",
243
+ ]
244
+
245
+
246
+ def _load_gliner():
247
+ """Lazy-load GLiNER model — only loaded once, on first use."""
248
+ global _GLINER_MODEL, _GLINER_LOADED
249
+ if _GLINER_LOADED:
250
+ return _GLINER_MODEL
251
+ _GLINER_LOADED = True
252
+ try:
253
+ from gliner import GLiNER
254
+ _GLINER_MODEL = GLiNER.from_pretrained("urchade/gliner_medium-v2.1")
255
+ logger.info("GLiNER model loaded (medium-v2.1, CPU)")
256
+ except Exception as e:
257
+ logger.warning(f"GLiNER failed to load: {e}. Falling back to regex + LLM.")
258
+ _GLINER_MODEL = None
259
+ return _GLINER_MODEL
260
+
261
+
262
+ def _gliner_classify(query: str) -> Optional[TargetProfile]:
263
+ """Use GLiNER to extract entities and determine target type.
264
+
265
+ Calibrated thresholds (empirically validated):
266
+ - person: 0.5 (Elon Musk 0.93, Dmitry Khoroshev 0.92)
267
+ - organization/company: 0.2 (Stripe 0.22, Binance 0.30, OpenAI 0.45)
268
+ - domain/email: 0.4
269
+ """
270
+ model = _load_gliner()
271
+ if model is None:
272
+ return None
273
+
274
+ try:
275
+ # Use lower threshold for org detection to catch single-word company names
276
+ entities = model.predict_entities(query, _ENTITY_LABELS, threshold=0.2)
277
+ except Exception as e:
278
+ logger.warning(f"GLiNER prediction failed: {e}")
279
+ return None
280
+
281
+ if not entities:
282
+ return None
283
+
284
+ # Separate entities by type
285
+ persons = []
286
+ orgs = []
287
+ domains_found = []
288
+ emails = []
289
+ locations = []
290
+ wallets = []
291
+
292
+ for ent in entities:
293
+ label = ent.get("label", "").lower()
294
+ text = ent.get("text", "").strip()
295
+ score = ent.get("score", 0.0)
296
+
297
+ # Person: high precision (0.5+) to avoid false positives
298
+ if label == "person" and score > 0.5:
299
+ persons.append(text)
300
+ # Organization/company: calibrated at 0.2 for single-word detection
301
+ elif label in ("organization", "company") and score > 0.25:
302
+ orgs.append(text)
303
+ elif label == "domain" and score > 0.4:
304
+ domains_found.append(text)
305
+ elif label == "email" and score > 0.5:
306
+ emails.append(text)
307
+ elif label == "location" and score > 0.4:
308
+ locations.append(text)
309
+ elif label in ("cryptocurrency wallet",) and score > 0.5:
310
+ wallets.append(text)
311
+
312
+ # ── Determine primary type ──
313
+ # Priority: wallet > email > domain > person > organization
314
+ if wallets:
315
+ return TargetProfile(
316
+ target_type="wallet",
317
+ primary_name=wallets[0],
318
+ confidence=0.85,
319
+ raw_query=query,
320
+ classified_by="gliner",
321
+ )
322
+
323
+ if emails:
324
+ domain = emails[0].split("@")[-1] if "@" in emails[0] else ""
325
+ return TargetProfile(
326
+ target_type="email",
327
+ primary_name=emails[0],
328
+ confidence=0.85,
329
+ associated_domains=[domain] if domain else [],
330
+ classified_by="gliner",
331
+ )
332
+
333
+ if domains_found:
334
+ return TargetProfile(
335
+ target_type="domain",
336
+ primary_name=domains_found[0],
337
+ confidence=0.80,
338
+ associated_domains=domains_found,
339
+ classified_by="gliner",
340
+ )
341
+
342
+ if persons:
343
+ profile = TargetProfile(
344
+ target_type="person",
345
+ primary_name=persons[0],
346
+ confidence=0.80,
347
+ raw_query=query,
348
+ known_aliases=persons[1:],
349
+ locations=locations,
350
+ classified_by="gliner",
351
+ )
352
+ if orgs:
353
+ profile.associated_orgs = orgs
354
+ return profile
355
+
356
+ if orgs:
357
+ return TargetProfile(
358
+ target_type="organization",
359
+ primary_name=orgs[0],
360
+ confidence=0.75,
361
+ raw_query=query,
362
+ associated_orgs=orgs[1:],
363
+ locations=locations,
364
+ classified_by="gliner",
365
+ )
366
+
367
+ return None
368
+
369
+
370
+ # ── Layer 3: Wikidata enrichment ──────────────────────────────────
371
+
372
+ async def _wikidata_enrich(profile: TargetProfile) -> TargetProfile:
373
+ """Enrich a profile with Wikidata data. No API key needed."""
374
+ search_term = profile.primary_name
375
+ if profile.raw_query and profile.raw_query != profile.primary_name:
376
+ search_term = profile.raw_query
377
+
378
+ try:
379
+ # Search for entity
380
+ url = (
381
+ "https://www.wikidata.org/w/api.php"
382
+ f"?action=wbsearchentities&search={quote(search_term)}"
383
+ "&language=en&format=json&limit=3"
384
+ )
385
+ import aiohttp
386
+ async with aiohttp.ClientSession() as session:
387
+ async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as resp:
388
+ data = await resp.json()
389
+ except Exception as e:
390
+ logger.debug(f"Wikidata search failed: {e}")
391
+ return profile
392
+
393
+ results = data.get("search", [])
394
+ if not results:
395
+ return profile
396
+
397
+ best = results[0]
398
+ profile.wikidata_qid = best.get("id")
399
+ profile.wikidata_label = best.get("label")
400
+ profile.wikidata_description = best.get("description", "")
401
+
402
+ # ── Fetch detailed entity data ──
403
+ if profile.wikidata_qid:
404
+ try:
405
+ detail_url = (
406
+ f"https://www.wikidata.org/wiki/Special:EntityData/"
407
+ f"{profile.wikidata_qid}.json"
408
+ )
409
+ import aiohttp
410
+ async with aiohttp.ClientSession() as session:
411
+ async with session.get(detail_url, timeout=aiohttp.ClientTimeout(total=10)) as resp:
412
+ entity_data = await resp.json()
413
+ except Exception:
414
+ return profile
415
+
416
+ claims = (
417
+ entity_data.get("entities", {})
418
+ .get(profile.wikidata_qid, {})
419
+ .get("claims", {})
420
+ )
421
+
422
+ # ── Extract social handles ──
423
+ # P2002 = Twitter username, P4264 = LinkedIn, P2013 = Facebook
424
+ twitter = _extract_wikidata_prop(claims, "P2002")
425
+ if twitter:
426
+ profile.social_handles.append(f"@{twitter}")
427
+
428
+ linkedin = _extract_wikidata_prop(claims, "P4264")
429
+ if linkedin:
430
+ profile.social_handles.append(f"linkedin:{linkedin}")
431
+
432
+ # ── Extract associated domains ──
433
+ # P856 = official website
434
+ website = _extract_wikidata_prop(claims, "P856")
435
+ if website:
436
+ domain = website.replace("https://", "").replace("http://", "").rstrip("/")
437
+ profile.associated_domains.append(domain)
438
+
439
+ # ── Extract employer / founder relationships ──
440
+ # P108 = employer, P112 = founded by
441
+ if profile.target_type == "person":
442
+ employer = _extract_wikidata_prop(claims, "P108")
443
+ if employer:
444
+ profile.associated_orgs.append(employer)
445
+
446
+ if profile.target_type in ("company", "organization"):
447
+ founder = _extract_wikidata_prop(claims, "P112")
448
+ if founder:
449
+ profile.known_aliases.append(founder)
450
+
451
+ # ── Extract location ──
452
+ # P17 = country, P159 = HQ location, P19 = place of birth
453
+ country = _extract_wikidata_prop(claims, "P17")
454
+ if country and country not in profile.locations:
455
+ profile.locations.append(country)
456
+
457
+ hq = _extract_wikidata_prop(claims, "P159")
458
+ if hq and hq not in profile.locations:
459
+ profile.locations.append(hq)
460
+
461
+ birthplace = _extract_wikidata_prop(claims, "P19")
462
+ if birthplace and birthplace not in profile.locations:
463
+ profile.locations.append(birthplace)
464
+
465
+ # ── Source-specific IDs ──
466
+ if profile.target_type in ("company", "organization"):
467
+ # OpenCorporates ID
468
+ oc_id = _extract_wikidata_prop(claims, "P1320")
469
+ if oc_id:
470
+ profile.source_ids["opencorporates"] = (
471
+ f"https://opencorporates.com/companies/{oc_id}"
472
+ )
473
+
474
+ # ── Correct misclassified type using Wikidata ──
475
+ # GLiNER/LLM can misclassify orgs as persons (e.g., "Aviloop" looks like a name).
476
+ # Wikidata is authoritative — use its instance_of and description to fix.
477
+ if profile.wikidata_qid and profile.wikidata_description:
478
+ desc_lower = profile.wikidata_description.lower()
479
+ org_keywords = [
480
+ "company", "corporation", "organization", "organisation",
481
+ "business", "website", "enterprise", "aviation", "airline",
482
+ "startup", "brand", "service", "platform", "firm", "agency",
483
+ "studio", "foundation", "institute", "association",
484
+ ]
485
+ is_org_by_desc = any(kw in desc_lower for kw in org_keywords)
486
+
487
+ # Check instance_of (P31) for business/org Wikidata types
488
+ instance_of = _extract_wikidata_prop(claims, "P31")
489
+ is_org_by_p31 = instance_of in (
490
+ "Q4830453", # business
491
+ "Q6881511", # enterprise
492
+ "Q43229", # organization
493
+ "Q35127", # website
494
+ "Q783794", # company
495
+ "Q167270", # trademark
496
+ "Q431289", # brand
497
+ )
498
+
499
+ if profile.target_type == "person" and (is_org_by_desc or is_org_by_p31):
500
+ logger.info(
501
+ f"wikidata_type_correction: '{profile.primary_name}' "
502
+ f"was classified as PERSON but Wikidata says '{profile.wikidata_description}' "
503
+ f"— switching to organization"
504
+ )
505
+ profile.target_type = "organization"
506
+ profile.investigation_angles = _DEFAULT_ANGLES.get("organization", ["general"])
507
+ profile.suggested_sources = _DEFAULT_SOURCES.get("organization", ["web_search"])
508
+ profile.classified_by = f"{profile.classified_by}+type_corrected"
509
+
510
+ # ── Update suggested sources based on what Wikidata told us ──
511
+ profile.suggested_sources = _DEFAULT_SOURCES.get(profile.target_type, ["web_search"])
512
+ profile.investigation_angles = _DEFAULT_ANGLES.get(profile.target_type, ["general"])
513
+
514
+ profile.classified_by = f"{profile.classified_by}+wikidata"
515
+ profile.confidence = min(profile.confidence + 0.1, 0.99)
516
+
517
+ return profile
518
+
519
+
520
+ def _extract_wikidata_prop(claims: dict, prop_id: str) -> Optional[str]:
521
+ """Extract a simple string value from Wikidata claims."""
522
+ prop = claims.get(prop_id, [])
523
+ if not prop:
524
+ return None
525
+ mainsnak = prop[0].get("mainsnak", {})
526
+ if mainsnak.get("snaktype") != "value":
527
+ return None
528
+ datavalue = mainsnak.get("datavalue", {}).get("value", {})
529
+
530
+ # String
531
+ if isinstance(datavalue, str):
532
+ return datavalue
533
+ # Entity reference (Q-item)
534
+ if isinstance(datavalue, dict):
535
+ item = datavalue.get("id", "")
536
+ # For Q-items, we return the ID — caller must resolve labels
537
+ return item
538
+ return None
539
+
540
+
541
+ # ── Layer 4: LLM fallback ────────────────────────────────────────
542
+
543
+ _LLM_FALLBACK_PROMPT = """You are an OSINT target classifier. Analyze this query and return STRICT JSON.
544
+
545
+ QUERY: {query}
546
+
547
+ Return:
548
+ {{
549
+ "target_type": "person|company|organization|domain|wallet|ip|email",
550
+ "primary_name": "canonical name",
551
+ "confidence": 0.0-1.0,
552
+ "associated_orgs": ["any orgs mentioned or implied"],
553
+ "associated_domains": ["any domains mentioned or implied"],
554
+ "social_handles": ["any social handles mentioned"],
555
+ "known_aliases": ["aliases"],
556
+ "locations": ["locations"],
557
+ "investigation_angles": ["angles to investigate"]
558
+ }}
559
+
560
+ RULES:
561
+ - "Stripe" is a company, never a topic
562
+ - "Elon Musk" is a person associated with Tesla, SpaceX, xAI
563
+ - "LockBit 3.0" is a criminal organization / hacker group
564
+ - Email addresses with @ → email type
565
+ - 0x... addresses → wallet type
566
+ - IP addresses → ip type
567
+ - Domains with TLD → domain type
568
+
569
+ SINGLE-WORD DISAMBIGUATION (CRITICAL):
570
+ - A SINGLE made-up word (not a dictionary word, not a real first/last name)
571
+ is almost always a COMPANY, BRAND, or ORGANIZATION — NOT a person.
572
+ Examples: "Aviloop", "Spotify", "Canva", "Figma", "Stripe", "Notion", "Shopify"
573
+ → target_type: "organization"
574
+ - A word that IS a real first name or last name (e.g., "Michael", "Chen", "Maria")
575
+ → target_type: "person"
576
+ - Two words that look like First Last (e.g., "Satoshi Nakamoto", "Jane Smith")
577
+ → target_type: "person"
578
+ - If the word contains company-like suffixes or patterns
579
+ (loop, ify, hub, ly, io, ai, oo, up, ify, box, pad, folio, base, flow, sync)
580
+ it is almost certainly a company → target_type: "organization"
581
+ - If the word sounds like a brand or product name, not a human name
582
+ → target_type: "organization"
583
+ - When in doubt between person and organization for a single word,
584
+ PREFER organization — companies are far more common single-word OSINT targets.
585
+
586
+ JSON:"""
587
+
588
+
589
+ async def _llm_classify(query: str, call_llm) -> TargetProfile:
590
+ """LLM fallback — only called when regex + GLiNER + Wikidata can't resolve."""
591
+ prompt = _LLM_FALLBACK_PROMPT.format(query=query[:500])
592
+
593
+ try:
594
+ raw = await call_llm(prompt, timeout=20)
595
+ except Exception:
596
+ return TargetProfile(
597
+ target_type="topic",
598
+ primary_name=query.strip(),
599
+ confidence=0.3,
600
+ raw_query=query,
601
+ classified_by="llm_timeout",
602
+ )
603
+
604
+ if not raw:
605
+ return TargetProfile(
606
+ target_type="topic",
607
+ primary_name=query.strip(),
608
+ confidence=0.3,
609
+ raw_query=query,
610
+ classified_by="llm_empty",
611
+ )
612
+
613
+ # Parse JSON
614
+ raw = raw.strip()
615
+ raw = re.sub(r"^```(?:json)?\s*", "", raw)
616
+ raw = re.sub(r"\s*```$", "", raw)
617
+
618
+ try:
619
+ parsed = json.loads(raw)
620
+ except json.JSONDecodeError:
621
+ m = re.search(r"\{.*\}", raw, re.DOTALL)
622
+ if m:
623
+ try:
624
+ parsed = json.loads(m.group(0))
625
+ except json.JSONDecodeError:
626
+ parsed = {}
627
+ else:
628
+ parsed = {}
629
+
630
+ target_type = str(parsed.get("target_type", "topic")).lower()
631
+ valid_types = {"person", "company", "organization", "domain", "wallet", "ip", "email"}
632
+ if target_type not in valid_types:
633
+ target_type = "topic"
634
+
635
+ # ── Post-LLM heuristic: single-word non-names are companies ──
636
+ # The LLM sometimes calls made-up words "person" (e.g., "Aviloop").
637
+ # If the target has no spaces and doesn't look like a human name, it's an org.
638
+ if target_type == "person" and " " not in query.strip():
639
+ query_lower = query.strip().lower()
640
+ # Real human first names (common ones — not exhaustive, just a safety check)
641
+ common_names = {
642
+ "john", "jane", "michael", "maria", "james", "sarah", "david",
643
+ "anna", "robert", "lisa", "william", "emma", "thomas", "olivia",
644
+ "daniel", "sophia", "matthew", "isabella", "alexander", "mia",
645
+ "joseph", "charlotte", "andrew", "amelia", "ryan", "harper",
646
+ "joshua", "evelyn", "ethan", "abigail", "christopher", "emily",
647
+ "nicholas", "elizabeth", "anthony", "sofia", "benjamin", "avery",
648
+ "samuel", "ella", "jacob", "scarlett", "logan", "grace",
649
+ "mohammed", "wei", "yuki", "satoshi", "filippo", "giulia",
650
+ "paolo", "lorenzo", "francesca", "marco", "elena", "andrea",
651
+ }
652
+ if query_lower not in common_names:
653
+ logger.info(
654
+ f"llm_type_correction: '{query}' classified as PERSON by LLM "
655
+ f"but is a single non-name word — switching to organization"
656
+ )
657
+ target_type = "organization"
658
+
659
+ return TargetProfile(
660
+ target_type=target_type,
661
+ primary_name=str(parsed.get("primary_name", query.strip())),
662
+ confidence=float(parsed.get("confidence", 0.5)),
663
+ raw_query=query,
664
+ associated_orgs=_list_str(parsed.get("associated_orgs", [])),
665
+ associated_domains=_list_str(parsed.get("associated_domains", [])),
666
+ social_handles=_list_str(parsed.get("social_handles", [])),
667
+ known_aliases=_list_str(parsed.get("known_aliases", [])),
668
+ locations=_list_str(parsed.get("locations", [])),
669
+ investigation_angles=_list_str(parsed.get("investigation_angles", [])),
670
+ classified_by="llm",
671
+ )
672
+
673
+
674
+ def _list_str(val) -> list[str]:
675
+ if isinstance(val, list):
676
+ return [str(v) for v in val if str(v).strip()]
677
+ return []
678
+
679
+
680
+ # ── Main entry point ─────────────────────────────────────────────
681
+
682
+
683
+ async def profile_target(
684
+ query: str,
685
+ call_llm=None,
686
+ skip_wikidata: bool = False,
687
+ ) -> TargetProfile:
688
+ """Profile an investigation target — fast, layered classification.
689
+
690
+ Steps:
691
+ 1. Regex for unambiguous types (instant)
692
+ 2. GLiNER NER for person vs organization (CPU, ~50ms)
693
+ 3. Wikidata API for enrichment (REST, ~200ms)
694
+ 4. LLM fallback only when all above fail (API call)
695
+
696
+ Returns a rich TargetProfile the orchestrator can use to drive
697
+ investigation phases.
698
+ """
699
+ if not query or not query.strip():
700
+ return TargetProfile(
701
+ target_type="topic",
702
+ primary_name=query or "",
703
+ confidence=0.0,
704
+ classified_by="empty",
705
+ )
706
+
707
+ # ── 1. Regex ──
708
+ profile = _regex_classify(query)
709
+ if profile and profile.confidence >= 0.90:
710
+ if not skip_wikidata:
711
+ profile = await _wikidata_enrich(profile)
712
+ return profile
713
+
714
+ # ── 2. GLiNER ──
715
+ gliner_profile = _gliner_classify(query)
716
+ if gliner_profile and gliner_profile.confidence >= 0.70:
717
+ if not skip_wikidata:
718
+ gliner_profile = await _wikidata_enrich(gliner_profile)
719
+ return gliner_profile
720
+
721
+ # If regex found something low-confidence and GLiNER also found something,
722
+ # prefer the higher-confidence result
723
+ if profile and gliner_profile:
724
+ best = profile if profile.confidence >= gliner_profile.confidence else gliner_profile
725
+ if not skip_wikidata:
726
+ best = await _wikidata_enrich(best)
727
+ return best
728
+
729
+ if profile:
730
+ if not skip_wikidata:
731
+ profile = await _wikidata_enrich(profile)
732
+ return profile
733
+
734
+ # ── 3. LLM fallback ──
735
+ if call_llm:
736
+ llm_profile = await _llm_classify(query, call_llm)
737
+ if not skip_wikidata and llm_profile.target_type in ("person", "company", "organization"):
738
+ llm_profile = await _wikidata_enrich(llm_profile)
739
+ return llm_profile
740
+
741
+ # ── 4. Nothing worked ──
742
+ return TargetProfile(
743
+ target_type="topic",
744
+ primary_name=query.strip(),
745
+ confidence=0.2,
746
+ raw_query=query,
747
+ classified_by="none",
748
+ )