osintengine 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osintengine-1.0.2.dist-info/METADATA +311 -0
- osintengine-1.0.2.dist-info/RECORD +103 -0
- osintengine-1.0.2.dist-info/WHEEL +5 -0
- osintengine-1.0.2.dist-info/entry_points.txt +2 -0
- osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
- osintengine-1.0.2.dist-info/top_level.txt +2 -0
- src/watson/__init__.py +10 -0
- src/watson/agent/__init__.py +96 -0
- src/watson/agents/__init__.py +101 -0
- src/watson/agents/protocol.py +196 -0
- src/watson/auth/__init__.py +68 -0
- src/watson/auth/store.py +145 -0
- src/watson/conversation.py +68 -0
- src/watson/core/__init__.py +0 -0
- src/watson/core/models.py +96 -0
- src/watson/ethics.py +205 -0
- src/watson/exports.py +182 -0
- src/watson/graph/__init__.py +69 -0
- src/watson/graph/entities.py +207 -0
- src/watson/graph/graph.py +489 -0
- src/watson/graph/osint_framework.py +266 -0
- src/watson/graph/relationships.py +116 -0
- src/watson/graph/transforms.py +787 -0
- src/watson/infra/__init__.py +43 -0
- src/watson/infra/cache.py +171 -0
- src/watson/infra/ratelimit.py +125 -0
- src/watson/infra/resilience.py +239 -0
- src/watson/infra/retry.py +186 -0
- src/watson/metrics.py +128 -0
- src/watson/opsec/__init__.py +290 -0
- src/watson/orchestration/__init__.py +23 -0
- src/watson/orchestration/engine.py +5587 -0
- src/watson/orchestration/executor.py +96 -0
- src/watson/orchestration/intent.py +139 -0
- src/watson/orchestration/intent_classifier.py +45 -0
- src/watson/orchestration/llm_config.py +160 -0
- src/watson/orchestration/resolution.py +555 -0
- src/watson/orchestration/scraper.py +94 -0
- src/watson/orchestration/synthesis.py +726 -0
- src/watson/orchestration/target_profile.py +748 -0
- src/watson/persistence/__init__.py +18 -0
- src/watson/persistence/models.py +161 -0
- src/watson/persistence/store.py +230 -0
- src/watson/pipeline/__init__.py +16 -0
- src/watson/pipeline/pre_synthesis.py +621 -0
- src/watson/search.py +103 -0
- src/watson/serializers/stix.py +451 -0
- src/watson/tools/__init__.py +31 -0
- src/watson/tools/base.py +97 -0
- src/watson/tools/blockchain.py +359 -0
- src/watson/tools/captcha.py +276 -0
- src/watson/tools/conflict.py +107 -0
- src/watson/tools/corporate.py +287 -0
- src/watson/tools/darkweb.py +144 -0
- src/watson/tools/geolocation.py +347 -0
- src/watson/tools/image_video.py +108 -0
- src/watson/tools/marinetraffic.py +142 -0
- src/watson/tools/people.py +476 -0
- src/watson/tools/registry.py +56 -0
- src/watson/tools/satellite.py +118 -0
- src/watson/tools/scraper.py +745 -0
- src/watson/tools/shodan.py +129 -0
- src/watson/tools/social_media.py +160 -0
- src/watson/tools/websites.py +291 -0
- src/watson/tools/wikidata.py +398 -0
- src/watson/utils/__init__.py +1 -0
- src/watson/utils/helpers.py +49 -0
- src/watson/utils/http.py +119 -0
- src/watson/verification/__init__.py +245 -0
- watson/__init__.py +6 -0
- watson/agents/__init__.py +20 -0
- watson/agents/base.py +103 -0
- watson/agents/direct.py +225 -0
- watson/agents/hermes.py +275 -0
- watson/agents/hermes_mcp.py +292 -0
- watson/api_keys.py +176 -0
- watson/browser_scraper.py +481 -0
- watson/cli.py +836 -0
- watson/crossref.py +175 -0
- watson/ethics.py +20 -0
- watson/graph.py +406 -0
- watson/mcp_server.py +601 -0
- watson/memory.py +552 -0
- watson/neo4j_graph.py +124 -0
- watson/opsec/__init__.py +307 -0
- watson/reporter.py +537 -0
- watson/scheduler.py +486 -0
- watson/serializers/stix.py +451 -0
- watson/terminal.py +410 -0
- watson/toolkit.py +252 -0
- watson/toolkit_api.py +347 -0
- watson/toolkit_automation.py +95 -0
- watson/toolkit_registry.py +345 -0
- watson/verification/__init__.py +251 -0
- watson/web/__init__.py +1 -0
- watson/web/app.py +1650 -0
- watson/web/middleware.py +301 -0
- watson/web/static/assets/index-B7hPOc0z.js +278 -0
- watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
- watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
- watson/web/static/index.html +14 -0
- watson/web/templates/chat.html +2 -0
- watson/web/templates/investigation-map.html +429 -0
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bellingcat OSINT Toolkit Registry — 338 tools × 24 categories.
|
|
3
|
+
|
|
4
|
+
Loads the official Bellingcat toolkit CSV and provides:
|
|
5
|
+
- Search by category, keyword, target type
|
|
6
|
+
- Auto-classification: given a target, determines which tools apply
|
|
7
|
+
- URL templates for parameterized tool access
|
|
8
|
+
- Direct API integrations for tools with public endpoints
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import csv
|
|
14
|
+
import json
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
from urllib.parse import quote
|
|
20
|
+
|
|
21
|
+
# ── CSV path (downloaded from bellingcat/toolkit releases) ──────────
|
|
22
|
+
_CSV_PATH = Path(__file__).resolve().parent.parent / "data" / "bellingcat_toolkit.csv"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class BellingcatTool:
|
|
27
|
+
"""Single tool from the Bellingcat toolkit."""
|
|
28
|
+
category: str
|
|
29
|
+
name: str
|
|
30
|
+
url: str
|
|
31
|
+
description: str
|
|
32
|
+
cost: str # Free, Paid, Partially Free
|
|
33
|
+
details: str = "" # GitBook link
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def is_free(self) -> bool:
|
|
37
|
+
return "free" in self.cost.lower() and "paid" not in self.cost.lower()
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def is_api_accessible(self) -> bool:
|
|
41
|
+
"""Heuristic: tools that likely have programmatic API access."""
|
|
42
|
+
api_indicators = ["api.", "/api/", "api-", "api_", "graphql", "rest."]
|
|
43
|
+
return any(ind in self.url.lower() for ind in api_indicators)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def is_url_templateable(self) -> bool:
|
|
47
|
+
"""Heuristic: tools whose URL can be parameterized with a query."""
|
|
48
|
+
return bool(self.url) and self.url.startswith("http") and not self.is_api_accessible
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ── Target-type → relevant Bellingcat categories ────────────────────
|
|
52
|
+
TARGET_CATEGORY_MAP: dict[str, list[str]] = {
|
|
53
|
+
"person": [
|
|
54
|
+
"People", "Facebook", "Instagram", "Twitter/X", "Tiktok",
|
|
55
|
+
"Other Platforms", "Multiple Platforms", "Facial Recognition",
|
|
56
|
+
"Reverse Image Search", "Metadata", "Archiving",
|
|
57
|
+
"Telegram", "Youtube", # added — key social platforms
|
|
58
|
+
],
|
|
59
|
+
"company": [
|
|
60
|
+
"Companies & Finance", "Websites", "Archiving", "Data Organization & Analysis",
|
|
61
|
+
"Maps", "Transport", "Geolocation", "Metadata", # added — supply chain tracking
|
|
62
|
+
],
|
|
63
|
+
"domain": [
|
|
64
|
+
"Websites", "Archiving", "Data Organization & Analysis",
|
|
65
|
+
"Companies & Finance", "Metadata", "Maps", # added — WHOIS, DNS, IP geolocation
|
|
66
|
+
],
|
|
67
|
+
"email": [
|
|
68
|
+
"People", "Websites", "Other Platforms", "Data Organization & Analysis",
|
|
69
|
+
"Metadata", "Archiving", # added — breach tracking, archive search
|
|
70
|
+
],
|
|
71
|
+
"username": [
|
|
72
|
+
"People", "Multiple Platforms", "Other Platforms", "Facebook",
|
|
73
|
+
"Instagram", "Twitter/X", "Tiktok", "Youtube", "Telegram",
|
|
74
|
+
],
|
|
75
|
+
"phone": [
|
|
76
|
+
"People", "Other Platforms", "Telegram", "Metadata", # added metadata
|
|
77
|
+
],
|
|
78
|
+
"image": [
|
|
79
|
+
"Reverse Image Search", "Facial Recognition", "Metadata",
|
|
80
|
+
"Misc", "Geolocation", "Maps", "Satellite Imagery", "Street View",
|
|
81
|
+
],
|
|
82
|
+
"location": [
|
|
83
|
+
"Geolocation", "Maps", "Satellite Imagery", "Street View",
|
|
84
|
+
"Environment & Wildlife", "Conflict", "Transport",
|
|
85
|
+
],
|
|
86
|
+
"social_media": [
|
|
87
|
+
"Facebook", "Instagram", "Twitter/X", "Tiktok", "Youtube",
|
|
88
|
+
"Telegram", "Multiple Platforms", "Other Platforms", "Archiving",
|
|
89
|
+
],
|
|
90
|
+
"vehicle": [
|
|
91
|
+
"Transport", "Maps", "Satellite Imagery", "Geolocation", "Street View",
|
|
92
|
+
],
|
|
93
|
+
"ship": [
|
|
94
|
+
"Transport", "Maps", "Satellite Imagery",
|
|
95
|
+
],
|
|
96
|
+
"aircraft": [
|
|
97
|
+
"Transport", "Maps", "Satellite Imagery",
|
|
98
|
+
],
|
|
99
|
+
"organization": [
|
|
100
|
+
"Companies & Finance", "Websites", "Archiving", "Conflict",
|
|
101
|
+
"Environment & Wildlife", "People", "Metadata", # added metadata
|
|
102
|
+
],
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# ── URL templates for parameterized tools ───────────────────────────
|
|
107
|
+
URL_TEMPLATES: dict[str, str] = {
|
|
108
|
+
# People
|
|
109
|
+
"Google": "https://www.google.com/search?q={query}",
|
|
110
|
+
"DuckDuckGo": "https://duckduckgo.com/?q={query}",
|
|
111
|
+
"Bing": "https://www.bing.com/search?q={query}",
|
|
112
|
+
"Yandex": "https://yandex.com/search/?text={query}",
|
|
113
|
+
"Baidu": "https://www.baidu.com/s?wd={query}",
|
|
114
|
+
# Social
|
|
115
|
+
"Twitter Advanced Search": "https://twitter.com/search?q={query}&f=live",
|
|
116
|
+
"TweetDeck": "https://tweetdeck.twitter.com/",
|
|
117
|
+
"Social Blade": "https://socialblade.com/youtube/search/{query}",
|
|
118
|
+
"WhatsMyName Web": "https://whatsmyname.app/?q={query}",
|
|
119
|
+
"Namechk": "https://namechk.com/",
|
|
120
|
+
"Namecheckr": "https://www.namecheckr.com/search/{query}",
|
|
121
|
+
"Instant Username Search": "https://instantusername.com/?q={query}",
|
|
122
|
+
"CheckUsernames": "https://checkusernames.com/",
|
|
123
|
+
# Websites
|
|
124
|
+
"urlscan.io": "https://urlscan.io/search/#{query}",
|
|
125
|
+
"Shodan": "https://www.shodan.io/search?query={query}",
|
|
126
|
+
"Censys": "https://search.censys.io/search?resource=hosts&sort=RELEVANCE&per_page=25&virtual_hosts=EXCLUDE&q={query}",
|
|
127
|
+
"ZoomEye": "https://www.zoomeye.org/searchResult?q={query}",
|
|
128
|
+
"VirusTotal": "https://www.virustotal.com/gui/search/{query}",
|
|
129
|
+
"crt.sh": "https://crt.sh/?q={query}",
|
|
130
|
+
"DNSDumpster": "https://dnsdumpster.com/",
|
|
131
|
+
"SecurityTrails": "https://securitytrails.com/domain/{query}",
|
|
132
|
+
"BuiltWith": "https://builtwith.com/{query}",
|
|
133
|
+
"Wappalyzer": "https://www.wappalyzer.com/lookup/{query}",
|
|
134
|
+
# Companies
|
|
135
|
+
"OpenCorporates": "https://opencorporates.com/companies?q={query}",
|
|
136
|
+
"OpenSanctions": "https://opensanctions.org/search/?q={query}",
|
|
137
|
+
"ICIJ Offshore Leaks": "https://offshoreleaks.icij.org/search?q={query}",
|
|
138
|
+
"Companies House (UK)": "https://find-and-update.company-information.service.gov.uk/search?q={query}",
|
|
139
|
+
"SEC EDGAR": "https://www.sec.gov/cgi-bin/browse-edgar?company={query}&action=getcompany",
|
|
140
|
+
"EU Sanctions Map": "https://sanctionsmap.eu/#/main?search=%7B%22value%22:%22{query}%22%7D",
|
|
141
|
+
# Maps / Satellite
|
|
142
|
+
"Google Maps": "https://www.google.com/maps/search/{query}",
|
|
143
|
+
"Google Earth": "https://earth.google.com/web/search/{query}",
|
|
144
|
+
"OpenStreetMap": "https://www.openstreetmap.org/search?query={query}",
|
|
145
|
+
"Bing Maps": "https://www.bing.com/maps?q={query}",
|
|
146
|
+
"Yandex Maps": "https://yandex.com/maps/?text={query}",
|
|
147
|
+
"Wikimapia": "https://wikimapia.org/#lang=en&lat=0&lon=0&z=1&search={query}",
|
|
148
|
+
"Sentinel Hub EO Browser": "https://apps.sentinel-hub.com/eo-browser/?zoom=12&lat=0&lng=0&query={query}",
|
|
149
|
+
"Zoom Earth": "https://zoom.earth/#view={query}",
|
|
150
|
+
"SunCalc": "https://www.suncalc.org/#/{query}",
|
|
151
|
+
# Transport
|
|
152
|
+
"MarineTraffic": "https://www.marinetraffic.com/en/ais/home/centerx:0/centery:0/zoom:2",
|
|
153
|
+
"VesselFinder": "https://www.vesselfinder.com/?imo={query}",
|
|
154
|
+
"FlightRadar24": "https://www.flightradar24.com/data/search?q={query}",
|
|
155
|
+
"FlightAware": "https://flightaware.com/live/flight/{query}",
|
|
156
|
+
"ADS-B Exchange": "https://globe.adsbexchange.com/?icao={query}",
|
|
157
|
+
"OpenSky Network": "https://opensky-network.org/aircraft-profile?icao24={query}",
|
|
158
|
+
"ShipSpotting": "https://www.shipspotting.com/photos/search?query={query}",
|
|
159
|
+
# Archiving
|
|
160
|
+
"Wayback Machine": "https://web.archive.org/web/*/https://{query}",
|
|
161
|
+
"archive.today": "https://archive.today/?run=1&url=https://{query}",
|
|
162
|
+
"CachedView": "https://cachedview.com/?q={query}",
|
|
163
|
+
# Data / Leaks
|
|
164
|
+
"Have I Been Pwned": "https://haveibeenpwned.com/account/{query}",
|
|
165
|
+
"DeHashed": "https://dehashed.com/search?query={query}",
|
|
166
|
+
"Intelligence X": "https://intelx.io/?s={query}",
|
|
167
|
+
"GhostProject": "https://ghostproject.fr/",
|
|
168
|
+
"LeakCheck": "https://leakcheck.io/search?query={query}",
|
|
169
|
+
# Telegram
|
|
170
|
+
"Telemetrio": "https://telemetr.io/en/channels?query={query}",
|
|
171
|
+
"Tgstat": "https://tgstat.com/search?q={query}",
|
|
172
|
+
"TelegramDB": "https://telegramdb.org/search?q={query}",
|
|
173
|
+
# Conflict
|
|
174
|
+
"ACLED": "https://acleddata.com/data-export-tool/",
|
|
175
|
+
"LiveUAMap": "https://liveuamap.com/",
|
|
176
|
+
# Image / Video
|
|
177
|
+
"Google Images": "https://www.google.com/search?tbm=isch&q={query}",
|
|
178
|
+
"Google Lens": "https://lens.google.com/uploadbyurl?url={query}",
|
|
179
|
+
"Yandex Images": "https://yandex.com/images/search?rpt=imageview&url={query}",
|
|
180
|
+
"Bing Images": "https://www.bing.com/images/search?q={query}",
|
|
181
|
+
"TinEye": "https://tineye.com/search?url={query}",
|
|
182
|
+
"PimEyes": "https://pimeyes.com/en/search/{query}",
|
|
183
|
+
"FaceCheck.id": "https://facecheck.id/",
|
|
184
|
+
"InVID Verification": "https://citizenevidence.org/2024/03/05/invid-weverify-verification-plugin/",
|
|
185
|
+
# Added — high-value tools missing templates
|
|
186
|
+
"SpiderFoot HX": "https://www.spiderfoot.net/hx/?q={query}",
|
|
187
|
+
"Maigret": "https://github.com/soxoj/maigret",
|
|
188
|
+
"GHunt": "https://github.com/mxrch/GHunt",
|
|
189
|
+
"Holehe": "https://github.com/megadose/holehe",
|
|
190
|
+
"Blackbird": "https://github.com/p1ngul1n0/blackbird",
|
|
191
|
+
"Sherlock": "https://github.com/sherlock-project/sherlock",
|
|
192
|
+
"WhatsApp Group Links": "https://whatsgrouplink.com/?s={query}",
|
|
193
|
+
"Epieos": "https://epieos.com/?q={query}",
|
|
194
|
+
"That's Them": "https://thatsthem.com/people/{query}",
|
|
195
|
+
"FamilyTreeNow": "https://www.familytreenow.com/search/genealogy/results?first=&last={query}",
|
|
196
|
+
"TruePeopleSearch": "https://www.truepeoplesearch.com/results?name={query}",
|
|
197
|
+
"FastPeopleSearch": "https://www.fastpeoplesearch.com/name/{query}",
|
|
198
|
+
"PeekYou": "https://www.peekyou.com/{query}",
|
|
199
|
+
"Spokeo": "https://www.spokeo.com/{query}",
|
|
200
|
+
"Whitepages": "https://www.whitepages.com/name/{query}",
|
|
201
|
+
"BeenVerified": "https://www.beenverified.com/search?q={query}",
|
|
202
|
+
"USPhoneBook": "https://www.usphonebook.com/{query}",
|
|
203
|
+
"SearchSystems": "https://publicrecords.searchsystems.net/search.php?q={query}",
|
|
204
|
+
"OCCRP Aleph": "https://aleph.occrp.org/search?q={query}",
|
|
205
|
+
"Sanctions List Search": "https://sanctionssearch.ofac.treas.gov/Details.aspx?id={query}",
|
|
206
|
+
"Interpol Red Notices": "https://www.interpol.int/How-we-work/Notices/Red-Notices/View-Red-Notices",
|
|
207
|
+
"WikiLeaks": "https://search.wikileaks.org/?q={query}",
|
|
208
|
+
"Panama Papers": "https://offshoreleaks.icij.org/search?q={query}",
|
|
209
|
+
"GeoSpy": "https://geospy.ai/",
|
|
210
|
+
"Overpass Turbo": "https://overpass-turbo.eu/?Q={query}",
|
|
211
|
+
"Picarta": "https://picarta.ai/",
|
|
212
|
+
"YouTube Geofind": "https://mattw.io/youtube-geofind/location",
|
|
213
|
+
"GeoGuessr": "https://www.geoguessr.com/",
|
|
214
|
+
"PeakVisor": "https://peakvisor.com/search?q={query}",
|
|
215
|
+
"Wikimapia": "https://wikimapia.org/#lang=en&lat=0&lon=0&z=1&search={query}",
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class BellingcatRegistry:
|
|
220
|
+
"""Loads, indexes, and queries the Bellingcat OSINT Toolkit."""
|
|
221
|
+
|
|
222
|
+
def __init__(self, csv_path: Path | None = None):
|
|
223
|
+
self._tools: list[BellingcatTool] = []
|
|
224
|
+
self._by_category: dict[str, list[BellingcatTool]] = {}
|
|
225
|
+
self._by_name: dict[str, BellingcatTool] = {}
|
|
226
|
+
self._by_keyword: dict[str, list[BellingcatTool]] = {}
|
|
227
|
+
self._path = csv_path or _CSV_PATH
|
|
228
|
+
self._loaded = False
|
|
229
|
+
|
|
230
|
+
def load(self) -> None:
|
|
231
|
+
"""Load and index the CSV."""
|
|
232
|
+
if self._loaded:
|
|
233
|
+
return
|
|
234
|
+
if not self._path.exists():
|
|
235
|
+
raise FileNotFoundError(f"Bellingcat CSV not found at {self._path}. Run download first.")
|
|
236
|
+
|
|
237
|
+
with open(self._path, newline="", encoding="utf-8") as f:
|
|
238
|
+
reader = csv.DictReader(f)
|
|
239
|
+
for row in reader:
|
|
240
|
+
tool = BellingcatTool(
|
|
241
|
+
category=row.get("Category", "").strip(),
|
|
242
|
+
name=row.get("Name", "").strip(),
|
|
243
|
+
url=row.get("URL", "").strip(),
|
|
244
|
+
description=row.get("Description", "").strip(),
|
|
245
|
+
cost=row.get("Cost", "").strip(),
|
|
246
|
+
details=row.get("Details", "").strip(),
|
|
247
|
+
)
|
|
248
|
+
self._tools.append(tool)
|
|
249
|
+
self._by_category.setdefault(tool.category, []).append(tool)
|
|
250
|
+
self._by_name[tool.name.lower()] = tool
|
|
251
|
+
|
|
252
|
+
# Build keyword index from name + description
|
|
253
|
+
text = f"{tool.name} {tool.description}".lower()
|
|
254
|
+
words = set(re.findall(r"[a-z0-9]+", text))
|
|
255
|
+
for w in words:
|
|
256
|
+
if len(w) > 2: # Skip very short words
|
|
257
|
+
self._by_keyword.setdefault(w, []).append(tool)
|
|
258
|
+
|
|
259
|
+
self._loaded = True
|
|
260
|
+
|
|
261
|
+
@property
|
|
262
|
+
def tools(self) -> list[BellingcatTool]:
|
|
263
|
+
self.load()
|
|
264
|
+
return self._tools
|
|
265
|
+
|
|
266
|
+
@property
|
|
267
|
+
def categories(self) -> list[str]:
|
|
268
|
+
self.load()
|
|
269
|
+
return sorted(self._by_category.keys())
|
|
270
|
+
|
|
271
|
+
def get_category(self, category: str) -> list[BellingcatTool]:
|
|
272
|
+
self.load()
|
|
273
|
+
return self._by_category.get(category, [])
|
|
274
|
+
|
|
275
|
+
def get(self, name: str) -> BellingcatTool | None:
|
|
276
|
+
self.load()
|
|
277
|
+
return self._by_name.get(name.lower())
|
|
278
|
+
|
|
279
|
+
def search(self, query: str, limit: int = 20) -> list[BellingcatTool]:
|
|
280
|
+
"""Free-text search across tool names and descriptions."""
|
|
281
|
+
self.load()
|
|
282
|
+
tokens = set(re.findall(r"[a-z0-9]+", query.lower()))
|
|
283
|
+
scores: dict[str, tuple[int, BellingcatTool]] = {}
|
|
284
|
+
for token in tokens:
|
|
285
|
+
if len(token) <= 2:
|
|
286
|
+
continue
|
|
287
|
+
for tool in self._by_keyword.get(token, []):
|
|
288
|
+
key = tool.name.lower()
|
|
289
|
+
if key in scores:
|
|
290
|
+
scores[key] = (scores[key][0] + 1, tool)
|
|
291
|
+
else:
|
|
292
|
+
scores[key] = (1, tool)
|
|
293
|
+
ranked = sorted(scores.values(), key=lambda x: -x[0])
|
|
294
|
+
return [t for _, t in ranked[:limit]]
|
|
295
|
+
|
|
296
|
+
def classify(self, target_type: str) -> list[str]:
|
|
297
|
+
"""Return the Bellingcat categories relevant to a target type."""
|
|
298
|
+
return TARGET_CATEGORY_MAP.get(target_type.lower(), ["People", "Websites", "Companies & Finance"])
|
|
299
|
+
|
|
300
|
+
def tools_for_target(self, target_type: str) -> list[BellingcatTool]:
|
|
301
|
+
"""Return all Bellingcat tools relevant to a target type."""
|
|
302
|
+
self.load()
|
|
303
|
+
categories = self.classify(target_type)
|
|
304
|
+
tools: list[BellingcatTool] = []
|
|
305
|
+
seen: set[str] = set()
|
|
306
|
+
for cat in categories:
|
|
307
|
+
for tool in self._by_category.get(cat, []):
|
|
308
|
+
if tool.name.lower() not in seen:
|
|
309
|
+
tools.append(tool)
|
|
310
|
+
seen.add(tool.name.lower())
|
|
311
|
+
return tools
|
|
312
|
+
|
|
313
|
+
def build_url(self, tool_name: str, query: str) -> str | None:
|
|
314
|
+
"""Build a parameterized URL for a tool if a template exists."""
|
|
315
|
+
if tool_name in URL_TEMPLATES:
|
|
316
|
+
return URL_TEMPLATES[tool_name].format(query=quote(query, safe=""))
|
|
317
|
+
# Try case-insensitive match
|
|
318
|
+
for name, template in URL_TEMPLATES.items():
|
|
319
|
+
if name.lower() == tool_name.lower():
|
|
320
|
+
return template.format(query=quote(query, safe=""))
|
|
321
|
+
return None
|
|
322
|
+
|
|
323
|
+
def get_url_templates(self) -> dict[str, str]:
|
|
324
|
+
"""Return all URL templates."""
|
|
325
|
+
return dict(URL_TEMPLATES)
|
|
326
|
+
|
|
327
|
+
def summary(self) -> dict[str, Any]:
|
|
328
|
+
"""Return a summary of the registry for the frontend."""
|
|
329
|
+
self.load()
|
|
330
|
+
return {
|
|
331
|
+
"total_tools": len(self._tools),
|
|
332
|
+
"categories": {
|
|
333
|
+
cat: len(tools)
|
|
334
|
+
for cat, tools in sorted(self._by_category.items())
|
|
335
|
+
},
|
|
336
|
+
"target_types": list(TARGET_CATEGORY_MAP.keys()),
|
|
337
|
+
"free_tools": sum(1 for t in self._tools if t.is_free),
|
|
338
|
+
"paid_tools": sum(1 for t in self._tools if t.cost == "Paid"),
|
|
339
|
+
"partial_tools": sum(1 for t in self._tools if "Partially" in t.cost),
|
|
340
|
+
"url_templates": len(URL_TEMPLATES),
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
# ── Singleton ───────────────────────────────────────────────────────
|
|
345
|
+
registry = BellingcatRegistry()
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""LLM Verification Layer — second-pass validation to filter false positives.
|
|
2
|
+
|
|
3
|
+
After Phase 6 synthesis, each finding is re-examined by the LLM with a strict
|
|
4
|
+
verification prompt. The LLM checks:
|
|
5
|
+
1. Does the source URL actually support the finding?
|
|
6
|
+
2. Is the tier (CONFIRMED/PROBABLE/POSSIBLE) appropriate?
|
|
7
|
+
3. Is anything hallucinated (fake entities, fabricated URLs, impossible claims)?
|
|
8
|
+
4. Should the finding be downgraded or dropped?
|
|
9
|
+
|
|
10
|
+
Output: adjusted tier + verification_notes for each finding.
|
|
11
|
+
|
|
12
|
+
Thresholds:
|
|
13
|
+
- Drop findings where verification_confidence < 30
|
|
14
|
+
- Downgrade findings where LLM flags overconfidence
|
|
15
|
+
- Keep findings that pass verification unchanged
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import logging
|
|
22
|
+
import re
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger("watson.verify")
|
|
26
|
+
|
|
27
|
+
# ── Verification prompt ───────────────────────────────────────
|
|
28
|
+
|
|
29
|
+
_VERIFY_SYSTEM = """You are a strict OSINT quality-assurance auditor. Your job is to validate
|
|
30
|
+
intelligence findings and catch false positives, hallucinations, and overconfident claims.
|
|
31
|
+
|
|
32
|
+
For each finding, check:
|
|
33
|
+
1. Does the source URL actually exist and support the claim? If the URL looks fabricated
|
|
34
|
+
(random-looking domain, 404 patterns, impossible paths), flag it.
|
|
35
|
+
2. Is the confidence tier appropriate? CONFIRMED needs multi-source corroboration.
|
|
36
|
+
PROBABLE needs a credible source. POSSIBLE is for weak signals.
|
|
37
|
+
3. Are there any hallucinated elements? Invented person names, fake organizations,
|
|
38
|
+
impossible technical details, fabricated numbers/SAR IDs.
|
|
39
|
+
4. Is the finding logically coherent? Does the description actually match the title?
|
|
40
|
+
|
|
41
|
+
Respond with ONLY a JSON object. No markdown, no explanation outside the JSON."""
|
|
42
|
+
|
|
43
|
+
_VERIFY_PROMPT = """Review these OSINT findings for quality. Return a JSON dict with:
|
|
44
|
+
- "verifications": list of { "finding_id": str, "pass": bool, "adjusted_tier": str,
|
|
45
|
+
"verification_confidence": int (0-100), "issues": [str], "notes": str }
|
|
46
|
+
|
|
47
|
+
Findings to verify:
|
|
48
|
+
{findings_json}
|
|
49
|
+
|
|
50
|
+
Rules:
|
|
51
|
+
- CONFIRMED tier requires: real source URL + specific verifiable claim + no hallucination risk
|
|
52
|
+
- PROBABLE tier requires: credible source URL + plausible claim
|
|
53
|
+
- POSSIBLE tier: weak signal, correlation, or unverified mention
|
|
54
|
+
- UNLIKELY/UNVERIFIED: should be dropped (pass=false)
|
|
55
|
+
- If source URL doesn't look real, flag and downgrade
|
|
56
|
+
- If finding contains invented names/numbers, drop it
|
|
57
|
+
- If description is generic/vague, downgrade to POSSIBLE
|
|
58
|
+
- If finding is about a different entity than the target, flag it
|
|
59
|
+
|
|
60
|
+
Return ONLY valid JSON."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ── Verification engine ───────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
class FindingVerifier:
|
|
66
|
+
"""Second-pass LLM verification for OSINT findings."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, call_llm=None):
|
|
69
|
+
"""call_llm: async function (prompt, timeout, max_tokens) → str | None."""
|
|
70
|
+
self._call_llm = call_llm
|
|
71
|
+
|
|
72
|
+
async def verify(
|
|
73
|
+
self,
|
|
74
|
+
findings: list[Any],
|
|
75
|
+
query: str = "",
|
|
76
|
+
batch_size: int = 15,
|
|
77
|
+
) -> list[dict]:
|
|
78
|
+
"""Verify findings in batches. Returns list of verification results."""
|
|
79
|
+
if not findings or not self._call_llm:
|
|
80
|
+
return []
|
|
81
|
+
|
|
82
|
+
results: list[dict] = []
|
|
83
|
+
|
|
84
|
+
# Batch findings to avoid token overflow
|
|
85
|
+
for i in range(0, len(findings), batch_size):
|
|
86
|
+
batch = findings[i : i + batch_size]
|
|
87
|
+
batch_results = await self._verify_batch(batch, query)
|
|
88
|
+
results.extend(batch_results)
|
|
89
|
+
|
|
90
|
+
return results
|
|
91
|
+
|
|
92
|
+
async def _verify_batch(self, findings: list[Any], query: str) -> list[dict]:
|
|
93
|
+
"""Verify a batch of findings."""
|
|
94
|
+
if not self._call_llm:
|
|
95
|
+
return []
|
|
96
|
+
# Serialize findings to compact JSON
|
|
97
|
+
findings_data = []
|
|
98
|
+
for i, f in enumerate(findings):
|
|
99
|
+
fid = getattr(f, "id", getattr(f, "case_id", str(id(f))[:12]))
|
|
100
|
+
if not fid:
|
|
101
|
+
fid = f"finding-{i}"
|
|
102
|
+
findings_data.append({
|
|
103
|
+
"id": fid,
|
|
104
|
+
"title": getattr(f, "title", "")[:200],
|
|
105
|
+
"description": getattr(f, "description", "")[:300],
|
|
106
|
+
"tier": getattr(f, "tier", "UNVERIFIED"),
|
|
107
|
+
"source_url": getattr(f, "source_url", "")[:200],
|
|
108
|
+
"source_type": getattr(f, "source_type", "osint"),
|
|
109
|
+
"confidence": getattr(f, "confidence", 0.5),
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
prompt = _VERIFY_PROMPT.format(
|
|
113
|
+
findings_json=json.dumps(findings_data, indent=2)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
raw = await self._call_llm(
|
|
118
|
+
prompt,
|
|
119
|
+
timeout=120,
|
|
120
|
+
max_tokens=2000,
|
|
121
|
+
system=_VERIFY_SYSTEM,
|
|
122
|
+
)
|
|
123
|
+
if not raw:
|
|
124
|
+
logger.warning("verify: LLM returned empty — skipping verification")
|
|
125
|
+
return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
|
|
126
|
+
"adjusted_tier": fd["tier"], "issues": [], "notes": "Verification skipped (LLM unavailable)"}
|
|
127
|
+
for fd in findings_data]
|
|
128
|
+
|
|
129
|
+
parsed = self._parse_response(raw)
|
|
130
|
+
return self._merge_results(findings_data, parsed)
|
|
131
|
+
|
|
132
|
+
except Exception as e:
|
|
133
|
+
logger.warning("verify: batch failed: %s", e)
|
|
134
|
+
return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
|
|
135
|
+
"adjusted_tier": fd["tier"], "issues": [],
|
|
136
|
+
"notes": f"Verification error: {e}"}
|
|
137
|
+
for fd in findings_data]
|
|
138
|
+
|
|
139
|
+
def _parse_response(self, raw: str) -> dict:
|
|
140
|
+
"""Parse LLM verification response."""
|
|
141
|
+
# Strip markdown fences
|
|
142
|
+
text = raw.strip()
|
|
143
|
+
if text.startswith("```"):
|
|
144
|
+
text = re.sub(r"^```(?:json)?\s*", "", text)
|
|
145
|
+
text = re.sub(r"\s*```$", "", text)
|
|
146
|
+
|
|
147
|
+
# Try JSON parse
|
|
148
|
+
try:
|
|
149
|
+
return json.loads(text)
|
|
150
|
+
except json.JSONDecodeError:
|
|
151
|
+
# Try to extract JSON object
|
|
152
|
+
m = re.search(r"\{.*\}", text, re.DOTALL)
|
|
153
|
+
if m:
|
|
154
|
+
try:
|
|
155
|
+
return json.loads(m.group(0))
|
|
156
|
+
except json.JSONDecodeError:
|
|
157
|
+
pass
|
|
158
|
+
|
|
159
|
+
logger.warning("verify: could not parse response: %s", raw[:200])
|
|
160
|
+
return {}
|
|
161
|
+
|
|
162
|
+
def _merge_results(self, findings_data: list[dict], parsed: dict) -> list[dict]:
|
|
163
|
+
"""Merge parsed verifications with original finding data."""
|
|
164
|
+
verifications = parsed.get("verifications", [])
|
|
165
|
+
if not verifications:
|
|
166
|
+
# No structured verifications — pass everything
|
|
167
|
+
return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
|
|
168
|
+
"adjusted_tier": fd["tier"], "issues": [], "notes": "No verification data"}
|
|
169
|
+
for fd in findings_data]
|
|
170
|
+
|
|
171
|
+
# Build lookup
|
|
172
|
+
vmap: dict[str, dict] = {v.get("finding_id", ""): v for v in verifications}
|
|
173
|
+
|
|
174
|
+
results = []
|
|
175
|
+
for fd in findings_data:
|
|
176
|
+
fid = fd.get("id", str(id(fd))[:12])
|
|
177
|
+
ftier = fd.get("tier", "UNVERIFIED")
|
|
178
|
+
v = vmap.get(fid, {})
|
|
179
|
+
results.append({
|
|
180
|
+
"finding_id": fid,
|
|
181
|
+
"pass": v.get("pass", True),
|
|
182
|
+
"verification_confidence": v.get("verification_confidence", 50),
|
|
183
|
+
"adjusted_tier": v.get("adjusted_tier", ftier),
|
|
184
|
+
"issues": v.get("issues", []),
|
|
185
|
+
"notes": v.get("notes", ""),
|
|
186
|
+
})
|
|
187
|
+
return results
|
|
188
|
+
|
|
189
|
+
# ── Apply verification results to findings ────────────────
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def apply(finding: Any, verification: dict) -> tuple[Any, bool]:
|
|
193
|
+
"""Apply verification to a finding.
|
|
194
|
+
|
|
195
|
+
Returns (possibly_modified_finding, was_dropped).
|
|
196
|
+
"""
|
|
197
|
+
if not verification:
|
|
198
|
+
return finding, False
|
|
199
|
+
|
|
200
|
+
vconf = verification.get("verification_confidence", 50)
|
|
201
|
+
passed = verification.get("pass", True)
|
|
202
|
+
issues = verification.get("issues", [])
|
|
203
|
+
|
|
204
|
+
# Drop findings with very low verification confidence
|
|
205
|
+
if vconf < 30:
|
|
206
|
+
logger.info("verify: dropping finding %s (confidence=%d, issues=%s)",
|
|
207
|
+
getattr(finding, "id", "?"), vconf, issues)
|
|
208
|
+
return finding, True
|
|
209
|
+
|
|
210
|
+
# Downgrade tier if LLM suggests it
|
|
211
|
+
adjusted = verification.get("adjusted_tier", "")
|
|
212
|
+
if adjusted and hasattr(finding, "tier"):
|
|
213
|
+
tier_order = ["CONFIRMED", "PROBABLE", "POSSIBLE", "UNLIKELY", "UNVERIFIED"]
|
|
214
|
+
old_idx = tier_order.index(finding.tier) if finding.tier in tier_order else 4
|
|
215
|
+
new_idx = tier_order.index(adjusted) if adjusted in tier_order else old_idx
|
|
216
|
+
if new_idx > old_idx:
|
|
217
|
+
logger.info("verify: downgrading finding %s %s→%s",
|
|
218
|
+
getattr(finding, "id", "?"), finding.tier, adjusted)
|
|
219
|
+
finding.tier = adjusted
|
|
220
|
+
|
|
221
|
+
# Attach verification notes
|
|
222
|
+
notes = verification.get("notes", "")
|
|
223
|
+
if notes and hasattr(finding, "description"):
|
|
224
|
+
finding.description = f"[Verified: {notes}] {finding.description}"
|
|
225
|
+
|
|
226
|
+
if issues and hasattr(finding, "description"):
|
|
227
|
+
finding.description = f"[⚠ Issues: {'; '.join(issues[:3])}] {finding.description}"
|
|
228
|
+
|
|
229
|
+
return finding, False
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
# ── Factory ───────────────────────────────────────────────────
|
|
233
|
+
|
|
234
|
+
def create_verifier(provider: str | None = None) -> FindingVerifier | None:
|
|
235
|
+
"""Create a verifier using the configured LLM provider.
|
|
236
|
+
|
|
237
|
+
Returns None if no LLM is available.
|
|
238
|
+
"""
|
|
239
|
+
try:
|
|
240
|
+
try:
|
|
241
|
+
from watson.orchestration.llm_config import call_llm
|
|
242
|
+
except ImportError:
|
|
243
|
+
from src.watson.orchestration.llm_config import call_llm
|
|
244
|
+
|
|
245
|
+
async def _call(prompt, timeout=120, max_tokens=2000, system=""):
|
|
246
|
+
return await call_llm(prompt, timeout=timeout, max_tokens=max_tokens, system=system)
|
|
247
|
+
|
|
248
|
+
return FindingVerifier(call_llm=_call)
|
|
249
|
+
except ImportError:
|
|
250
|
+
logger.warning("verify: could not import llm_config — verification disabled")
|
|
251
|
+
return None
|
watson/web/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Watson Web — FastAPI product shell."""
|