osintengine 1.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. osintengine-1.0.2.dist-info/METADATA +311 -0
  2. osintengine-1.0.2.dist-info/RECORD +103 -0
  3. osintengine-1.0.2.dist-info/WHEEL +5 -0
  4. osintengine-1.0.2.dist-info/entry_points.txt +2 -0
  5. osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
  6. osintengine-1.0.2.dist-info/top_level.txt +2 -0
  7. src/watson/__init__.py +10 -0
  8. src/watson/agent/__init__.py +96 -0
  9. src/watson/agents/__init__.py +101 -0
  10. src/watson/agents/protocol.py +196 -0
  11. src/watson/auth/__init__.py +68 -0
  12. src/watson/auth/store.py +145 -0
  13. src/watson/conversation.py +68 -0
  14. src/watson/core/__init__.py +0 -0
  15. src/watson/core/models.py +96 -0
  16. src/watson/ethics.py +205 -0
  17. src/watson/exports.py +182 -0
  18. src/watson/graph/__init__.py +69 -0
  19. src/watson/graph/entities.py +207 -0
  20. src/watson/graph/graph.py +489 -0
  21. src/watson/graph/osint_framework.py +266 -0
  22. src/watson/graph/relationships.py +116 -0
  23. src/watson/graph/transforms.py +787 -0
  24. src/watson/infra/__init__.py +43 -0
  25. src/watson/infra/cache.py +171 -0
  26. src/watson/infra/ratelimit.py +125 -0
  27. src/watson/infra/resilience.py +239 -0
  28. src/watson/infra/retry.py +186 -0
  29. src/watson/metrics.py +128 -0
  30. src/watson/opsec/__init__.py +290 -0
  31. src/watson/orchestration/__init__.py +23 -0
  32. src/watson/orchestration/engine.py +5587 -0
  33. src/watson/orchestration/executor.py +96 -0
  34. src/watson/orchestration/intent.py +139 -0
  35. src/watson/orchestration/intent_classifier.py +45 -0
  36. src/watson/orchestration/llm_config.py +160 -0
  37. src/watson/orchestration/resolution.py +555 -0
  38. src/watson/orchestration/scraper.py +94 -0
  39. src/watson/orchestration/synthesis.py +726 -0
  40. src/watson/orchestration/target_profile.py +748 -0
  41. src/watson/persistence/__init__.py +18 -0
  42. src/watson/persistence/models.py +161 -0
  43. src/watson/persistence/store.py +230 -0
  44. src/watson/pipeline/__init__.py +16 -0
  45. src/watson/pipeline/pre_synthesis.py +621 -0
  46. src/watson/search.py +103 -0
  47. src/watson/serializers/stix.py +451 -0
  48. src/watson/tools/__init__.py +31 -0
  49. src/watson/tools/base.py +97 -0
  50. src/watson/tools/blockchain.py +359 -0
  51. src/watson/tools/captcha.py +276 -0
  52. src/watson/tools/conflict.py +107 -0
  53. src/watson/tools/corporate.py +287 -0
  54. src/watson/tools/darkweb.py +144 -0
  55. src/watson/tools/geolocation.py +347 -0
  56. src/watson/tools/image_video.py +108 -0
  57. src/watson/tools/marinetraffic.py +142 -0
  58. src/watson/tools/people.py +476 -0
  59. src/watson/tools/registry.py +56 -0
  60. src/watson/tools/satellite.py +118 -0
  61. src/watson/tools/scraper.py +745 -0
  62. src/watson/tools/shodan.py +129 -0
  63. src/watson/tools/social_media.py +160 -0
  64. src/watson/tools/websites.py +291 -0
  65. src/watson/tools/wikidata.py +398 -0
  66. src/watson/utils/__init__.py +1 -0
  67. src/watson/utils/helpers.py +49 -0
  68. src/watson/utils/http.py +119 -0
  69. src/watson/verification/__init__.py +245 -0
  70. watson/__init__.py +6 -0
  71. watson/agents/__init__.py +20 -0
  72. watson/agents/base.py +103 -0
  73. watson/agents/direct.py +225 -0
  74. watson/agents/hermes.py +275 -0
  75. watson/agents/hermes_mcp.py +292 -0
  76. watson/api_keys.py +176 -0
  77. watson/browser_scraper.py +481 -0
  78. watson/cli.py +836 -0
  79. watson/crossref.py +175 -0
  80. watson/ethics.py +20 -0
  81. watson/graph.py +406 -0
  82. watson/mcp_server.py +601 -0
  83. watson/memory.py +552 -0
  84. watson/neo4j_graph.py +124 -0
  85. watson/opsec/__init__.py +307 -0
  86. watson/reporter.py +537 -0
  87. watson/scheduler.py +486 -0
  88. watson/serializers/stix.py +451 -0
  89. watson/terminal.py +410 -0
  90. watson/toolkit.py +252 -0
  91. watson/toolkit_api.py +347 -0
  92. watson/toolkit_automation.py +95 -0
  93. watson/toolkit_registry.py +345 -0
  94. watson/verification/__init__.py +251 -0
  95. watson/web/__init__.py +1 -0
  96. watson/web/app.py +1650 -0
  97. watson/web/middleware.py +301 -0
  98. watson/web/static/assets/index-B7hPOc0z.js +278 -0
  99. watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
  100. watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
  101. watson/web/static/index.html +14 -0
  102. watson/web/templates/chat.html +2 -0
  103. watson/web/templates/investigation-map.html +429 -0
@@ -0,0 +1,345 @@
1
+ """
2
+ Bellingcat OSINT Toolkit Registry — 338 tools × 24 categories.
3
+
4
+ Loads the official Bellingcat toolkit CSV and provides:
5
+ - Search by category, keyword, target type
6
+ - Auto-classification: given a target, determines which tools apply
7
+ - URL templates for parameterized tool access
8
+ - Direct API integrations for tools with public endpoints
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import csv
14
+ import json
15
+ import re
16
+ from dataclasses import dataclass, field
17
+ from pathlib import Path
18
+ from typing import Any
19
+ from urllib.parse import quote
20
+
21
+ # ── CSV path (downloaded from bellingcat/toolkit releases) ──────────
22
+ _CSV_PATH = Path(__file__).resolve().parent.parent / "data" / "bellingcat_toolkit.csv"
23
+
24
+
25
+ @dataclass
26
+ class BellingcatTool:
27
+ """Single tool from the Bellingcat toolkit."""
28
+ category: str
29
+ name: str
30
+ url: str
31
+ description: str
32
+ cost: str # Free, Paid, Partially Free
33
+ details: str = "" # GitBook link
34
+
35
+ @property
36
+ def is_free(self) -> bool:
37
+ return "free" in self.cost.lower() and "paid" not in self.cost.lower()
38
+
39
+ @property
40
+ def is_api_accessible(self) -> bool:
41
+ """Heuristic: tools that likely have programmatic API access."""
42
+ api_indicators = ["api.", "/api/", "api-", "api_", "graphql", "rest."]
43
+ return any(ind in self.url.lower() for ind in api_indicators)
44
+
45
+ @property
46
+ def is_url_templateable(self) -> bool:
47
+ """Heuristic: tools whose URL can be parameterized with a query."""
48
+ return bool(self.url) and self.url.startswith("http") and not self.is_api_accessible
49
+
50
+
51
+ # ── Target-type → relevant Bellingcat categories ────────────────────
52
+ TARGET_CATEGORY_MAP: dict[str, list[str]] = {
53
+ "person": [
54
+ "People", "Facebook", "Instagram", "Twitter/X", "Tiktok",
55
+ "Other Platforms", "Multiple Platforms", "Facial Recognition",
56
+ "Reverse Image Search", "Metadata", "Archiving",
57
+ "Telegram", "Youtube", # added — key social platforms
58
+ ],
59
+ "company": [
60
+ "Companies & Finance", "Websites", "Archiving", "Data Organization & Analysis",
61
+ "Maps", "Transport", "Geolocation", "Metadata", # added — supply chain tracking
62
+ ],
63
+ "domain": [
64
+ "Websites", "Archiving", "Data Organization & Analysis",
65
+ "Companies & Finance", "Metadata", "Maps", # added — WHOIS, DNS, IP geolocation
66
+ ],
67
+ "email": [
68
+ "People", "Websites", "Other Platforms", "Data Organization & Analysis",
69
+ "Metadata", "Archiving", # added — breach tracking, archive search
70
+ ],
71
+ "username": [
72
+ "People", "Multiple Platforms", "Other Platforms", "Facebook",
73
+ "Instagram", "Twitter/X", "Tiktok", "Youtube", "Telegram",
74
+ ],
75
+ "phone": [
76
+ "People", "Other Platforms", "Telegram", "Metadata", # added metadata
77
+ ],
78
+ "image": [
79
+ "Reverse Image Search", "Facial Recognition", "Metadata",
80
+ "Misc", "Geolocation", "Maps", "Satellite Imagery", "Street View",
81
+ ],
82
+ "location": [
83
+ "Geolocation", "Maps", "Satellite Imagery", "Street View",
84
+ "Environment & Wildlife", "Conflict", "Transport",
85
+ ],
86
+ "social_media": [
87
+ "Facebook", "Instagram", "Twitter/X", "Tiktok", "Youtube",
88
+ "Telegram", "Multiple Platforms", "Other Platforms", "Archiving",
89
+ ],
90
+ "vehicle": [
91
+ "Transport", "Maps", "Satellite Imagery", "Geolocation", "Street View",
92
+ ],
93
+ "ship": [
94
+ "Transport", "Maps", "Satellite Imagery",
95
+ ],
96
+ "aircraft": [
97
+ "Transport", "Maps", "Satellite Imagery",
98
+ ],
99
+ "organization": [
100
+ "Companies & Finance", "Websites", "Archiving", "Conflict",
101
+ "Environment & Wildlife", "People", "Metadata", # added metadata
102
+ ],
103
+ }
104
+
105
+
106
+ # ── URL templates for parameterized tools ───────────────────────────
107
+ URL_TEMPLATES: dict[str, str] = {
108
+ # People
109
+ "Google": "https://www.google.com/search?q={query}",
110
+ "DuckDuckGo": "https://duckduckgo.com/?q={query}",
111
+ "Bing": "https://www.bing.com/search?q={query}",
112
+ "Yandex": "https://yandex.com/search/?text={query}",
113
+ "Baidu": "https://www.baidu.com/s?wd={query}",
114
+ # Social
115
+ "Twitter Advanced Search": "https://twitter.com/search?q={query}&f=live",
116
+ "TweetDeck": "https://tweetdeck.twitter.com/",
117
+ "Social Blade": "https://socialblade.com/youtube/search/{query}",
118
+ "WhatsMyName Web": "https://whatsmyname.app/?q={query}",
119
+ "Namechk": "https://namechk.com/",
120
+ "Namecheckr": "https://www.namecheckr.com/search/{query}",
121
+ "Instant Username Search": "https://instantusername.com/?q={query}",
122
+ "CheckUsernames": "https://checkusernames.com/",
123
+ # Websites
124
+ "urlscan.io": "https://urlscan.io/search/#{query}",
125
+ "Shodan": "https://www.shodan.io/search?query={query}",
126
+ "Censys": "https://search.censys.io/search?resource=hosts&sort=RELEVANCE&per_page=25&virtual_hosts=EXCLUDE&q={query}",
127
+ "ZoomEye": "https://www.zoomeye.org/searchResult?q={query}",
128
+ "VirusTotal": "https://www.virustotal.com/gui/search/{query}",
129
+ "crt.sh": "https://crt.sh/?q={query}",
130
+ "DNSDumpster": "https://dnsdumpster.com/",
131
+ "SecurityTrails": "https://securitytrails.com/domain/{query}",
132
+ "BuiltWith": "https://builtwith.com/{query}",
133
+ "Wappalyzer": "https://www.wappalyzer.com/lookup/{query}",
134
+ # Companies
135
+ "OpenCorporates": "https://opencorporates.com/companies?q={query}",
136
+ "OpenSanctions": "https://opensanctions.org/search/?q={query}",
137
+ "ICIJ Offshore Leaks": "https://offshoreleaks.icij.org/search?q={query}",
138
+ "Companies House (UK)": "https://find-and-update.company-information.service.gov.uk/search?q={query}",
139
+ "SEC EDGAR": "https://www.sec.gov/cgi-bin/browse-edgar?company={query}&action=getcompany",
140
+ "EU Sanctions Map": "https://sanctionsmap.eu/#/main?search=%7B%22value%22:%22{query}%22%7D",
141
+ # Maps / Satellite
142
+ "Google Maps": "https://www.google.com/maps/search/{query}",
143
+ "Google Earth": "https://earth.google.com/web/search/{query}",
144
+ "OpenStreetMap": "https://www.openstreetmap.org/search?query={query}",
145
+ "Bing Maps": "https://www.bing.com/maps?q={query}",
146
+ "Yandex Maps": "https://yandex.com/maps/?text={query}",
147
+ "Wikimapia": "https://wikimapia.org/#lang=en&lat=0&lon=0&z=1&search={query}",
148
+ "Sentinel Hub EO Browser": "https://apps.sentinel-hub.com/eo-browser/?zoom=12&lat=0&lng=0&query={query}",
149
+ "Zoom Earth": "https://zoom.earth/#view={query}",
150
+ "SunCalc": "https://www.suncalc.org/#/{query}",
151
+ # Transport
152
+ "MarineTraffic": "https://www.marinetraffic.com/en/ais/home/centerx:0/centery:0/zoom:2",
153
+ "VesselFinder": "https://www.vesselfinder.com/?imo={query}",
154
+ "FlightRadar24": "https://www.flightradar24.com/data/search?q={query}",
155
+ "FlightAware": "https://flightaware.com/live/flight/{query}",
156
+ "ADS-B Exchange": "https://globe.adsbexchange.com/?icao={query}",
157
+ "OpenSky Network": "https://opensky-network.org/aircraft-profile?icao24={query}",
158
+ "ShipSpotting": "https://www.shipspotting.com/photos/search?query={query}",
159
+ # Archiving
160
+ "Wayback Machine": "https://web.archive.org/web/*/https://{query}",
161
+ "archive.today": "https://archive.today/?run=1&url=https://{query}",
162
+ "CachedView": "https://cachedview.com/?q={query}",
163
+ # Data / Leaks
164
+ "Have I Been Pwned": "https://haveibeenpwned.com/account/{query}",
165
+ "DeHashed": "https://dehashed.com/search?query={query}",
166
+ "Intelligence X": "https://intelx.io/?s={query}",
167
+ "GhostProject": "https://ghostproject.fr/",
168
+ "LeakCheck": "https://leakcheck.io/search?query={query}",
169
+ # Telegram
170
+ "Telemetrio": "https://telemetr.io/en/channels?query={query}",
171
+ "Tgstat": "https://tgstat.com/search?q={query}",
172
+ "TelegramDB": "https://telegramdb.org/search?q={query}",
173
+ # Conflict
174
+ "ACLED": "https://acleddata.com/data-export-tool/",
175
+ "LiveUAMap": "https://liveuamap.com/",
176
+ # Image / Video
177
+ "Google Images": "https://www.google.com/search?tbm=isch&q={query}",
178
+ "Google Lens": "https://lens.google.com/uploadbyurl?url={query}",
179
+ "Yandex Images": "https://yandex.com/images/search?rpt=imageview&url={query}",
180
+ "Bing Images": "https://www.bing.com/images/search?q={query}",
181
+ "TinEye": "https://tineye.com/search?url={query}",
182
+ "PimEyes": "https://pimeyes.com/en/search/{query}",
183
+ "FaceCheck.id": "https://facecheck.id/",
184
+ "InVID Verification": "https://citizenevidence.org/2024/03/05/invid-weverify-verification-plugin/",
185
+ # Added — high-value tools missing templates
186
+ "SpiderFoot HX": "https://www.spiderfoot.net/hx/?q={query}",
187
+ "Maigret": "https://github.com/soxoj/maigret",
188
+ "GHunt": "https://github.com/mxrch/GHunt",
189
+ "Holehe": "https://github.com/megadose/holehe",
190
+ "Blackbird": "https://github.com/p1ngul1n0/blackbird",
191
+ "Sherlock": "https://github.com/sherlock-project/sherlock",
192
+ "WhatsApp Group Links": "https://whatsgrouplink.com/?s={query}",
193
+ "Epieos": "https://epieos.com/?q={query}",
194
+ "That's Them": "https://thatsthem.com/people/{query}",
195
+ "FamilyTreeNow": "https://www.familytreenow.com/search/genealogy/results?first=&last={query}",
196
+ "TruePeopleSearch": "https://www.truepeoplesearch.com/results?name={query}",
197
+ "FastPeopleSearch": "https://www.fastpeoplesearch.com/name/{query}",
198
+ "PeekYou": "https://www.peekyou.com/{query}",
199
+ "Spokeo": "https://www.spokeo.com/{query}",
200
+ "Whitepages": "https://www.whitepages.com/name/{query}",
201
+ "BeenVerified": "https://www.beenverified.com/search?q={query}",
202
+ "USPhoneBook": "https://www.usphonebook.com/{query}",
203
+ "SearchSystems": "https://publicrecords.searchsystems.net/search.php?q={query}",
204
+ "OCCRP Aleph": "https://aleph.occrp.org/search?q={query}",
205
+ "Sanctions List Search": "https://sanctionssearch.ofac.treas.gov/Details.aspx?id={query}",
206
+ "Interpol Red Notices": "https://www.interpol.int/How-we-work/Notices/Red-Notices/View-Red-Notices",
207
+ "WikiLeaks": "https://search.wikileaks.org/?q={query}",
208
+ "Panama Papers": "https://offshoreleaks.icij.org/search?q={query}",
209
+ "GeoSpy": "https://geospy.ai/",
210
+ "Overpass Turbo": "https://overpass-turbo.eu/?Q={query}",
211
+ "Picarta": "https://picarta.ai/",
212
+ "YouTube Geofind": "https://mattw.io/youtube-geofind/location",
213
+ "GeoGuessr": "https://www.geoguessr.com/",
214
+ "PeakVisor": "https://peakvisor.com/search?q={query}",
215
+ "Wikimapia": "https://wikimapia.org/#lang=en&lat=0&lon=0&z=1&search={query}",
216
+ }
217
+
218
+
219
+ class BellingcatRegistry:
220
+ """Loads, indexes, and queries the Bellingcat OSINT Toolkit."""
221
+
222
+ def __init__(self, csv_path: Path | None = None):
223
+ self._tools: list[BellingcatTool] = []
224
+ self._by_category: dict[str, list[BellingcatTool]] = {}
225
+ self._by_name: dict[str, BellingcatTool] = {}
226
+ self._by_keyword: dict[str, list[BellingcatTool]] = {}
227
+ self._path = csv_path or _CSV_PATH
228
+ self._loaded = False
229
+
230
+ def load(self) -> None:
231
+ """Load and index the CSV."""
232
+ if self._loaded:
233
+ return
234
+ if not self._path.exists():
235
+ raise FileNotFoundError(f"Bellingcat CSV not found at {self._path}. Run download first.")
236
+
237
+ with open(self._path, newline="", encoding="utf-8") as f:
238
+ reader = csv.DictReader(f)
239
+ for row in reader:
240
+ tool = BellingcatTool(
241
+ category=row.get("Category", "").strip(),
242
+ name=row.get("Name", "").strip(),
243
+ url=row.get("URL", "").strip(),
244
+ description=row.get("Description", "").strip(),
245
+ cost=row.get("Cost", "").strip(),
246
+ details=row.get("Details", "").strip(),
247
+ )
248
+ self._tools.append(tool)
249
+ self._by_category.setdefault(tool.category, []).append(tool)
250
+ self._by_name[tool.name.lower()] = tool
251
+
252
+ # Build keyword index from name + description
253
+ text = f"{tool.name} {tool.description}".lower()
254
+ words = set(re.findall(r"[a-z0-9]+", text))
255
+ for w in words:
256
+ if len(w) > 2: # Skip very short words
257
+ self._by_keyword.setdefault(w, []).append(tool)
258
+
259
+ self._loaded = True
260
+
261
+ @property
262
+ def tools(self) -> list[BellingcatTool]:
263
+ self.load()
264
+ return self._tools
265
+
266
+ @property
267
+ def categories(self) -> list[str]:
268
+ self.load()
269
+ return sorted(self._by_category.keys())
270
+
271
+ def get_category(self, category: str) -> list[BellingcatTool]:
272
+ self.load()
273
+ return self._by_category.get(category, [])
274
+
275
+ def get(self, name: str) -> BellingcatTool | None:
276
+ self.load()
277
+ return self._by_name.get(name.lower())
278
+
279
+ def search(self, query: str, limit: int = 20) -> list[BellingcatTool]:
280
+ """Free-text search across tool names and descriptions."""
281
+ self.load()
282
+ tokens = set(re.findall(r"[a-z0-9]+", query.lower()))
283
+ scores: dict[str, tuple[int, BellingcatTool]] = {}
284
+ for token in tokens:
285
+ if len(token) <= 2:
286
+ continue
287
+ for tool in self._by_keyword.get(token, []):
288
+ key = tool.name.lower()
289
+ if key in scores:
290
+ scores[key] = (scores[key][0] + 1, tool)
291
+ else:
292
+ scores[key] = (1, tool)
293
+ ranked = sorted(scores.values(), key=lambda x: -x[0])
294
+ return [t for _, t in ranked[:limit]]
295
+
296
+ def classify(self, target_type: str) -> list[str]:
297
+ """Return the Bellingcat categories relevant to a target type."""
298
+ return TARGET_CATEGORY_MAP.get(target_type.lower(), ["People", "Websites", "Companies & Finance"])
299
+
300
+ def tools_for_target(self, target_type: str) -> list[BellingcatTool]:
301
+ """Return all Bellingcat tools relevant to a target type."""
302
+ self.load()
303
+ categories = self.classify(target_type)
304
+ tools: list[BellingcatTool] = []
305
+ seen: set[str] = set()
306
+ for cat in categories:
307
+ for tool in self._by_category.get(cat, []):
308
+ if tool.name.lower() not in seen:
309
+ tools.append(tool)
310
+ seen.add(tool.name.lower())
311
+ return tools
312
+
313
+ def build_url(self, tool_name: str, query: str) -> str | None:
314
+ """Build a parameterized URL for a tool if a template exists."""
315
+ if tool_name in URL_TEMPLATES:
316
+ return URL_TEMPLATES[tool_name].format(query=quote(query, safe=""))
317
+ # Try case-insensitive match
318
+ for name, template in URL_TEMPLATES.items():
319
+ if name.lower() == tool_name.lower():
320
+ return template.format(query=quote(query, safe=""))
321
+ return None
322
+
323
+ def get_url_templates(self) -> dict[str, str]:
324
+ """Return all URL templates."""
325
+ return dict(URL_TEMPLATES)
326
+
327
+ def summary(self) -> dict[str, Any]:
328
+ """Return a summary of the registry for the frontend."""
329
+ self.load()
330
+ return {
331
+ "total_tools": len(self._tools),
332
+ "categories": {
333
+ cat: len(tools)
334
+ for cat, tools in sorted(self._by_category.items())
335
+ },
336
+ "target_types": list(TARGET_CATEGORY_MAP.keys()),
337
+ "free_tools": sum(1 for t in self._tools if t.is_free),
338
+ "paid_tools": sum(1 for t in self._tools if t.cost == "Paid"),
339
+ "partial_tools": sum(1 for t in self._tools if "Partially" in t.cost),
340
+ "url_templates": len(URL_TEMPLATES),
341
+ }
342
+
343
+
344
+ # ── Singleton ───────────────────────────────────────────────────────
345
+ registry = BellingcatRegistry()
@@ -0,0 +1,251 @@
1
+ """LLM Verification Layer — second-pass validation to filter false positives.
2
+
3
+ After Phase 6 synthesis, each finding is re-examined by the LLM with a strict
4
+ verification prompt. The LLM checks:
5
+ 1. Does the source URL actually support the finding?
6
+ 2. Is the tier (CONFIRMED/PROBABLE/POSSIBLE) appropriate?
7
+ 3. Is anything hallucinated (fake entities, fabricated URLs, impossible claims)?
8
+ 4. Should the finding be downgraded or dropped?
9
+
10
+ Output: adjusted tier + verification_notes for each finding.
11
+
12
+ Thresholds:
13
+ - Drop findings where verification_confidence < 30
14
+ - Downgrade findings where LLM flags overconfidence
15
+ - Keep findings that pass verification unchanged
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import logging
22
+ import re
23
+ from typing import Any
24
+
25
+ logger = logging.getLogger("watson.verify")
26
+
27
+ # ── Verification prompt ───────────────────────────────────────
28
+
29
+ _VERIFY_SYSTEM = """You are a strict OSINT quality-assurance auditor. Your job is to validate
30
+ intelligence findings and catch false positives, hallucinations, and overconfident claims.
31
+
32
+ For each finding, check:
33
+ 1. Does the source URL actually exist and support the claim? If the URL looks fabricated
34
+ (random-looking domain, 404 patterns, impossible paths), flag it.
35
+ 2. Is the confidence tier appropriate? CONFIRMED needs multi-source corroboration.
36
+ PROBABLE needs a credible source. POSSIBLE is for weak signals.
37
+ 3. Are there any hallucinated elements? Invented person names, fake organizations,
38
+ impossible technical details, fabricated numbers/SAR IDs.
39
+ 4. Is the finding logically coherent? Does the description actually match the title?
40
+
41
+ Respond with ONLY a JSON object. No markdown, no explanation outside the JSON."""
42
+
43
+ _VERIFY_PROMPT = """Review these OSINT findings for quality. Return a JSON dict with:
44
+ - "verifications": list of { "finding_id": str, "pass": bool, "adjusted_tier": str,
45
+ "verification_confidence": int (0-100), "issues": [str], "notes": str }
46
+
47
+ Findings to verify:
48
+ {findings_json}
49
+
50
+ Rules:
51
+ - CONFIRMED tier requires: real source URL + specific verifiable claim + no hallucination risk
52
+ - PROBABLE tier requires: credible source URL + plausible claim
53
+ - POSSIBLE tier: weak signal, correlation, or unverified mention
54
+ - UNLIKELY/UNVERIFIED: should be dropped (pass=false)
55
+ - If source URL doesn't look real, flag and downgrade
56
+ - If finding contains invented names/numbers, drop it
57
+ - If description is generic/vague, downgrade to POSSIBLE
58
+ - If finding is about a different entity than the target, flag it
59
+
60
+ Return ONLY valid JSON."""
61
+
62
+
63
+ # ── Verification engine ───────────────────────────────────────
64
+
65
+ class FindingVerifier:
66
+ """Second-pass LLM verification for OSINT findings."""
67
+
68
+ def __init__(self, call_llm=None):
69
+ """call_llm: async function (prompt, timeout, max_tokens) → str | None."""
70
+ self._call_llm = call_llm
71
+
72
+ async def verify(
73
+ self,
74
+ findings: list[Any],
75
+ query: str = "",
76
+ batch_size: int = 15,
77
+ ) -> list[dict]:
78
+ """Verify findings in batches. Returns list of verification results."""
79
+ if not findings or not self._call_llm:
80
+ return []
81
+
82
+ results: list[dict] = []
83
+
84
+ # Batch findings to avoid token overflow
85
+ for i in range(0, len(findings), batch_size):
86
+ batch = findings[i : i + batch_size]
87
+ batch_results = await self._verify_batch(batch, query)
88
+ results.extend(batch_results)
89
+
90
+ return results
91
+
92
+ async def _verify_batch(self, findings: list[Any], query: str) -> list[dict]:
93
+ """Verify a batch of findings."""
94
+ if not self._call_llm:
95
+ return []
96
+ # Serialize findings to compact JSON
97
+ findings_data = []
98
+ for i, f in enumerate(findings):
99
+ fid = getattr(f, "id", getattr(f, "case_id", str(id(f))[:12]))
100
+ if not fid:
101
+ fid = f"finding-{i}"
102
+ findings_data.append({
103
+ "id": fid,
104
+ "title": getattr(f, "title", "")[:200],
105
+ "description": getattr(f, "description", "")[:300],
106
+ "tier": getattr(f, "tier", "UNVERIFIED"),
107
+ "source_url": getattr(f, "source_url", "")[:200],
108
+ "source_type": getattr(f, "source_type", "osint"),
109
+ "confidence": getattr(f, "confidence", 0.5),
110
+ })
111
+
112
+ prompt = _VERIFY_PROMPT.format(
113
+ findings_json=json.dumps(findings_data, indent=2)
114
+ )
115
+
116
+ try:
117
+ raw = await self._call_llm(
118
+ prompt,
119
+ timeout=120,
120
+ max_tokens=2000,
121
+ system=_VERIFY_SYSTEM,
122
+ )
123
+ if not raw:
124
+ logger.warning("verify: LLM returned empty — skipping verification")
125
+ return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
126
+ "adjusted_tier": fd["tier"], "issues": [], "notes": "Verification skipped (LLM unavailable)"}
127
+ for fd in findings_data]
128
+
129
+ parsed = self._parse_response(raw)
130
+ return self._merge_results(findings_data, parsed)
131
+
132
+ except Exception as e:
133
+ logger.warning("verify: batch failed: %s", e)
134
+ return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
135
+ "adjusted_tier": fd["tier"], "issues": [],
136
+ "notes": f"Verification error: {e}"}
137
+ for fd in findings_data]
138
+
139
+ def _parse_response(self, raw: str) -> dict:
140
+ """Parse LLM verification response."""
141
+ # Strip markdown fences
142
+ text = raw.strip()
143
+ if text.startswith("```"):
144
+ text = re.sub(r"^```(?:json)?\s*", "", text)
145
+ text = re.sub(r"\s*```$", "", text)
146
+
147
+ # Try JSON parse
148
+ try:
149
+ return json.loads(text)
150
+ except json.JSONDecodeError:
151
+ # Try to extract JSON object
152
+ m = re.search(r"\{.*\}", text, re.DOTALL)
153
+ if m:
154
+ try:
155
+ return json.loads(m.group(0))
156
+ except json.JSONDecodeError:
157
+ pass
158
+
159
+ logger.warning("verify: could not parse response: %s", raw[:200])
160
+ return {}
161
+
162
+ def _merge_results(self, findings_data: list[dict], parsed: dict) -> list[dict]:
163
+ """Merge parsed verifications with original finding data."""
164
+ verifications = parsed.get("verifications", [])
165
+ if not verifications:
166
+ # No structured verifications — pass everything
167
+ return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
168
+ "adjusted_tier": fd["tier"], "issues": [], "notes": "No verification data"}
169
+ for fd in findings_data]
170
+
171
+ # Build lookup
172
+ vmap: dict[str, dict] = {v.get("finding_id", ""): v for v in verifications}
173
+
174
+ results = []
175
+ for fd in findings_data:
176
+ fid = fd.get("id", str(id(fd))[:12])
177
+ ftier = fd.get("tier", "UNVERIFIED")
178
+ v = vmap.get(fid, {})
179
+ results.append({
180
+ "finding_id": fid,
181
+ "pass": v.get("pass", True),
182
+ "verification_confidence": v.get("verification_confidence", 50),
183
+ "adjusted_tier": v.get("adjusted_tier", ftier),
184
+ "issues": v.get("issues", []),
185
+ "notes": v.get("notes", ""),
186
+ })
187
+ return results
188
+
189
+ # ── Apply verification results to findings ────────────────
190
+
191
+ @staticmethod
192
+ def apply(finding: Any, verification: dict) -> tuple[Any, bool]:
193
+ """Apply verification to a finding.
194
+
195
+ Returns (possibly_modified_finding, was_dropped).
196
+ """
197
+ if not verification:
198
+ return finding, False
199
+
200
+ vconf = verification.get("verification_confidence", 50)
201
+ passed = verification.get("pass", True)
202
+ issues = verification.get("issues", [])
203
+
204
+ # Drop findings with very low verification confidence
205
+ if vconf < 30:
206
+ logger.info("verify: dropping finding %s (confidence=%d, issues=%s)",
207
+ getattr(finding, "id", "?"), vconf, issues)
208
+ return finding, True
209
+
210
+ # Downgrade tier if LLM suggests it
211
+ adjusted = verification.get("adjusted_tier", "")
212
+ if adjusted and hasattr(finding, "tier"):
213
+ tier_order = ["CONFIRMED", "PROBABLE", "POSSIBLE", "UNLIKELY", "UNVERIFIED"]
214
+ old_idx = tier_order.index(finding.tier) if finding.tier in tier_order else 4
215
+ new_idx = tier_order.index(adjusted) if adjusted in tier_order else old_idx
216
+ if new_idx > old_idx:
217
+ logger.info("verify: downgrading finding %s %s→%s",
218
+ getattr(finding, "id", "?"), finding.tier, adjusted)
219
+ finding.tier = adjusted
220
+
221
+ # Attach verification notes
222
+ notes = verification.get("notes", "")
223
+ if notes and hasattr(finding, "description"):
224
+ finding.description = f"[Verified: {notes}] {finding.description}"
225
+
226
+ if issues and hasattr(finding, "description"):
227
+ finding.description = f"[⚠ Issues: {'; '.join(issues[:3])}] {finding.description}"
228
+
229
+ return finding, False
230
+
231
+
232
+ # ── Factory ───────────────────────────────────────────────────
233
+
234
+ def create_verifier(provider: str | None = None) -> FindingVerifier | None:
235
+ """Create a verifier using the configured LLM provider.
236
+
237
+ Returns None if no LLM is available.
238
+ """
239
+ try:
240
+ try:
241
+ from watson.orchestration.llm_config import call_llm
242
+ except ImportError:
243
+ from src.watson.orchestration.llm_config import call_llm
244
+
245
+ async def _call(prompt, timeout=120, max_tokens=2000, system=""):
246
+ return await call_llm(prompt, timeout=timeout, max_tokens=max_tokens, system=system)
247
+
248
+ return FindingVerifier(call_llm=_call)
249
+ except ImportError:
250
+ logger.warning("verify: could not import llm_config — verification disabled")
251
+ return None
watson/web/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """Watson Web — FastAPI product shell."""