osintengine 1.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. osintengine-1.0.2.dist-info/METADATA +311 -0
  2. osintengine-1.0.2.dist-info/RECORD +103 -0
  3. osintengine-1.0.2.dist-info/WHEEL +5 -0
  4. osintengine-1.0.2.dist-info/entry_points.txt +2 -0
  5. osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
  6. osintengine-1.0.2.dist-info/top_level.txt +2 -0
  7. src/watson/__init__.py +10 -0
  8. src/watson/agent/__init__.py +96 -0
  9. src/watson/agents/__init__.py +101 -0
  10. src/watson/agents/protocol.py +196 -0
  11. src/watson/auth/__init__.py +68 -0
  12. src/watson/auth/store.py +145 -0
  13. src/watson/conversation.py +68 -0
  14. src/watson/core/__init__.py +0 -0
  15. src/watson/core/models.py +96 -0
  16. src/watson/ethics.py +205 -0
  17. src/watson/exports.py +182 -0
  18. src/watson/graph/__init__.py +69 -0
  19. src/watson/graph/entities.py +207 -0
  20. src/watson/graph/graph.py +489 -0
  21. src/watson/graph/osint_framework.py +266 -0
  22. src/watson/graph/relationships.py +116 -0
  23. src/watson/graph/transforms.py +787 -0
  24. src/watson/infra/__init__.py +43 -0
  25. src/watson/infra/cache.py +171 -0
  26. src/watson/infra/ratelimit.py +125 -0
  27. src/watson/infra/resilience.py +239 -0
  28. src/watson/infra/retry.py +186 -0
  29. src/watson/metrics.py +128 -0
  30. src/watson/opsec/__init__.py +290 -0
  31. src/watson/orchestration/__init__.py +23 -0
  32. src/watson/orchestration/engine.py +5587 -0
  33. src/watson/orchestration/executor.py +96 -0
  34. src/watson/orchestration/intent.py +139 -0
  35. src/watson/orchestration/intent_classifier.py +45 -0
  36. src/watson/orchestration/llm_config.py +160 -0
  37. src/watson/orchestration/resolution.py +555 -0
  38. src/watson/orchestration/scraper.py +94 -0
  39. src/watson/orchestration/synthesis.py +726 -0
  40. src/watson/orchestration/target_profile.py +748 -0
  41. src/watson/persistence/__init__.py +18 -0
  42. src/watson/persistence/models.py +161 -0
  43. src/watson/persistence/store.py +230 -0
  44. src/watson/pipeline/__init__.py +16 -0
  45. src/watson/pipeline/pre_synthesis.py +621 -0
  46. src/watson/search.py +103 -0
  47. src/watson/serializers/stix.py +451 -0
  48. src/watson/tools/__init__.py +31 -0
  49. src/watson/tools/base.py +97 -0
  50. src/watson/tools/blockchain.py +359 -0
  51. src/watson/tools/captcha.py +276 -0
  52. src/watson/tools/conflict.py +107 -0
  53. src/watson/tools/corporate.py +287 -0
  54. src/watson/tools/darkweb.py +144 -0
  55. src/watson/tools/geolocation.py +347 -0
  56. src/watson/tools/image_video.py +108 -0
  57. src/watson/tools/marinetraffic.py +142 -0
  58. src/watson/tools/people.py +476 -0
  59. src/watson/tools/registry.py +56 -0
  60. src/watson/tools/satellite.py +118 -0
  61. src/watson/tools/scraper.py +745 -0
  62. src/watson/tools/shodan.py +129 -0
  63. src/watson/tools/social_media.py +160 -0
  64. src/watson/tools/websites.py +291 -0
  65. src/watson/tools/wikidata.py +398 -0
  66. src/watson/utils/__init__.py +1 -0
  67. src/watson/utils/helpers.py +49 -0
  68. src/watson/utils/http.py +119 -0
  69. src/watson/verification/__init__.py +245 -0
  70. watson/__init__.py +6 -0
  71. watson/agents/__init__.py +20 -0
  72. watson/agents/base.py +103 -0
  73. watson/agents/direct.py +225 -0
  74. watson/agents/hermes.py +275 -0
  75. watson/agents/hermes_mcp.py +292 -0
  76. watson/api_keys.py +176 -0
  77. watson/browser_scraper.py +481 -0
  78. watson/cli.py +836 -0
  79. watson/crossref.py +175 -0
  80. watson/ethics.py +20 -0
  81. watson/graph.py +406 -0
  82. watson/mcp_server.py +601 -0
  83. watson/memory.py +552 -0
  84. watson/neo4j_graph.py +124 -0
  85. watson/opsec/__init__.py +307 -0
  86. watson/reporter.py +537 -0
  87. watson/scheduler.py +486 -0
  88. watson/serializers/stix.py +451 -0
  89. watson/terminal.py +410 -0
  90. watson/toolkit.py +252 -0
  91. watson/toolkit_api.py +347 -0
  92. watson/toolkit_automation.py +95 -0
  93. watson/toolkit_registry.py +345 -0
  94. watson/verification/__init__.py +251 -0
  95. watson/web/__init__.py +1 -0
  96. watson/web/app.py +1650 -0
  97. watson/web/middleware.py +301 -0
  98. watson/web/static/assets/index-B7hPOc0z.js +278 -0
  99. watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
  100. watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
  101. watson/web/static/index.html +14 -0
  102. watson/web/templates/chat.html +2 -0
  103. watson/web/templates/investigation-map.html +429 -0
watson/cli.py ADDED
@@ -0,0 +1,836 @@
1
+ """
2
+ Watson CLI — terminal interface for the OSINT investigation engine.
3
+
4
+ Commands:
5
+ watson setup First-time setup wizard
6
+ watson onboard Same as setup
7
+ watson doctor System health check
8
+ watson tools List available APIs
9
+ watson config Show/edit configuration
10
+ watson web Start the web interface
11
+ watson chat Open browser to the web interface
12
+ watson investigate Run a single investigation
13
+ watson graph Show knowledge graph stats
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import argparse
19
+ import asyncio
20
+ import importlib
21
+ import json
22
+ import os
23
+ import random
24
+ import shutil
25
+ import subprocess
26
+ import sys
27
+ from pathlib import Path
28
+
29
+ # Ensure src/ is on path so `from src.watson...` imports work
30
+ # without requiring PYTHONPATH=.:src in the shell.
31
+ _src = Path(__file__).resolve().parent.parent / "src"
32
+ if str(_src) not in sys.path:
33
+ sys.path.insert(0, str(_src))
34
+
35
+ # ANSI colors
36
+ W = "\033[38;5;208m" # amber
37
+ B = "\033[1m"
38
+ D = "\033[2m"
39
+ G = "\033[32m"
40
+ R = "\033[31m"
41
+ Y = "\033[33m"
42
+ C = "\033[36m"
43
+ M = "\033[35m" # magenta — for graph/MCP
44
+ X = "\033[0m"
45
+
46
+ VERSION = "1.0.0"
47
+ CODENAME = "A Study in Scarlet"
48
+
49
+ WATSON_BANNER = f"""
50
+ {W}╔══════════════════════════════════════════════════════════════╗
51
+ ║ ║
52
+ ║ {B}██╗ ██╗ █████╗ ████████╗███████╗ ██████╗ ███╗ ██╗{W} ║
53
+ ║ {B}██║ ██║██╔══██╗╚══██╔══╝██╔════╝██╔═══██╗████╗ ██║{W} ║
54
+ ║ {B}██║ █╗ ██║███████║ ██║ ███████╗██║ ██║██╔██╗ ██║{W} ║
55
+ ║ {B}██║███╗██║██╔══██║ ██║ ╚════██║██║ ██║██║╚██╗██║{W} ║
56
+ ║ {B}╚███╔███╔╝██║ ██║ ██║ ███████║╚██████╔╝██║ ╚████║{W} ║
57
+ ║ {B} ╚══╝╚══╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝ ╚═════╝ ╚═╝ ╚═══╝{W} ║
58
+ ║ ║
59
+ ║ {D}{VERSION} · {CODENAME}{W} ║
60
+ ║ {D}Multi-source OSINT. Graph-native. Community-powered.{W} ║
61
+ ║ ║
62
+ ╚══════════════════════════════════════════════════════════════╝{X}
63
+ """
64
+
65
+ SHERLOCK_QUOTES = [
66
+ '"It is a capital mistake to theorize before one has data."',
67
+ '"The world is full of obvious things which nobody by any chance ever observes."',
68
+ '"There is nothing more deceptive than an obvious fact."',
69
+ '"You see, but you do not observe."',
70
+ '"Data! Data! Data! I can\'t make bricks without clay."',
71
+ '"When you have eliminated the impossible, whatever remains, however improbable, must be the truth."',
72
+ '"The little things are infinitely the most important."',
73
+ '"What one man can invent, another can discover."',
74
+ '"There is nothing more stimulating than a case where everything goes against you."',
75
+ '"I am not the law, but I represent justice so far as my feeble powers go."',
76
+ ]
77
+
78
+
79
+ def _quote() -> str:
80
+ return f"{D}{random.choice(SHERLOCK_QUOTES)}{X}"
81
+
82
+
83
+ def _config_dir() -> Path:
84
+ return Path.home() / ".watson"
85
+
86
+
87
+ def _config_path() -> Path:
88
+ return _config_dir() / "config.json"
89
+
90
+
91
+ def _load_config() -> dict:
92
+ if _config_path().exists():
93
+ return json.loads(_config_path().read_text())
94
+ return {}
95
+
96
+
97
+ def _save_config(config: dict):
98
+ _config_dir().mkdir(exist_ok=True)
99
+ _config_path().write_text(json.dumps(config, indent=2))
100
+
101
+
102
+ # ═══════════════════════════════════════════════════════════════════
103
+ # ONBOARDING
104
+ # ═══════════════════════════════════════════════════════════════════
105
+
106
+
107
+ def _check_agent_available(name: str) -> bool:
108
+ """Check if an agent backend is available on the system."""
109
+ if name == "direct":
110
+ return True # Built-in, no external binary needed
111
+ # MCP adapter uses the same binary as CLI adapter
112
+ if name == "hermes-mcp":
113
+ return shutil.which("hermes") is not None
114
+ return shutil.which(name) is not None
115
+
116
+
117
+ def _discover_agents() -> list[dict]:
118
+ """Auto-discover available agent adapters from watson/agents/."""
119
+ agents_dir = Path(__file__).resolve().parent / "agents"
120
+ discovered: list[dict] = []
121
+
122
+ # Always include Direct as built-in
123
+ discovered.append({
124
+ "name": "direct",
125
+ "description": "Any OpenAI-compatible API (OpenAI, Anthropic, DeepSeek, Groq)",
126
+ "available": True,
127
+ })
128
+
129
+ for f in sorted(agents_dir.glob("*.py")):
130
+ if f.name.startswith("_") or f.name == "base.py" or f.name == "direct.py":
131
+ continue
132
+ try:
133
+ spec = importlib.util.spec_from_file_location(
134
+ f"watson.agents.{f.stem}", str(f)
135
+ )
136
+ if spec and spec.loader:
137
+ mod = importlib.util.module_from_spec(spec)
138
+ spec.loader.exec_module(mod)
139
+ for attr_name in dir(mod):
140
+ attr = getattr(mod, attr_name)
141
+ if (isinstance(attr, type) and
142
+ hasattr(attr, "name") and
143
+ attr_name.endswith("Adapter") and
144
+ attr_name != "AgentAdapter"):
145
+ agent_name = getattr(attr, "name", "")
146
+ if not agent_name or agent_name == "base":
147
+ continue
148
+ discovered.append({
149
+ "name": agent_name,
150
+ "description": getattr(attr, "description", f"{agent_name.title()} agent"),
151
+ "available": _check_agent_available(agent_name),
152
+ })
153
+ break
154
+ except Exception:
155
+ pass
156
+
157
+ return discovered
158
+
159
+
160
+ def cmd_onboard():
161
+ """First-time setup wizard — v1 'A Study in Scarlet'."""
162
+ print(WATSON_BANNER)
163
+ print(f" {_quote()}")
164
+ print(f"\n{G}Welcome to Watson {VERSION} — {CODENAME}.{X}")
165
+ print(f"{D}Everything you need ships free. Premium features scale when you do.{X}\n")
166
+
167
+ existing = _load_config()
168
+ config = {
169
+ "agent": existing.get("agent", "direct"),
170
+ "api_key": existing.get("api_key", ""),
171
+ "api_base": existing.get("api_base", ""),
172
+ "model": existing.get("model", ""),
173
+ "cases_dir": existing.get("cases_dir", os.path.expanduser("~/watson-cases")),
174
+ "mcp_url": existing.get("mcp_url", "http://localhost:8700"),
175
+ "mcp_api_key": existing.get("mcp_api_key", ""),
176
+ "publish_to_mcp": existing.get("publish_to_mcp", False),
177
+ "paid_api_keys": existing.get("paid_api_keys", {}),
178
+ }
179
+
180
+ # ── Step 1: Engine ──
181
+ print(f"{Y}═══ Step 1/4: Choose Your Engine{X}\n")
182
+
183
+ agents = _discover_agents()
184
+
185
+ for i, agent in enumerate(agents):
186
+ idx = i + 1
187
+ status = f"{G}available{X}" if agent["available"] else f"{Y}not detected{X}"
188
+ print(f" [{idx}] {B}{agent['name'].title()}{X} — {agent['description']}")
189
+ print(f" {D}Status: {status}{X}")
190
+ if not agent["available"]:
191
+ print(f" {Y}⚠ Install {agent['name']} first, or choose a different engine.{X}")
192
+ print()
193
+
194
+ # Default: match existing config, or first available agent
195
+ default_idx = 1
196
+ for i, agent in enumerate(agents):
197
+ if agent["available"] and agent["name"] == existing.get("agent", "direct"):
198
+ default_idx = i + 1
199
+ break
200
+ else:
201
+ for i, agent in enumerate(agents):
202
+ if agent["available"]:
203
+ default_idx = i + 1
204
+ break
205
+
206
+ choice = input(f" {G}Choice [{default_idx}]:{X} ").strip() or str(default_idx)
207
+ try:
208
+ chosen_idx = int(choice) - 1
209
+ if 0 <= chosen_idx < len(agents):
210
+ chosen = agents[chosen_idx]
211
+ if not chosen["available"]:
212
+ print(f"\n {Y}⚠ {chosen['name'].title()} is not installed. Using Direct.{X}\n")
213
+ config["agent"] = "direct"
214
+ else:
215
+ config["agent"] = chosen["name"]
216
+ else:
217
+ config["agent"] = agents[0]["name"]
218
+ except ValueError:
219
+ config["agent"] = agents[0]["name"]
220
+
221
+ if config["agent"] == "direct":
222
+ print()
223
+ api_key = os.environ.get("WATSON_API_KEY", config.get("api_key", ""))
224
+ masked = f"{api_key[:8]}..." if len(api_key) > 8 else "(not set)"
225
+ print(f" {D}API key (env WATSON_API_KEY): {masked}{X}")
226
+ key = input(f" {Y}Enter or paste new key [skip to keep current]:{X} ").strip()
227
+ if key:
228
+ config["api_key"] = key
229
+
230
+ base = input(f" {Y}API base URL [https://api.openai.com/v1]:{X} ").strip()
231
+ if base:
232
+ config["api_base"] = base
233
+ elif config.get("api_base"):
234
+ pass # keep existing
235
+ else:
236
+ config["api_base"] = "https://api.openai.com/v1"
237
+
238
+ model = input(f" {Y}Model [gpt-4o]:{X} ").strip() or config.get("model") or "gpt-4o"
239
+ config["model"] = model
240
+
241
+ # Also save to the web UI key store so the server picks it up
242
+ try:
243
+ from watson.api_keys import set_key
244
+ provider_slug = "deepseek" if "deepseek" in config.get("api_base", "") else "openai"
245
+ if config.get("api_key"):
246
+ set_key(provider_slug, config["api_key"])
247
+ print(f" {D}→ Also saved to web UI key store as '{provider_slug}'{X}")
248
+ except Exception:
249
+ pass
250
+
251
+ # ── Step 2: OSINT API keys (optional) ──
252
+ print(f"\n{Y}═══ Step 2/4: OSINT API Keys {D}(optional — free tier works without any){X}\n")
253
+ print(f" {D}Paid APIs unlock deeper investigations. All are optional.{X}")
254
+ print(f" {D}Without any keys, Watson still uses 10+ free sources.{X}\n")
255
+
256
+ paid_keys = config.get("paid_api_keys", {})
257
+ apis = [
258
+ ("OpenSanctions", "opensanctions", "Sanctions, entities, corporate registry"),
259
+ ("VirusTotal", "virustotal", "Domain/IP reputation, malware analysis"),
260
+ ("OpenCorporates", "opencorporates", "Company registries worldwide"),
261
+ ("HIBP", "hibp", "Have I Been Pwned — breach data"),
262
+ ]
263
+ for name, slug, desc in apis:
264
+ current = paid_keys.get(slug, "")
265
+ status = f"{G}set{X}" if current else f"{D}not set{X}"
266
+ print(f" [{name}] {desc} — {status}")
267
+ key = input(f" {Y} Key [skip]:{X} ").strip()
268
+ if key:
269
+ paid_keys[slug] = key
270
+
271
+ config["paid_api_keys"] = paid_keys
272
+
273
+ # ── Step 3: Community Graph (MCP) ──
274
+ print(f"\n{Y}═══ Step 3/4: Community Graph{X}\n")
275
+ print(f" {B}The Watson Knowledge Graph{X} connects investigations across users.")
276
+ print(f" {D}When you publish a case, entities are shared so others can discover")
277
+ print(f" {D}connections from your work — and you benefit from theirs.{X}\n")
278
+ print(f" {D}By default, your graph is local to this machine. Nothing is shared")
279
+ print(f" {D}unless you opt in per-investigation.{X}\n")
280
+
281
+ mcp_url = input(f" {Y}MCP server URL [{config['mcp_url']}]:{X} ").strip()
282
+ if mcp_url:
283
+ config["mcp_url"] = mcp_url
284
+
285
+ if "localhost" in config["mcp_url"] or "127.0.0.1" in config["mcp_url"]:
286
+ print(f" {D}→ Local graph — your data stays on this machine.{X}")
287
+ else:
288
+ print(f" {D}→ Remote graph at {config['mcp_url']} — cases publish to shared instance.{X}")
289
+ mcp_key = input(f" {Y}MCP API key (for publishing):{X} ").strip()
290
+ if mcp_key:
291
+ config["mcp_api_key"] = mcp_key
292
+
293
+ # ── Step 4: Case storage ──
294
+ print(f"\n{Y}═══ Step 4/4: Case Storage{X}\n")
295
+ cases = input(f" {Y}Case files directory [{config['cases_dir']}]:{X} ").strip()
296
+ if cases:
297
+ config["cases_dir"] = os.path.expanduser(cases)
298
+ Path(config["cases_dir"]).mkdir(parents=True, exist_ok=True)
299
+
300
+ # ── Save ──
301
+ _save_config(config)
302
+
303
+ print(f"\n{G}═══ Configuration Complete ═══{X}\n")
304
+ print(f" Engine: {B}{config['agent']}{X}")
305
+ print(f" Model: {D}{config.get('model','(hermes)')}{X}")
306
+ print(f" Cases: {D}{config['cases_dir']}{X}")
307
+ print(f" Graph: {D}{config['mcp_url']}{X}")
308
+ paid_count = sum(1 for v in config.get("paid_api_keys", {}).values() if v)
309
+ print(f" Paid APIs: {G if paid_count else D}{paid_count}/4 configured{X}")
310
+ print(f"\n {_quote()}")
311
+ print(f"\n {G}Watson is ready.{X}")
312
+
313
+ # Offer to launch interface immediately
314
+ launch = input(f"\n {Y}Launch the web interface now? [Y/n]:{X} ").strip().lower()
315
+ if launch in ("", "y", "yes"):
316
+ print(f"\n {G}Starting Watson...{X}\n")
317
+ _launch_web("127.0.0.1", 8777)
318
+ else:
319
+ py = sys.executable
320
+ print(f"\n {D}Start anytime with:{X}")
321
+ print(f" {B}{py} -m watson.cli web{X}")
322
+ print(f"\n {D}Other commands:{X}")
323
+ print(f" {B}{py} -m watson.cli doctor{D} — system health check{X}")
324
+ print(f" {B}{py} -m watson.cli tools{D} — list available APIs{X}")
325
+ print(f" {B}{py} -m watson.cli chat{D} — open the web interface{X}\n")
326
+
327
+
328
+ # ═══════════════════════════════════════════════════════════════════
329
+ # CONFIG
330
+ # ═══════════════════════════════════════════════════════════════════
331
+
332
+
333
+ def cmd_config(args):
334
+ """Show or update configuration."""
335
+ if not _config_path().exists():
336
+ print(f"{R}No config found. Run '{B}watson onboard{R}' first.{X}")
337
+ sys.exit(1)
338
+
339
+ config = _load_config()
340
+
341
+ if args.key and args.value:
342
+ # Nested keys: paid_api_keys.opensanctions
343
+ keys = args.key.split(".")
344
+ target = config
345
+ for k in keys[:-1]:
346
+ target = target.setdefault(k, {})
347
+ target[keys[-1]] = args.value
348
+ _save_config(config)
349
+ print(f"{G}✓ {args.key} = {args.value}{X}")
350
+ return
351
+
352
+ if args.key:
353
+ keys = args.key.split(".")
354
+ target = config
355
+ for k in keys:
356
+ target = target.get(k, {}) if isinstance(target, dict) else target
357
+ val = target if target != {} else f"{R}(not set){X}"
358
+ print(f"{args.key}: {val}")
359
+ return
360
+
361
+ # Show all
362
+ print(f"{B}Watson {VERSION} Configuration{X}\n")
363
+ for k, v in config.items():
364
+ if k in ("api_key", "mcp_api_key"):
365
+ masked = v[:8] + "..." if v else "(not set)"
366
+ print(f" {C}{k}{X}: {masked}")
367
+ elif k == "paid_api_keys":
368
+ print(f" {C}{k}{X}:")
369
+ for slug, key in v.items():
370
+ status = f"{G}●●●●{X}" if key else f"{D}(not set){X}"
371
+ print(f" {slug}: {status}")
372
+ else:
373
+ print(f" {C}{k}{X}: {v}")
374
+ print(f"\n {D}Config file: {_config_path()}{X}")
375
+
376
+
377
+ def _launch_web(host: str = "127.0.0.1", port: int = 8777) -> None:
378
+ """Start the web server in background and open browser when ready."""
379
+ import webbrowser
380
+ import time
381
+
382
+ project_root = Path(__file__).resolve().parent.parent
383
+ config = _load_config()
384
+ mcp_url = config.get("mcp_url", "http://localhost:8700")
385
+
386
+ env = os.environ.copy()
387
+ env["PYTHONPATH"] = f"src:{project_root}:{env.get('PYTHONPATH', '')}"
388
+ env["WATSON_MCP_URL"] = mcp_url
389
+ if config.get("mcp_api_key"):
390
+ env["MCP_API_KEY"] = config["mcp_api_key"]
391
+
392
+ print(f" {D}Starting server on http://{host}:{port} ...{X}")
393
+
394
+ proc = subprocess.Popen(
395
+ [sys.executable, "-m", "uvicorn", "watson.web.app:app",
396
+ "--host", host, "--port", str(port)],
397
+ cwd=str(project_root),
398
+ env=env,
399
+ stdout=subprocess.DEVNULL,
400
+ stderr=subprocess.DEVNULL,
401
+ )
402
+
403
+ # Wait for server to be ready
404
+ import socket
405
+ for _ in range(15):
406
+ time.sleep(0.5)
407
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
408
+ sock.settimeout(1)
409
+ if sock.connect_ex((host, port)) == 0:
410
+ sock.close()
411
+ break
412
+ sock.close()
413
+ else:
414
+ print(f" {Y}Server may still be starting — open http://{host}:{port}{X}")
415
+ return
416
+
417
+ print(f" {G}Server ready. Opening browser...{X}")
418
+ time.sleep(0.5)
419
+ webbrowser.open(f"http://{host}:{port}")
420
+
421
+ # Keep running until Ctrl+C
422
+ try:
423
+ print(f" {D}Press Ctrl+C to stop.{X}")
424
+ proc.wait()
425
+ except KeyboardInterrupt:
426
+ print(f"\n {Y}Shutting down...{X}")
427
+ proc.terminate()
428
+ proc.wait()
429
+
430
+
431
+ # ═══════════════════════════════════════════════════════════════════
432
+ # WEB
433
+ # ═══════════════════════════════════════════════════════════════════
434
+
435
+
436
+ def cmd_web(args):
437
+ """Start the web UI."""
438
+ if not _config_path().exists():
439
+ print(f"{Y}No config found — running onboarding first...{X}\n")
440
+ cmd_onboard()
441
+
442
+ host = args.host or "0.0.0.0"
443
+ port = args.port or 8777
444
+
445
+ print(WATSON_BANNER)
446
+ print(f" {_quote()}")
447
+ print(f"\n {G}Starting Watson web interface...{X}")
448
+
449
+ _launch_web("127.0.0.1", port)
450
+
451
+
452
+ # ═══════════════════════════════════════════════════════════════════
453
+ # GRAPH — show knowledge graph status
454
+ # ═══════════════════════════════════════════════════════════════════
455
+
456
+
457
+ def cmd_graph(args):
458
+ """Show knowledge graph stats and recent entities."""
459
+ config = _load_config()
460
+ mcp_url = config.get("mcp_url", "http://localhost:8700")
461
+
462
+ print(f"{B}Watson Knowledge Graph{X}\n")
463
+
464
+ try:
465
+ import httpx
466
+ resp = httpx.get(f"{mcp_url}/api/stats", timeout=5)
467
+ if resp.status_code == 200:
468
+ stats = resp.json()
469
+ print(f" {G}Server:{X} {mcp_url}")
470
+ print(f" {G}Entities:{X} {stats.get('entity_count', 0)}")
471
+ print(f" {G}Relations:{X} {stats.get('relation_count', 0)}")
472
+ print(f" {G}Cases:{X} {stats.get('case_count', 0)}")
473
+
474
+ types = stats.get("entity_types", {})
475
+ if types:
476
+ print(f"\n {B}Entity types:{X}")
477
+ for t, count in sorted(types.items(), key=lambda x: -x[1]):
478
+ print(f" {t}: {count}")
479
+
480
+ top = stats.get("top_entities", [])
481
+ if top:
482
+ print(f"\n {B}Most-connected entities:{X}")
483
+ for e in top[:5]:
484
+ print(f" {e['value'][:60]} ({e['type']}) — {e['case_count']} cases")
485
+ else:
486
+ print(f" {Y}Graph server returned {resp.status_code}{X}")
487
+ except Exception as e:
488
+ print(f" {R}Graph not reachable at {mcp_url}{X}")
489
+ print(f" {D}Start Watson with '{B}watson web{D}' to auto-start a local graph.{X}")
490
+ if "localhost" in mcp_url:
491
+ print(f" {D} or run: {B}uvicorn watson.mcp_server:mcp --port 8700{X}")
492
+
493
+ print(f"\n {D}MCP tools: watson_search, watson_traverse, watson_context, watson_case, watson_stats{X}")
494
+ print(f" {D}Self-hosting: see SELF_HOSTING.md{X}")
495
+
496
+
497
+ # ═══════════════════════════════════════════════════════════════════
498
+ # INVESTIGATE — CLI mode (for headless/scripted runs)
499
+ # ═══════════════════════════════════════════════════════════════════
500
+
501
+
502
+ def cmd_investigate(args):
503
+ """Run a single investigation from the terminal."""
504
+ if not args.query:
505
+ print(f"{R}Usage: watson investigate <target>{X}")
506
+ print(f" Example: watson investigate \"Elon Musk\"")
507
+ print(f" Target types: person, company, domain, email, IP, wallet")
508
+ sys.exit(1)
509
+
510
+ config = _load_config()
511
+ if not config:
512
+ print(f"{Y}No config found — running onboarding first...{X}\n")
513
+ cmd_onboard()
514
+ config = _load_config()
515
+
516
+ async def _run():
517
+ from src.watson.orchestration import get_engine
518
+
519
+ engine = get_engine()
520
+ query = args.query
521
+
522
+ print(WATSON_BANNER)
523
+ print(f" {_quote()}")
524
+ print(f"\n{C}🔍 Investigating: {query}{X}\n")
525
+
526
+ def on_event(event_type, data):
527
+ if event_type == "progress":
528
+ msg = data.get("message", "")
529
+ print(f" {D}{msg}{X}")
530
+ elif event_type == "finding":
531
+ title = data.get("title", "")[:120]
532
+ confidence = data.get("confidence", 0)
533
+ bar = "🟢" if confidence > 0.7 else "🟡" if confidence > 0.4 else "🔴"
534
+ print(f" {bar} {title}")
535
+
536
+ result = await engine.investigate(
537
+ query=query,
538
+ focus="",
539
+ on_event=on_event,
540
+ mode="deep_investigation",
541
+ save_mode="auto",
542
+ )
543
+
544
+ print(f"\n{G}═══ Investigation Complete ═══{X}")
545
+ print(f" Case: {result['case_id']}")
546
+ print(f" Findings: {result['findings_count']} ({result['confirmed_count']} confirmed)")
547
+ print(f" Verifiability: {result['verifiability_score']:.0%}")
548
+ print(f" Saved to: {config.get('cases_dir', '~/watson-cases')}")
549
+ print()
550
+
551
+ asyncio.run(_run())
552
+
553
+
554
+ # ═══════════════════════════════════════════════════════════════════
555
+ # SETUP
556
+ # ═══════════════════════════════════════════════════════════════════
557
+
558
+
559
+ def cmd_setup(args):
560
+ """First-time setup wizard — alias for onboard."""
561
+ cmd_onboard()
562
+
563
+
564
+ # ═══════════════════════════════════════════════════════════════════
565
+ # CHAT
566
+ # ═══════════════════════════════════════════════════════════════════
567
+
568
+
569
+ def cmd_chat(args):
570
+ """Open the web interface — starts server if needed, opens browser."""
571
+ import webbrowser
572
+
573
+ print(WATSON_BANNER)
574
+ print(f" {_quote()}\n")
575
+
576
+ host = args.host or "127.0.0.1"
577
+ port = args.port or 8777
578
+
579
+ # Check if server already running
580
+ import socket
581
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
582
+ sock.settimeout(1)
583
+ server_running = sock.connect_ex(("127.0.0.1", port)) == 0
584
+ sock.close()
585
+
586
+ if server_running:
587
+ url = f"http://{host}:{port}"
588
+ print(f" {G}Watson is already running at {C}{url}{X}")
589
+ print(f" Opening in your browser now...\n")
590
+ webbrowser.open(url)
591
+ else:
592
+ print(f" {Y}No server detected on port {port}. Starting one now...{X}\n")
593
+ # Start in background and open browser
594
+ subprocess.Popen(
595
+ [sys.executable, "-m", "watson.cli", "web", "--host", host, "--port", str(port)],
596
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
597
+ )
598
+ import time
599
+ time.sleep(2)
600
+ url = f"http://{host}:{port}"
601
+ print(f" {G}Opening {C}{url}{X} in your browser...\n")
602
+ webbrowser.open(url)
603
+
604
+
605
+ # ═══════════════════════════════════════════════════════════════════
606
+ # DOCTOR
607
+ # ═══════════════════════════════════════════════════════════════════
608
+
609
+
610
+ def cmd_doctor(args):
611
+ """System health check — deps, config, APIs, graph, frontend."""
612
+ print(WATSON_BANNER)
613
+ print(f" {_quote()}")
614
+ print(f"\n{G}═══ Watson System Health Check ═══{X}\n")
615
+
616
+ all_ok = True
617
+
618
+ # ── Python ──
619
+ py_ver = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
620
+ py_ok = sys.version_info >= (3, 10)
621
+ icon = f"{G}✓{X}" if py_ok else f"{R}✗{X}"
622
+ print(f" {icon} Python {py_ver}")
623
+ if not py_ok:
624
+ all_ok = False
625
+
626
+ # ── Dependencies ──
627
+ deps = ["fastapi", "uvicorn", "aiohttp", "httpx", "pydantic", "PIL", "jinja2"]
628
+ for dep in deps:
629
+ try:
630
+ __import__(dep if dep != "PIL" else "PIL")
631
+ print(f" {G}✓{X} {dep}")
632
+ except ImportError:
633
+ print(f" {R}✗{X} {dep} {D}(run: pip install -r requirements.txt){X}")
634
+ all_ok = False
635
+
636
+ # ── Config ──
637
+ config = _load_config()
638
+ if config:
639
+ paid = sum(1 for v in config.get("paid_api_keys", {}).values() if v)
640
+ print(f" {G}✓{X} Config found — engine: {config.get('agent','?')}, {paid}/4 paid APIs")
641
+ else:
642
+ print(f" {Y}○{X} No config — run {B}{sys.executable} -m watson.cli onboard{X}")
643
+ all_ok = False
644
+
645
+ # ── API connectivity ──
646
+ if config:
647
+ paid_keys = config.get("paid_api_keys", {})
648
+ apis = {
649
+ "OpenSanctions": paid_keys.get("opensanctions"),
650
+ "VirusTotal": paid_keys.get("virustotal"),
651
+ "OpenCorporates": paid_keys.get("opencorporates"),
652
+ "HIBP": paid_keys.get("hibp"),
653
+ }
654
+ for name, key in apis.items():
655
+ if key:
656
+ print(f" {G}✓{X} {name} key configured")
657
+ else:
658
+ print(f" {D}○{X} {name} {D}(no key — free tier only){X}")
659
+
660
+ # ── Graph server ──
661
+ mcp_url = config.get("mcp_url", "http://localhost:8700") if config else "http://localhost:8700"
662
+ try:
663
+ import httpx
664
+ r = httpx.get(f"{mcp_url}/api/stats", timeout=3)
665
+ if r.status_code == 200:
666
+ stats = r.json()
667
+ print(f" {G}✓{X} Graph server — {stats.get('entity_count', 0)} entities, {stats.get('case_count', 0)} cases")
668
+ else:
669
+ print(f" {Y}○{X} Graph server returned {r.status_code}")
670
+ except Exception:
671
+ print(f" {Y}○{X} Graph server not reachable at {mcp_url}")
672
+ print(f" {D}Start with: {sys.executable} -m watson.cli web{D} (auto-starts local graph){X}")
673
+
674
+ # ── Frontend ──
675
+ static_dir = Path(__file__).resolve().parent / "web" / "static"
676
+ index_html = static_dir / "index.html"
677
+ assets_dir = static_dir / "assets"
678
+ if index_html.exists() and assets_dir.exists() and list(assets_dir.glob("*.js")):
679
+ print(f" {G}✓{X} Frontend built ({len(list(assets_dir.glob('*')))} files)")
680
+ else:
681
+ print(f" {Y}○{X} Frontend not built — {D}run: cd frontend && npm run build{X}")
682
+ all_ok = False
683
+
684
+ # ── Bellingcat toolkit ──
685
+ csv_path = Path(__file__).resolve().parent.parent / "data" / "bellingcat_toolkit.csv"
686
+ if csv_path.exists():
687
+ print(f" {G}✓{X} Bellingcat toolkit loaded")
688
+ else:
689
+ print(f" {D}○{X} Bellingcat CSV not found (toolkit search disabled){X}")
690
+
691
+ # ── Tools count ──
692
+ try:
693
+ from .toolkit import DIRECT_APIS
694
+ free_count = sum(1 for v in DIRECT_APIS.values() if not v.get("requires_key"))
695
+ paid_count = sum(1 for v in DIRECT_APIS.values() if v.get("requires_key"))
696
+ print(f" {G}✓{X} {len(DIRECT_APIS)} direct APIs ({free_count} free, {paid_count} paid)")
697
+ except Exception:
698
+ pass
699
+
700
+ print()
701
+ if all_ok:
702
+ print(f" {G}All checks passed. Watson is ready.{X}")
703
+ print(f" {D}Run {B}{sys.executable} -m watson.cli web{D} to start.{X}")
704
+ else:
705
+ print(f" {Y}Some checks need attention. Run {B}{sys.executable} -m watson.cli onboard{D} to fix.{X}")
706
+ print()
707
+
708
+
709
+ # ═══════════════════════════════════════════════════════════════════
710
+ # TOOLS
711
+ # ═══════════════════════════════════════════════════════════════════
712
+
713
+
714
+ def cmd_tools(args):
715
+ """List available OSINT APIs and tools with configuration status."""
716
+ config = _load_config()
717
+ paid_keys = config.get("paid_api_keys", {}) if config else {}
718
+
719
+ print(WATSON_BANNER)
720
+ print(f" {_quote()}")
721
+ print(f"\n{G}═══ Available OSINT APIs ═══{X}\n")
722
+
723
+ try:
724
+ from .toolkit import DIRECT_APIS
725
+ print(f" {B}── Direct API Integrations ({len(DIRECT_APIS)}) ──{X}\n")
726
+ for name, cfg in sorted(DIRECT_APIS.items()):
727
+ needs_key = cfg.get("requires_key", False)
728
+ if needs_key:
729
+ slug = name.lower().replace(" ", "").replace(".", "")
730
+ # map to config slugs
731
+ key_map = {
732
+ "opensanctions": "opensanctions",
733
+ "opencorporates": "opencorporates",
734
+ "virustotal": "virustotal",
735
+ "hibp": "hibp",
736
+ }
737
+ configured = bool(paid_keys.get(key_map.get(slug, slug)))
738
+ status = f"{G}key set{X}" if configured else f"{Y}key needed{X}"
739
+ cost = "Paid"
740
+ else:
741
+ status = f"{G}free — no key needed{X}"
742
+ cost = "Free"
743
+ print(f" {B}{name}{X}")
744
+ print(f" {D}Cost: {cost} | Status: {status}{X}")
745
+ except Exception as e:
746
+ print(f" {R}Could not load API registry: {e}{X}\n")
747
+
748
+ # Bellingcat toolkit stats
749
+ try:
750
+ from .toolkit_registry import registry
751
+ summary = registry.summary()
752
+ print(f"\n {B}── Bellingcat OSINT Toolkit ──{X}")
753
+ print(f" {D}{summary['total_tools']} tools across {len(summary['categories'])} categories{X}")
754
+ print(f" {D}{summary['free_tools']} free · {summary['paid_tools']} paid · {summary['partial_tools']} partially free{X}")
755
+ print(f" {D}{summary['url_templates']} tools have URL templates for direct querying{X}")
756
+ print(f"\n {D}Target types supported: {', '.join(summary['target_types'])}{X}")
757
+ except Exception:
758
+ print(f"\n {D}Bellingcat toolkit not loaded (CSV not found){X}")
759
+
760
+ print(f"\n {D}Configure API keys with: {B}{sys.executable} -m watson.cli onboard{X}")
761
+ print(f" {D}Run doctor to verify: {B}{sys.executable} -m watson.cli doctor{X}\n")
762
+
763
+
764
+ # ═══════════════════════════════════════════════════════════════════
765
+ # ENTRY POINT
766
+ # ═══════════════════════════════════════════════════════════════════
767
+
768
+
769
+ def main_entry():
770
+ parser = argparse.ArgumentParser(
771
+ prog="watson",
772
+ description=f"Watson OSINT {VERSION} — {CODENAME}. Multi-source investigation engine.",
773
+ )
774
+ sub = parser.add_subparsers(dest="command", help="Command")
775
+
776
+ # onboard
777
+ sub.add_parser("onboard", help="First-time setup wizard")
778
+ # setup (alias)
779
+ sub.add_parser("setup", help="First-time setup wizard (alias for onboard)")
780
+
781
+ # config
782
+ p_config = sub.add_parser("config", help="Show or update configuration")
783
+ p_config.add_argument("key", nargs="?", help="Config key to get/set (use dots for nested: paid_api_keys.opensanctions)")
784
+ p_config.add_argument("value", nargs="?", help="New value (omit to read)")
785
+
786
+ # web
787
+ p_web = sub.add_parser("web", help="Start the web interface")
788
+ p_web.add_argument("--host", help="Bind address (default: 0.0.0.0)")
789
+ p_web.add_argument("--port", type=int, help="Port (default: 8777)")
790
+
791
+ # investigate
792
+ p_inv = sub.add_parser("investigate", help="Run a single investigation")
793
+ p_inv.add_argument("query", nargs="?", help="Target: person, company, domain, email, IP, wallet")
794
+
795
+ # graph
796
+ p_graph = sub.add_parser("graph", help="Show knowledge graph stats and entities")
797
+
798
+ # chat
799
+ p_chat = sub.add_parser("chat", help="Open the web interface (starts server if needed)")
800
+ p_chat.add_argument("--host", help="Server host (default: 127.0.0.1)")
801
+ p_chat.add_argument("--port", type=int, help="Port (default: 8777)")
802
+
803
+ # doctor
804
+ sub.add_parser("doctor", help="Run system health check — deps, config, APIs, graph")
805
+
806
+ # tools
807
+ sub.add_parser("tools", help="List available OSINT APIs and their status")
808
+
809
+ args = parser.parse_args()
810
+
811
+ if args.command in ("onboard", "setup"):
812
+ cmd_onboard()
813
+ elif args.command == "config":
814
+ cmd_config(args)
815
+ elif args.command == "web":
816
+ cmd_web(args)
817
+ elif args.command == "investigate":
818
+ cmd_investigate(args)
819
+ elif args.command == "graph":
820
+ cmd_graph(args)
821
+ elif args.command == "chat":
822
+ cmd_chat(args)
823
+ elif args.command == "doctor":
824
+ cmd_doctor(args)
825
+ elif args.command == "tools":
826
+ cmd_tools(args)
827
+ else:
828
+ print(WATSON_BANNER)
829
+ print(f" {_quote()}\n")
830
+ print(f" {G}First time? Run {B}{sys.executable} -m watson.cli setup{X}")
831
+ print(f" {D}Already set up? Run {B}{sys.executable} -m watson.cli chat{D} to open the interface{X}\n")
832
+ parser.print_help()
833
+
834
+
835
+ if __name__ == "__main__":
836
+ main_entry()