osintengine 1.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. osintengine-1.0.2.dist-info/METADATA +311 -0
  2. osintengine-1.0.2.dist-info/RECORD +103 -0
  3. osintengine-1.0.2.dist-info/WHEEL +5 -0
  4. osintengine-1.0.2.dist-info/entry_points.txt +2 -0
  5. osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
  6. osintengine-1.0.2.dist-info/top_level.txt +2 -0
  7. src/watson/__init__.py +10 -0
  8. src/watson/agent/__init__.py +96 -0
  9. src/watson/agents/__init__.py +101 -0
  10. src/watson/agents/protocol.py +196 -0
  11. src/watson/auth/__init__.py +68 -0
  12. src/watson/auth/store.py +145 -0
  13. src/watson/conversation.py +68 -0
  14. src/watson/core/__init__.py +0 -0
  15. src/watson/core/models.py +96 -0
  16. src/watson/ethics.py +205 -0
  17. src/watson/exports.py +182 -0
  18. src/watson/graph/__init__.py +69 -0
  19. src/watson/graph/entities.py +207 -0
  20. src/watson/graph/graph.py +489 -0
  21. src/watson/graph/osint_framework.py +266 -0
  22. src/watson/graph/relationships.py +116 -0
  23. src/watson/graph/transforms.py +787 -0
  24. src/watson/infra/__init__.py +43 -0
  25. src/watson/infra/cache.py +171 -0
  26. src/watson/infra/ratelimit.py +125 -0
  27. src/watson/infra/resilience.py +239 -0
  28. src/watson/infra/retry.py +186 -0
  29. src/watson/metrics.py +128 -0
  30. src/watson/opsec/__init__.py +290 -0
  31. src/watson/orchestration/__init__.py +23 -0
  32. src/watson/orchestration/engine.py +5587 -0
  33. src/watson/orchestration/executor.py +96 -0
  34. src/watson/orchestration/intent.py +139 -0
  35. src/watson/orchestration/intent_classifier.py +45 -0
  36. src/watson/orchestration/llm_config.py +160 -0
  37. src/watson/orchestration/resolution.py +555 -0
  38. src/watson/orchestration/scraper.py +94 -0
  39. src/watson/orchestration/synthesis.py +726 -0
  40. src/watson/orchestration/target_profile.py +748 -0
  41. src/watson/persistence/__init__.py +18 -0
  42. src/watson/persistence/models.py +161 -0
  43. src/watson/persistence/store.py +230 -0
  44. src/watson/pipeline/__init__.py +16 -0
  45. src/watson/pipeline/pre_synthesis.py +621 -0
  46. src/watson/search.py +103 -0
  47. src/watson/serializers/stix.py +451 -0
  48. src/watson/tools/__init__.py +31 -0
  49. src/watson/tools/base.py +97 -0
  50. src/watson/tools/blockchain.py +359 -0
  51. src/watson/tools/captcha.py +276 -0
  52. src/watson/tools/conflict.py +107 -0
  53. src/watson/tools/corporate.py +287 -0
  54. src/watson/tools/darkweb.py +144 -0
  55. src/watson/tools/geolocation.py +347 -0
  56. src/watson/tools/image_video.py +108 -0
  57. src/watson/tools/marinetraffic.py +142 -0
  58. src/watson/tools/people.py +476 -0
  59. src/watson/tools/registry.py +56 -0
  60. src/watson/tools/satellite.py +118 -0
  61. src/watson/tools/scraper.py +745 -0
  62. src/watson/tools/shodan.py +129 -0
  63. src/watson/tools/social_media.py +160 -0
  64. src/watson/tools/websites.py +291 -0
  65. src/watson/tools/wikidata.py +398 -0
  66. src/watson/utils/__init__.py +1 -0
  67. src/watson/utils/helpers.py +49 -0
  68. src/watson/utils/http.py +119 -0
  69. src/watson/verification/__init__.py +245 -0
  70. watson/__init__.py +6 -0
  71. watson/agents/__init__.py +20 -0
  72. watson/agents/base.py +103 -0
  73. watson/agents/direct.py +225 -0
  74. watson/agents/hermes.py +275 -0
  75. watson/agents/hermes_mcp.py +292 -0
  76. watson/api_keys.py +176 -0
  77. watson/browser_scraper.py +481 -0
  78. watson/cli.py +836 -0
  79. watson/crossref.py +175 -0
  80. watson/ethics.py +20 -0
  81. watson/graph.py +406 -0
  82. watson/mcp_server.py +601 -0
  83. watson/memory.py +552 -0
  84. watson/neo4j_graph.py +124 -0
  85. watson/opsec/__init__.py +307 -0
  86. watson/reporter.py +537 -0
  87. watson/scheduler.py +486 -0
  88. watson/serializers/stix.py +451 -0
  89. watson/terminal.py +410 -0
  90. watson/toolkit.py +252 -0
  91. watson/toolkit_api.py +347 -0
  92. watson/toolkit_automation.py +95 -0
  93. watson/toolkit_registry.py +345 -0
  94. watson/verification/__init__.py +251 -0
  95. watson/web/__init__.py +1 -0
  96. watson/web/app.py +1650 -0
  97. watson/web/middleware.py +301 -0
  98. watson/web/static/assets/index-B7hPOc0z.js +278 -0
  99. watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
  100. watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
  101. watson/web/static/index.html +14 -0
  102. watson/web/templates/chat.html +2 -0
  103. watson/web/templates/investigation-map.html +429 -0
watson/web/app.py ADDED
@@ -0,0 +1,1650 @@
1
+ """
2
+ Watson Web — FastAPI application (Phase 1 rewrite).
3
+
4
+ Routes:
5
+ GET / — Chat UI
6
+ GET /health — Health check
7
+ POST /api/chat — Tool-calling chat (streaming SSE)
8
+ POST /api/agent/investigate — Agent investigation
9
+ GET /api/agent/stream/<id> — Investigation SSE stream
10
+ POST /api/agent/chat/stream — Chat SSE (with tools)
11
+ + memory, scheduler, bellingcat, export, knowledge, terminal
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import threading
17
+ import asyncio
18
+ import json
19
+ import logging
20
+ import os
21
+ import re
22
+ import sys
23
+ import time
24
+ import uuid
25
+ from datetime import datetime, timezone
26
+ from pathlib import Path
27
+ from typing import Optional
28
+
29
+ # Ensure both src/ and project root are on path for imports
30
+ # src/watson/* → src.watson.agent, src.watson.auth, ...
31
+ # watson/* → watson.reporter, watson.web.middleware, ...
32
+ _src = Path(__file__).resolve().parent.parent.parent / "src"
33
+ _root = Path(__file__).resolve().parent.parent.parent
34
+ for p in (str(_root), str(_src)):
35
+ if p not in sys.path:
36
+ sys.path.insert(0, p)
37
+
38
+ from fastapi import FastAPI, HTTPException, Query, Request
39
+ from fastapi.middleware.cors import CORSMiddleware
40
+ from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, Response
41
+ from fastapi.staticfiles import StaticFiles
42
+ from pydantic import BaseModel
43
+
44
+ # ── App ─────────────────────────────────────────────────────────
45
+
46
+ app = FastAPI(
47
+ title="Watson OSINT",
48
+ description="The OSINT investigation engine. Evidence-based. Graph-native.",
49
+ version="0.3.0",
50
+ )
51
+
52
+ @app.on_event("startup")
53
+ async def _bind_sse_loop():
54
+ """Bind the event loop to SSEManager so events can be enqueued
55
+ from worker threads via call_soon_threadsafe."""
56
+ sse.set_loop(asyncio.get_running_loop())
57
+
58
+ app.add_middleware(
59
+ CORSMiddleware,
60
+ allow_origins=["*"],
61
+ allow_methods=["*"],
62
+ allow_headers=["*"],
63
+ )
64
+
65
+ # Auth middleware
66
+ from src.watson.auth import AuthMiddleware
67
+ app.add_middleware(AuthMiddleware)
68
+
69
+ # ── Health + Metrics ─────────────────────────────────────────────
70
+
71
+ from src.watson.infra.cache import all_cache_stats
72
+ from src.watson.infra.retry import _circuits
73
+ from src.watson.persistence import get_store
74
+ from src.watson.metrics import prometheus_endpoint, track_investigation
75
+
76
+ @app.get("/api/metrics")
77
+ async def metrics():
78
+ """Prometheus-compatible metrics endpoint."""
79
+ return Response(content=prometheus_endpoint(), media_type="text/plain")
80
+
81
+ @app.get("/api/health")
82
+ async def health():
83
+ """Health check with system status."""
84
+ store = get_store()
85
+ stats = store.get_stats()
86
+ cache_stats = all_cache_stats()
87
+ open_circuits = sum(1 for cb in _circuits.values() if cb.is_open)
88
+ return {
89
+ "status": "ok",
90
+ "version": "0.3.0-enterprise",
91
+ "investigations": stats,
92
+ "caches": {k: {"size": v["size"], "hit_rate": f"{v.get('hits',0)/max(v.get('hits',0)+v.get('misses',1),1):.1%}"}
93
+ for k, v in cache_stats.items()},
94
+ "circuit_breakers_open": open_circuits,
95
+ }
96
+
97
+ BASE_DIR = Path(__file__).resolve().parent
98
+ TEMPLATES_DIR = BASE_DIR / "templates"
99
+ STATIC_DIR = BASE_DIR / "static"
100
+ if STATIC_DIR.exists():
101
+ app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="assets")
102
+ app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
103
+
104
+ # MCP community graph server URL — configurable for shared instances
105
+ # Priority: env var > config file > default
106
+ def _load_mcp_config() -> dict:
107
+ """Load MCP settings from config file (env overrides)."""
108
+ mcp = {
109
+ "url": os.environ.get("WATSON_MCP_URL", ""),
110
+ "key": os.environ.get("MCP_API_KEY", ""),
111
+ }
112
+ try:
113
+ cfg_path = Path.home() / ".watson" / "config.json"
114
+ if cfg_path.exists():
115
+ cfg = json.loads(cfg_path.read_text())
116
+ if not mcp["url"]:
117
+ mcp["url"] = cfg.get("mcp_url", "")
118
+ if not mcp["key"]:
119
+ mcp["key"] = cfg.get("mcp_api_key", "")
120
+ except Exception:
121
+ pass
122
+ if not mcp["url"]:
123
+ mcp["url"] = "http://localhost:8700"
124
+ return mcp
125
+
126
+ _mcp = _load_mcp_config()
127
+ MCP_SERVER_URL = _mcp["url"]
128
+ MCP_API_KEY = _mcp["key"]
129
+
130
+ # ── Env loading (must be first — MCP server subprocess needs these vars) ──
131
+
132
+ def _load_env():
133
+ env_path = os.path.expanduser("~/.hermes/.env")
134
+ if not os.path.exists(env_path):
135
+ return
136
+ with open(env_path) as f:
137
+ for line in f:
138
+ if "=" in line and not line.startswith("#"):
139
+ k, v = line.strip().split("=", 1)
140
+ k, v = k.strip(), v.strip().strip('"').strip("'")
141
+ if k and v and k not in os.environ:
142
+ os.environ[k] = v
143
+
144
+ _load_env()
145
+
146
+ # ── Auto-start local MCP server ──────────────────────────────────
147
+
148
+ def _start_mcp_server():
149
+ """Start the MCP knowledge graph server as a subprocess if URL is local."""
150
+ if "localhost" not in MCP_SERVER_URL and "127.0.0.1" not in MCP_SERVER_URL:
151
+ return # Remote MCP — assume it's already running
152
+
153
+ mcp_port = MCP_SERVER_URL.rsplit(":", 1)[-1]
154
+ import subprocess as _sp
155
+ import sys as _sys
156
+ try:
157
+ proc = _sp.Popen(
158
+ [_sys.executable, "-m", "uvicorn", "watson.mcp_server:mcp",
159
+ "--host", "0.0.0.0", "--port", mcp_port],
160
+ cwd=str(Path(__file__).resolve().parents[2]), # project root
161
+ env={**os.environ, "PYTHONPATH": f".:{os.environ.get('PYTHONPATH', '')}"},
162
+ stdout=_sp.DEVNULL, stderr=_sp.DEVNULL,
163
+ )
164
+ logging.getLogger("watson").info("mcp_server_started", extra={"port": mcp_port, "pid": proc.pid})
165
+ except Exception as e:
166
+ logging.getLogger("watson").warning("mcp_server_start_failed: %s", e)
167
+
168
+ _start_mcp_server()
169
+
170
+ # ── Enterprise middleware ─────────────────────────────────────────
171
+
172
+ from watson.web.middleware import init_app, register_task
173
+
174
+ init_app(app) # Auth, rate limiting, structured logging, tracing, graceful shutdown
175
+
176
+ logger = logging.getLogger("watson")
177
+
178
+ import datetime as _dt
179
+
180
+
181
+ def _json_default(obj):
182
+ """JSON default handler: datetime → ISO, everything else → truncated str."""
183
+ if isinstance(obj, _dt.datetime):
184
+ return obj.isoformat()
185
+ if isinstance(obj, _dt.date):
186
+ return obj.isoformat()
187
+ # Safety net — never let str() produce huge blobs
188
+ s = str(obj)
189
+ return s[:200] if len(s) > 200 else s
190
+
191
+
192
+ _JSON_MAX_BYTES = 500_000 # 500 KB — drop events bigger than this
193
+
194
+
195
+ def _safe_serialize(ev_type: str, ev_data: dict) -> tuple[bool, str]:
196
+ """Serialize event data to SSE-safe JSON. Returns (ok, sse_string).
197
+
198
+ ok=False means serialization failed outright (exception during json.dumps).
199
+ ok=True with data starting with _TRUNCATED marker means the payload was
200
+ oversized and has been replaced with a stub.
201
+ """
202
+ try:
203
+ j = json.dumps(ev_data, default=_json_default)
204
+ if len(j) > _JSON_MAX_BYTES:
205
+ logger.warning(
206
+ "sse_event_too_large: type=%s size=%d — truncating", ev_type, len(j)
207
+ )
208
+ j = json.dumps({
209
+ "_truncated": True,
210
+ "_original_type": ev_type,
211
+ "_original_size": len(j),
212
+ "message": f"Event payload too large ({len(j)} bytes). "
213
+ f"Full data available in saved case report.",
214
+ })
215
+ return True, f"event: {ev_type}\ndata: {j}\n\n"
216
+ except Exception as e:
217
+ logger.warning("sse_serialize_failed: type=%s error=%s", ev_type, e)
218
+ return False, f"event: error\ndata: {json.dumps({'message': 'Serialization failed'})}\n\n"
219
+
220
+
221
+ # ── SSE helper (thread-safe) ─────────────────────────────────────
222
+
223
+ class SSEManager:
224
+ """Per-client SSE event queues — thread-safe via call_soon_threadsafe.
225
+
226
+ asyncio.Queue produces events correctly when enqueued from the event loop
227
+ thread. Investigations run in a worker thread (asyncio.to_thread), so we
228
+ use loop.call_soon_threadsafe() to safely enqueue from any thread.
229
+
230
+ Previous approaches that failed:
231
+ - asyncio.Queue + put_nowait(): race condition corrupts the queue
232
+ - queue.Queue + run_in_executor(get): leaked threads poison event delivery
233
+ (stale blocked threads capture events meant for the active consumer)
234
+ """
235
+ def __init__(self):
236
+ self._queues: dict[str, asyncio.Queue] = {}
237
+ self._lock = threading.Lock()
238
+ self._loop: asyncio.AbstractEventLoop | None = None
239
+
240
+ def set_loop(self, loop: asyncio.AbstractEventLoop):
241
+ self._loop = loop
242
+
243
+ def create(self, client_id: str) -> asyncio.Queue:
244
+ q = asyncio.Queue()
245
+ with self._lock:
246
+ self._queues[client_id] = q
247
+ return q
248
+
249
+ def send(self, client_id: str, event: str, data: dict):
250
+ with self._lock:
251
+ q = self._queues.get(client_id)
252
+ if q and self._loop:
253
+ # ── Pre-serialize to catch bad data before it hits the stream ──
254
+ ok, _pre = _safe_serialize(event, data)
255
+ if ok:
256
+ # call_soon_threadsafe queues a callback to run put_nowait
257
+ # in the event loop thread — the only safe way to touch asyncio.Queue
258
+ # from a worker thread.
259
+ self._loop.call_soon_threadsafe(
260
+ lambda q=q, item=(event, data): q.put_nowait(item)
261
+ )
262
+ else:
263
+ logger.warning("sse_dropped: type=%s — unserializable", event)
264
+
265
+ def remove(self, client_id: str):
266
+ with self._lock:
267
+ self._queues.pop(client_id, None)
268
+
269
+ sse = SSEManager()
270
+
271
+ # ── Models ───────────────────────────────────────────────────────
272
+
273
+ class InvestigateRequest(BaseModel):
274
+ query: str
275
+ client_id: Optional[str] = None
276
+ image_path: Optional[str] = None
277
+ depth: int = 2
278
+ context: str = ""
279
+ mode: str = "deep_investigation" # "background_check" | "due_diligence" | "deep_investigation"
280
+ publish_to_graph: bool = False # Per-case consent: publish findings to community knowledge graph
281
+
282
+ @property
283
+ def safe_depth(self) -> int:
284
+ return max(1, min(self.depth, 5)) # Clamp 1-5
285
+
286
+ class ChatRequest(BaseModel):
287
+ message: str
288
+ findings: list = []
289
+ history: list = []
290
+ last_query: str = ""
291
+
292
+ # ── Routes ───────────────────────────────────────────────────────
293
+
294
+ _NO_CACHE = {
295
+ "Cache-Control": "no-cache, no-store, must-revalidate",
296
+ "Pragma": "no-cache",
297
+ "Expires": "0",
298
+ }
299
+
300
+ @app.get("/", response_class=HTMLResponse)
301
+ async def root():
302
+ """Serve the React app (built to static/) or fall back to old HTML."""
303
+ index = STATIC_DIR / "index.html"
304
+ if index.exists():
305
+ return HTMLResponse(content=index.read_text(), headers=_NO_CACHE)
306
+ # Fallback to old template
307
+ path = TEMPLATES_DIR / "investigation-map.html"
308
+ if path.exists():
309
+ return HTMLResponse(content=path.read_text(), headers=_NO_CACHE)
310
+ return HTMLResponse("<h1>Watson — run <code>cd frontend && npm run build</code></h1>", status_code=404)
311
+
312
+ @app.get("/chat", response_class=HTMLResponse)
313
+ async def chat():
314
+ """Serve the React app at /chat too."""
315
+ return await root()
316
+ return HTMLResponse(content=path.read_text(), headers=_NO_CACHE)
317
+
318
+ @app.get("/health")
319
+ async def health():
320
+ """Health check with dependency verification."""
321
+ deps = {}
322
+
323
+ # Check DeepSeek API
324
+ api_key = os.environ.get("DEEPSEEK_API_KEY", "")
325
+ deps["deepseek_api"] = "configured" if api_key and api_key != "***" else "missing"
326
+
327
+ # Check memory DB
328
+ try:
329
+ from watson.memory import memory as mem
330
+ stats = mem.stats()
331
+ deps["memory_db"] = {"status": "ok", "investigations": stats.get("investigations", 0)}
332
+ except Exception:
333
+ deps["memory_db"] = {"status": "degraded"}
334
+
335
+ # Check knowledge graph
336
+ try:
337
+ from watson.neo4j_graph import NEO4J_AVAILABLE
338
+ deps["neo4j"] = "available" if NEO4J_AVAILABLE else "fallback_json"
339
+ except Exception:
340
+ deps["neo4j"] = "unknown"
341
+
342
+ all_ok = all(
343
+ v not in ("missing",) and (isinstance(v, dict) and v.get("status") != "degraded") or isinstance(v, str) and v not in ("missing",)
344
+ for v in deps.values()
345
+ ) or True # Don't fail health for missing configs in dev
346
+
347
+ return {
348
+ "status": "ok",
349
+ "version": "1.0.0",
350
+ "server": "FastAPI/uvicorn",
351
+ "dependencies": deps,
352
+ }
353
+
354
+ @app.get("/api/tools")
355
+ async def api_tools():
356
+ return {
357
+ "total": 12,
358
+ "categories": [
359
+ {"category": "toolkit", "count": 1, "tools": [{"name": "osint-toolkit", "description": "338 OSINT investigation tools", "free": True}]},
360
+ {"category": "corporate", "count": 1, "tools": [{"name": "corporate-finance", "description": "Company registries", "free": True}]},
361
+ {"category": "websites", "count": 2, "tools": [{"name": "websites-domains", "description": "Domain investigation", "free": True}, {"name": "browser-automation", "description": "Headless browser", "free": True}]},
362
+ {"category": "social_media", "count": 1, "tools": [{"name": "social-media", "description": "Social media search", "free": True}]},
363
+ {"category": "people", "count": 2, "tools": [{"name": "people-search", "description": "Person lookup", "free": True}, {"name": "scraper", "description": "Wiki/OpenSanctions", "free": True}]},
364
+ ],
365
+ }
366
+
367
+ # ── Agent Investigation ──────────────────────────────────────────
368
+
369
+ @app.post("/api/agent/investigate")
370
+ async def agent_investigate(req: InvestigateRequest):
371
+ """Start a Watson v1 investigation. Returns client_id for SSE stream."""
372
+ import gc
373
+ gc.collect()
374
+
375
+ from src.watson.orchestration import get_engine
376
+
377
+ client_id = f"agent-{uuid.uuid4().hex[:8]}"
378
+ q = sse.create(client_id)
379
+
380
+ async def run():
381
+ def push(event_type, data):
382
+ sse.send(client_id, event_type, data)
383
+ # ── Mode-based hard timeout — prevents hung investigations from blocking server ──
384
+ _mode_timeouts = {
385
+ "background_check": 120,
386
+ "due_diligence": 420,
387
+ "deep_investigation": 900,
388
+ "twin_connection": 300,
389
+ }
390
+ _hard_timeout = _mode_timeouts.get(req.mode, 600)
391
+ try:
392
+ engine = get_engine()
393
+ # Register interrupt queue for interactive steering
394
+ engine.register_interrupt_queue(client_id)
395
+ # ── Run investigation in a dedicated thread so CPU-bound loops
396
+ # don't block the uvicorn event loop. The asyncio.wait_for timeout
397
+ # will fire even if the thread hangs. ──
398
+ def _run_sync():
399
+ loop = asyncio.new_event_loop()
400
+ asyncio.set_event_loop(loop)
401
+ try:
402
+ return loop.run_until_complete(engine.investigate(
403
+ query=req.query,
404
+ focus=req.context,
405
+ on_event=push,
406
+ mode=req.mode,
407
+ save_mode="auto",
408
+ ))
409
+ finally:
410
+ loop.close()
411
+ result = await asyncio.wait_for(
412
+ asyncio.to_thread(_run_sync),
413
+ timeout=_hard_timeout,
414
+ )
415
+
416
+ # Send final report event with markdown
417
+ sse.send(client_id, "report", {
418
+ "case_id": result["case_id"],
419
+ "findings_count": result["findings_count"],
420
+ "confirmed": result["confirmed_count"],
421
+ "verifiability": f"{result['verifiability_score']:.0%}",
422
+ "markdown": result.get("markdown", ""),
423
+ "published_to_graph": req.publish_to_graph,
424
+ })
425
+
426
+ # ── Per-case consent: publish to community knowledge graph ──
427
+ if req.publish_to_graph and result.get("findings"):
428
+ publish_error = None
429
+ try:
430
+ # Check if key is available for remote MCP
431
+ is_local = "localhost" in MCP_SERVER_URL or "127.0.0.1" in MCP_SERVER_URL
432
+ if not is_local and not MCP_API_KEY:
433
+ publish_error = "No MCP API key configured. Set it in onboarding or with: export MCP_API_KEY=your-key"
434
+ else:
435
+ # Use the engine's report if available, otherwise publish manually
436
+ engine_report = getattr(result, '_report', None)
437
+ if engine_report:
438
+ _pub_result, _pub_err = _publish_to_mcp(engine_report)
439
+ if _pub_err:
440
+ publish_error = _pub_err
441
+ else:
442
+ # Fallback: publish entities from findings directly
443
+ import httpx
444
+ entities = []
445
+ for f in result["findings"]:
446
+ if hasattr(f, 'tier') and f.tier in ("CONFIRMED", "PROBABLE"):
447
+ for ent in getattr(f, 'entities', []):
448
+ # Normalize entities — they come in as strings, dicts, or objects
449
+ if isinstance(ent, str):
450
+ val, etype = ent.strip(), "unknown"
451
+ elif isinstance(ent, dict):
452
+ val = ent.get("value", ent.get("canonical", ""))
453
+ etype = ent.get("type", "unknown")
454
+ elif hasattr(ent, 'value'):
455
+ val = getattr(ent, 'value', '') or getattr(ent, 'canonical', '')
456
+ etype = getattr(ent, 'type', 'unknown')
457
+ else:
458
+ val, etype = str(ent), "unknown"
459
+ # Skip empty values — MCP validator rejects them
460
+ if not val or not val.strip():
461
+ continue
462
+ entities.append({
463
+ "value": val.strip()[:500],
464
+ "type": etype or "unknown",
465
+ "source": ent.get("source", "") if isinstance(ent, dict) else "",
466
+ "tier": f.tier if hasattr(f, 'tier') else "PROBABLE",
467
+ "case_id": result["case_id"],
468
+ })
469
+ if entities:
470
+ payload = {
471
+ "case_id": result["case_id"],
472
+ "target": req.query,
473
+ "target_type": "",
474
+ "findings_count": len(result["findings"]),
475
+ "confirmed_count": result.get("confirmed_count", 0),
476
+ "verifiability": "",
477
+ "date": "",
478
+ "entities": entities,
479
+ }
480
+ resp = httpx.post(
481
+ f"{MCP_SERVER_URL}/api/ingest",
482
+ json=payload, timeout=10,
483
+ headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {},
484
+ )
485
+ if resp.status_code != 200:
486
+ publish_error = f"MCP server returned {resp.status_code}"
487
+ except Exception as e:
488
+ publish_error = str(e)
489
+
490
+ if publish_error:
491
+ sse.send(client_id, "graph_error", {
492
+ "message": f"Graph publish failed: {publish_error}",
493
+ "case_id": result["case_id"],
494
+ })
495
+ else:
496
+ sse.send(client_id, "graph_published", {
497
+ "case_id": result["case_id"],
498
+ "url": MCP_SERVER_URL,
499
+ })
500
+
501
+ logger.info("investigation_complete", extra={
502
+ "client_id": client_id, "query": req.query[:80],
503
+ "findings": result["findings_count"],
504
+ "case": result["case_id"],
505
+ })
506
+ except asyncio.TimeoutError:
507
+ logger.error("investigation_timeout", extra={
508
+ "client_id": client_id, "query": req.query[:80],
509
+ "mode": req.mode,
510
+ })
511
+ sse.send(client_id, "error", {
512
+ "message": f"Investigation timed out after {_hard_timeout}s — try a shallower mode or narrower query"
513
+ })
514
+ except Exception as e:
515
+ logger.error("investigation_failed", extra={
516
+ "client_id": client_id, "query": req.query[:80], "error": str(e),
517
+ })
518
+ sse.send(client_id, "error", {"message": str(e)})
519
+ finally:
520
+ await asyncio.sleep(0.5)
521
+ sse.send(client_id, "_close", {})
522
+ # Cleanup interrupt queue
523
+ from src.watson.orchestration import get_engine as _get_eng
524
+ _eng = _get_eng()
525
+ _eng.remove_interrupt_queue(client_id)
526
+
527
+ task = asyncio.create_task(run())
528
+ register_task(task)
529
+ return {"client_id": client_id, "status": "started"}
530
+
531
+ @app.get("/api/agent/stream/{client_id}")
532
+ async def agent_stream(client_id: str):
533
+ """SSE stream for an investigation in progress."""
534
+ q = sse._queues.get(client_id)
535
+ if not q:
536
+ async def error_gen():
537
+ yield f"event: error\ndata: {json.dumps({'message': 'Stream not found'})}\n\n"
538
+ yield "event: done\ndata: {}\n\n"
539
+ return StreamingResponse(error_gen(), media_type="text/event-stream")
540
+
541
+ async def generate():
542
+ try:
543
+ while True:
544
+ try:
545
+ ev_type, ev_data = await asyncio.wait_for(q.get(), timeout=30.0)
546
+ _ok, sse_str = _safe_serialize(ev_type, ev_data)
547
+ yield sse_str
548
+ if ev_type == "_close":
549
+ break
550
+ except asyncio.TimeoutError:
551
+ yield ": heartbeat\n\n"
552
+ finally:
553
+ sse.remove(client_id)
554
+
555
+ return StreamingResponse(
556
+ generate(),
557
+ media_type="text/event-stream",
558
+ headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
559
+ )
560
+
561
+
562
+ # ── Interactive Investigation Steering ──────────────────────────
563
+
564
+ class InterruptRequest(BaseModel):
565
+ action: str = "context" # "context", "stop", "skip_phase"
566
+ text: str = ""
567
+
568
+ @app.post("/api/agent/investigate/{client_id}/interrupt")
569
+ async def agent_interrupt(client_id: str, req: InterruptRequest):
570
+ """Send a steering command to a running investigation."""
571
+ from src.watson.orchestration import get_engine
572
+ engine = get_engine()
573
+ ok = engine.send_interrupt(client_id, {"action": req.action, "text": req.text})
574
+ if ok:
575
+ return {"status": "sent", "client_id": client_id}
576
+ return JSONResponse(
577
+ status_code=404,
578
+ content={"status": "not_found", "message": f"No active investigation: {client_id}"}
579
+ )
580
+
581
+
582
+ # ── Agent Chat with Tools (SSE) ──────────────────────────────────
583
+
584
+ WATSON_SOUL = (
585
+ "You are Watson, an autonomous OSINT investigator.\n\n"
586
+ "ABSOLUTE RULES — VIOLATION MEANS FAILURE:\n"
587
+ "- NEVER fabricate specific identifiers: no invented SAR numbers, VINs, wallet addresses, "
588
+ "transaction hashes, case IDs, or document numbers.\n"
589
+ "- NEVER invent reports or documents you cannot link to a public URL.\n"
590
+ "- NEVER fabricate dollar amounts from inaccessible databases.\n"
591
+ "- MARK UNCERTAINTY: [CONFIRMED], [PLAUSIBLE BUT UNVERIFIED], [HYPOTHETICAL — NOT REAL].\n"
592
+ "- When OSINT data runs out, STOP and state the boundary.\n\n"
593
+ "CAPABILITIES: web_search, fetch_url, whois_lookup, dns_lookup, crt_sh_search, "
594
+ "wayback_machine, etherscan_lookup, opencorporates_search, news_search, "
595
+ "run_terminal_command, investigate_target.\n\n"
596
+ "TONE: Sharp, direct, evidence-based. Brief like an intelligence analyst.\n"
597
+ "Use bullet points. Cite sources. Flag gaps.\n\n"
598
+ "CRITICAL — AFTER EVERY ANSWER:\n"
599
+ "If the user has CURRENT FINDINGS from an active investigation, you MUST end your response "
600
+ "with a 'Follow-up leads' section suggesting 2-3 concrete next investigation steps based on:\n"
601
+ "- Entities discovered but not yet explored (domains, people, companies in the findings)\n"
602
+ "- Gaps in the evidence (missing WHOIS details, unverified registrars, unresearched nameservers)\n"
603
+ "- Cross-references that deserve deeper investigation\n"
604
+ "Format each lead as: '🔍 Investigate [entity/value] — [why it matters]'\n"
605
+ "This is mandatory — the investigation is incomplete without follow-up leads.\n"
606
+ )
607
+
608
+ @app.post("/api/agent/chat/stream")
609
+ async def agent_chat_stream(req: ChatRequest):
610
+ """Streaming chat — delegates to investigation engine for chat-style queries."""
611
+ async def respond():
612
+ yield f"event: token\ndata: {json.dumps({'token': 'Chat mode coming soon. Use /api/agent/investigate for full OSINT investigations.'})}\n\n"
613
+ yield "event: done\ndata: {}\n\n"
614
+ return StreamingResponse(respond(), media_type="text/event-stream")
615
+
616
+ # ── Agent Tools ──────────────────────────────────────────────────
617
+
618
+ @app.get("/api/agent/detect-intent")
619
+ async def detect_intent(msg: str = Query("", description="Message to classify")):
620
+ """Determine whether a message is an investigation request or a chat question.
621
+ Public endpoint — no auth required."""
622
+ msg = msg.strip()
623
+ if not msg:
624
+ return {"intent": "chat", "confidence": 1.0}
625
+
626
+ # TECHNICAL TARGETS — clearly investigable, auto-dispatch
627
+ if re.search(r"\.(com|org|net|io|gov|edu|uk|de|fr|ru|cn|jp|ai|dev|onion)\b", msg, re.IGNORECASE):
628
+ return {"intent": "investigate", "confidence": 0.95, "reason": "domain_detected"}
629
+ if re.search(r"@[\w.-]+\.[a-z]{2,}", msg):
630
+ return {"intent": "investigate", "confidence": 0.95, "reason": "email_detected"}
631
+ if re.search(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", msg):
632
+ return {"intent": "investigate", "confidence": 0.95, "reason": "ip_detected"}
633
+ if re.search(r"0x[a-fA-F0-9]{40}", msg):
634
+ return {"intent": "investigate", "confidence": 0.95, "reason": "crypto_detected"}
635
+
636
+ # EXPLICIT INVESTIGATION COMMANDS
637
+ if re.match(r"^(investigate|look\s+into|research|dig\s+into|check\s+out|find\s+everything\s+(?:about|on))\s+", msg, re.IGNORECASE):
638
+ return {"intent": "investigate", "confidence": 0.85, "reason": "explicit_command"}
639
+
640
+ # OSINT-INTENT PHRASES — these signal "investigate X", not a chat question.
641
+ # "Amazon due diligence", "background check on Acme", "Tesla controversies",
642
+ # "risks of investing in X", "Acme lawsuits/sanctions/scandal".
643
+ osint_intent = re.search(
644
+ r"(?i)\b(due\s+diligence|background\s+check|kyc|aml|"
645
+ r"controvers(?:y|ies)|lawsuit?s?|litigation|sanction(?:s|ed)?|"
646
+ r"scandal|fraud|investigation|red\s+flags?|reputation|"
647
+ r"risk\s+(?:assessment|profile|exposure)|adverse\s+media|"
648
+ r"compliance|regulatory\s+action|antitrust|wrongdoing)\b",
649
+ msg,
650
+ )
651
+ if osint_intent:
652
+ return {"intent": "investigate", "confidence": 0.88,
653
+ "reason": f"osint_intent:{osint_intent.group(1).lower()}"}
654
+
655
+ # OSINT-QUESTION PATTERNS — questions that demand investigation, not chat.
656
+ # "What is the corporate structure of X", "Who owns Y", "How did Z become involved"
657
+ # These look like "what/who/how" questions but are OSINT targets.
658
+ osint_question = re.search(
659
+ r"(?i)\b("
660
+ r"corporate\s+structure|beneficial\s+owner(?:s|ship)?|"
661
+ r"who\s+owns?|who\s+controls?|"
662
+ r"ownership\s+(?:structure|chain|network)|"
663
+ r"shell\s+compan|subsidiary|parent\s+compan|"
664
+ r"supply\s+chain|shipping\s+(?:compan|fleet|network)|"
665
+ r"how\s+did\s+.+\s+become\s+involved|"
666
+ r"what\s+(?:companies|entities|firms)\s+(?:are|own|control|operate)"
667
+ r")\b",
668
+ msg,
669
+ )
670
+ if osint_question:
671
+ return {"intent": "investigate", "confidence": 0.82,
672
+ "reason": f"osint_question:{osint_question.group(1).lower()[:40]}"}
673
+
674
+ # CHAT PATTERNS — questions about knowledge, not OSINT targets
675
+ chat_patterns = [
676
+ r"^(what|who|where|when|why|how)\s+(is|are|was|were|do|does|did|can|could|would|should|shall|will|may|might)",
677
+ r"\?$", # ends with question mark
678
+ r"^(tell|explain|describe|show|list|summarize|compare|define|elaborate)",
679
+ r"^(hello|hi|hey|help|thanks|thank|what's up|howdy)",
680
+ r"^(what|who|where|when|why|how)\s+(about|to|can|could|do)",
681
+ r"^(can|will|would)\s+you\s+(tell|explain|show|help|list)",
682
+ r"^(i\s+(?:have|need|want)\s+(?:a|some|to)\s+(?:question|ask|know|understand))",
683
+ ]
684
+ for pattern in chat_patterns:
685
+ if re.search(pattern, msg, re.IGNORECASE):
686
+ return {"intent": "chat", "confidence": 0.90, "reason": "conversational_pattern"}
687
+
688
+ # ── LLM CLASSIFIER FALLBACK ──
689
+ # If none of the fast-path regexes matched, ask the LLM. This replaces
690
+ # the old "default to chat" which trapped investigable targets like
691
+ # "Area 51" and article headlines in a 5-turn conversation loop.
692
+ try:
693
+ from src.watson.orchestration.intent_classifier import classify_intent
694
+ from src.watson.orchestration.llm_config import call_llm
695
+
696
+ intent, confidence, reason = await classify_intent(msg, call_llm)
697
+ return {"intent": intent, "confidence": confidence, "reason": reason}
698
+ except Exception as e:
699
+ logger.warning("intent_classifier_error: %s", e)
700
+
701
+ # Absolute last resort — short non-question messages are probably targets
702
+ word_count = len(msg.split())
703
+ if "?" not in msg and word_count <= 15 and any(c.isalpha() for c in msg):
704
+ return {"intent": "investigate", "confidence": 0.50,
705
+ "reason": "fallback_short_topic"}
706
+ return {"intent": "chat", "confidence": 0.50, "reason": "fallback_ambiguous"}
707
+
708
+ @app.post("/api/agent/terminal")
709
+ async def agent_terminal(req: Request):
710
+ """Terminal command execution — admin key required, allowlist enforced."""
711
+ import subprocess
712
+
713
+ # Admin auth is handled by middleware — if we get here, it passed
714
+ data = await req.json()
715
+ cmd = data.get("command", "").strip()
716
+ if not cmd:
717
+ return JSONResponse({"error": "No command"}, status_code=400)
718
+
719
+ # Command allowlist — only safe OSINT tools
720
+ ALLOWED = {"whois", "dig", "nslookup", "curl", "wget", "traceroute", "ping",
721
+ "openssl", "python3", "git", "date", "echo", "cat", "head", "tail"}
722
+ cmd_root = cmd.split()[0].split("/")[-1] if cmd.split() else ""
723
+ if cmd_root not in ALLOWED and not cmd.startswith(("python3 -c", "git ", "curl ", "echo ")):
724
+ logger.warning("terminal_blocked", extra={"command": cmd[:100], "root": cmd_root})
725
+ return JSONResponse(
726
+ {"error": f"Command '{cmd_root}' not in allowlist. Allowed: {', '.join(sorted(ALLOWED))}"},
727
+ status_code=403,
728
+ )
729
+
730
+ try:
731
+ result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=30)
732
+ logger.info("terminal_executed", extra={"command": cmd[:100], "exit_code": result.returncode})
733
+ return {"stdout": result.stdout.strip()[:10000], "stderr": result.stderr.strip()[:5000], "exit_code": result.returncode}
734
+ except subprocess.TimeoutExpired:
735
+ return {"error": "Timed out (30s)", "exit_code": -1}
736
+ except Exception as e:
737
+ logger.error("terminal_failed", extra={"command": cmd[:100], "error": str(e)})
738
+ return {"error": str(e), "exit_code": -1}
739
+
740
+ # ── Memory Endpoints ─────────────────────────────────────────────
741
+
742
+ @app.get("/api/memory/search")
743
+ async def memory_search(q: str = Query(...), limit: int = 10):
744
+ try:
745
+ from watson.memory import memory as mem
746
+ return {"results": mem.search(q, limit=limit)}
747
+ except Exception as e:
748
+ logger.warning("memory_search_failed", extra={"query": q[:100], "error": str(e)})
749
+ return {"results": []}
750
+
751
+ @app.get("/api/memory/recent")
752
+ async def memory_recent(limit: int = 20):
753
+ try:
754
+ from watson.memory import memory as mem
755
+ return {"investigations": mem.list_recent(limit=limit)}
756
+ except Exception as e:
757
+ logger.warning("memory_recent_failed", extra={"error": str(e)})
758
+ return {"investigations": []}
759
+
760
+ @app.get("/api/memory/stats")
761
+ async def memory_stats():
762
+ try:
763
+ from watson.memory import memory as mem
764
+ return mem.stats()
765
+ except Exception as e:
766
+ logger.warning("memory_stats_failed", extra={"error": str(e)})
767
+ return {"investigations": 0, "entities": 0, "findings": 0}
768
+
769
+ # ── Scheduler ────────────────────────────────────────────────────
770
+
771
+ @app.get("/api/scheduler/jobs")
772
+ async def scheduler_list():
773
+ try:
774
+ from watson.core.scheduler import Scheduler
775
+ sched = Scheduler()
776
+ return {"jobs": sched.list_jobs(), "total": len(sched.list_jobs())}
777
+ except Exception as e:
778
+ logger.warning("scheduler_list_failed", extra={"error": str(e)})
779
+ return {"jobs": [], "total": 0}
780
+
781
+ # ── Toolkit Registry ─────────────────────────────────────────────
782
+
783
+ @app.get("/api/bellingcat/summary")
784
+ async def bellingcat_summary():
785
+ try:
786
+ from watson.toolkit_registry import BellingcatRegistry
787
+ reg = BellingcatRegistry()
788
+ return reg.summary()
789
+ except Exception as e:
790
+ logger.warning("bellingcat_summary_failed", extra={"error": str(e)})
791
+ return {"total_tools": 0, "categories": 0}
792
+
793
+ # ── Intelligence Ledger ──────────────────────────────────────────
794
+
795
+ @app.get("/api/ledger/stats")
796
+ async def ledger_stats():
797
+ """Cross-case intelligence accumulation statistics."""
798
+ try:
799
+ from src.watson.ethics import get_ledger
800
+ ledger = get_ledger()
801
+ return ledger.get_stats()
802
+ except ImportError:
803
+ try:
804
+ from watson.ethics import get_ledger
805
+ ledger = get_ledger()
806
+ return ledger.get_stats()
807
+ except Exception:
808
+ return {"total_investigations": 0, "status": "ledger_unavailable"}
809
+
810
+ @app.get("/api/ledger/entity")
811
+ async def ledger_entity(q: str = Query(..., description="Entity to query")):
812
+ """Query prior intelligence on a specific entity."""
813
+ try:
814
+ from src.watson.ethics import get_ledger
815
+ ledger = get_ledger()
816
+ result = ledger.get_entity_intel(q)
817
+ if result:
818
+ return {"found": True, "entity": q, "intel": result}
819
+ return {"found": False, "entity": q}
820
+ except ImportError:
821
+ try:
822
+ from watson.ethics import get_ledger
823
+ ledger = get_ledger()
824
+ result = ledger.get_entity_intel(q)
825
+ return {"found": bool(result), "entity": q, "intel": result}
826
+ except Exception:
827
+ return {"found": False, "entity": q, "status": "ledger_unavailable"}
828
+
829
+ # ── Orchestrator (multi-agent, enterprise) ─────────────────────────
830
+
831
+ @app.post("/api/agent/orchestrate")
832
+ async def agent_orchestrate(req: InvestigateRequest):
833
+ """Multi-turn investigation with persistence, retry, and adversarial resilience."""
834
+ import gc
835
+ gc.collect()
836
+
837
+ from src.watson.orchestration import get_engine
838
+ from src.watson.metrics import investigations_total, findings_total, findings_confirmed
839
+
840
+ engine = get_engine(max_hops=5)
841
+ client_id = f"orch-{uuid.uuid4().hex[:8]}"
842
+ q = sse.create(client_id)
843
+
844
+ async def run():
845
+ def push(event_type, data):
846
+ sse.send(client_id, event_type, data)
847
+ try:
848
+ inv = await engine.investigate(
849
+ query=req.query,
850
+ context=req.context or "",
851
+ on_event=push,
852
+ image_path=req.image_path or "",
853
+ )
854
+
855
+ findings_total.inc(inv.total_findings)
856
+ findings_confirmed.inc(inv.confirmed_count)
857
+
858
+ # Generate report
859
+ import json as _json
860
+ cross_refs = _json.loads(inv.cross_references) if inv.cross_references else []
861
+
862
+ sse.send(client_id, "report", {
863
+ "investigation_id": inv.investigation_id,
864
+ "status": inv.status.value,
865
+ "findings_count": inv.total_findings,
866
+ "hops": inv.total_hops,
867
+ "confirmed": inv.confirmed_count,
868
+ "cross_references": len(cross_refs),
869
+ "created_at": inv.created_at,
870
+ })
871
+ sse.send(client_id, "cross_references", {"patterns": cross_refs[:5]})
872
+
873
+ logger.info("orchestration_complete", extra={
874
+ "investigation_id": inv.investigation_id,
875
+ "query": req.query[:80],
876
+ "findings": inv.total_findings,
877
+ "hops": inv.total_hops,
878
+ "confirmed": inv.confirmed_count,
879
+ })
880
+ except Exception as e:
881
+ logger.error("orchestration_failed", extra={
882
+ "client_id": client_id, "query": req.query[:80], "error": str(e),
883
+ })
884
+ sse.send(client_id, "error", {"message": str(e)})
885
+ finally:
886
+ await asyncio.sleep(0.5)
887
+ sse.send(client_id, "_close", {})
888
+ # Cleanup interrupt queue
889
+ from src.watson.orchestration import get_engine as _get_eng
890
+ _eng = _get_eng()
891
+ _eng.remove_interrupt_queue(client_id)
892
+
893
+ task = asyncio.create_task(run())
894
+ register_task(task)
895
+ return {"client_id": client_id, "status": "started", "mode": "multi-agent"}
896
+
897
+
898
+ @app.get("/api/agent/agents")
899
+ async def list_agents():
900
+ """List all available specialized agents with capabilities."""
901
+ from src.watson.agents import get_all_agents
902
+ agents = get_all_agents()
903
+ return {
904
+ "agents": [
905
+ {
906
+ "role": a.role.value,
907
+ "description": a.description,
908
+ "capabilities": a.capabilities,
909
+ "tool_count": a.tool_count,
910
+ }
911
+ for a in agents.values()
912
+ ],
913
+ "total": len(agents),
914
+ }
915
+
916
+
917
+ # ── Legacy compat ────────────────────────────────────────────────
918
+
919
+ @app.post("/api/chat")
920
+ async def legacy_chat(req: ChatRequest):
921
+ """Legacy alias → forwards to streaming chat."""
922
+ return await agent_chat_stream(req)
923
+
924
+
925
+ # ═══════════════════════════════════════════════════════════════
926
+ # PHASE B — Auth, Exports, Search, Workspaces
927
+ # ═══════════════════════════════════════════════════════════════
928
+
929
+ # ── Auth ─────────────────────────────────────────────────────
930
+
931
+ @app.post("/api/auth/register")
932
+ async def auth_register(req: Request):
933
+ """Register a new workspace + admin user."""
934
+ body = await req.json()
935
+ email = body.get("email", "").strip()
936
+ workspace_name = body.get("workspace", "").strip()
937
+
938
+ if not email or "@" not in email:
939
+ return JSONResponse(status_code=400, content={"error": "Valid email required"})
940
+ if not workspace_name:
941
+ return JSONResponse(status_code=400, content={"error": "Workspace name required"})
942
+
943
+ try:
944
+ from src.watson.auth.store import get_auth_store
945
+ store = get_auth_store()
946
+ ws, user = store.create_workspace(workspace_name, email)
947
+ api_key = store.generate_api_key(user.user_id)
948
+ token = store.create_token(user)
949
+
950
+ return {
951
+ "user_id": user.user_id,
952
+ "workspace_id": ws.workspace_id,
953
+ "workspace_name": ws.name,
954
+ "api_key": api_key,
955
+ "token": token.token,
956
+ "expires_at": token.expires_at,
957
+ }
958
+ except Exception as e:
959
+ return JSONResponse(status_code=409, content={"error": str(e)})
960
+
961
+ @app.post("/api/auth/login")
962
+ async def auth_login(req: Request):
963
+ """Login with API key or email + workspace."""
964
+ body = await req.json()
965
+ api_key = body.get("api_key", "").strip()
966
+
967
+ try:
968
+ from src.watson.auth.store import get_auth_store
969
+ store = get_auth_store()
970
+
971
+ if api_key:
972
+ user = store.validate_api_key(api_key)
973
+ else:
974
+ email = body.get("email", "").strip()
975
+ workspace_id = body.get("workspace_id", "").strip()
976
+ user = store.get_user_by_email(email, workspace_id)
977
+
978
+ if not user:
979
+ return JSONResponse(status_code=401, content={"error": "Invalid credentials"})
980
+
981
+ token = store.create_token(user)
982
+ return {
983
+ "user_id": user.user_id,
984
+ "workspace_id": user.workspace_id,
985
+ "role": user.role.value,
986
+ "token": token.token,
987
+ "expires_at": token.expires_at,
988
+ }
989
+ except Exception as e:
990
+ return JSONResponse(status_code=500, content={"error": str(e)})
991
+
992
+
993
+ # ── Workspaces ───────────────────────────────────────────────
994
+
995
+ @app.get("/api/workspace/{workspace_id}")
996
+ async def workspace_get(workspace_id: str, req: Request):
997
+ """Get workspace details."""
998
+ from src.watson.auth.store import get_auth_store
999
+ store = get_auth_store()
1000
+ ws = store.get_workspace(workspace_id)
1001
+ if not ws:
1002
+ return JSONResponse(status_code=404, content={"error": "Workspace not found"})
1003
+ users = store.list_users(workspace_id)
1004
+ return {
1005
+ "workspace": {
1006
+ "id": ws.workspace_id, "name": ws.name,
1007
+ "created_at": ws.created_at,
1008
+ },
1009
+ "users": [{"id": u.user_id, "email": u.email, "role": u.role.value} for u in users],
1010
+ }
1011
+
1012
+
1013
+ # ── Exports ──────────────────────────────────────────────────
1014
+
1015
+ class ExportRequest(BaseModel):
1016
+ investigation_id: str
1017
+ format: str = "json" # json, stix, misp, pdf, markdown
1018
+
1019
+
1020
+ @app.post("/api/export")
1021
+ async def export_investigation(export: ExportRequest):
1022
+ """Export investigation in requested format."""
1023
+ from src.watson.persistence import get_store
1024
+ from src.watson.exports import BellingcatReport
1025
+
1026
+ store = get_store()
1027
+ inv, steps = store.get_full_investigation(export.investigation_id)
1028
+ if inv is None:
1029
+ return JSONResponse(status_code=404, content={"error": "Investigation not found"})
1030
+
1031
+ # Collect findings from all steps
1032
+ all_findings = []
1033
+ for s in steps:
1034
+ if s.findings_json:
1035
+ try:
1036
+ all_findings.extend(json.loads(s.findings_json))
1037
+ except json.JSONDecodeError:
1038
+ pass
1039
+
1040
+ cross_refs = json.loads(inv.cross_references) if inv.cross_references else []
1041
+
1042
+ report = BellingcatReport(
1043
+ query=inv.original_query,
1044
+ investigation_id=inv.investigation_id,
1045
+ target_type=inv.target_type,
1046
+ target_value=inv.target_value,
1047
+ )
1048
+ report.add_findings(all_findings)
1049
+ report.add_cross_references(cross_refs)
1050
+ report.hops = inv.total_hops
1051
+
1052
+ # Save to cases directory
1053
+ cases_dir = Path("cases/exports")
1054
+ cases_dir.mkdir(parents=True, exist_ok=True)
1055
+
1056
+ fmt = export.format.lower()
1057
+ if fmt == "json":
1058
+ path = report.to_json(cases_dir / f"{inv.investigation_id}.json")
1059
+ elif fmt == "stix":
1060
+ path = report.to_stix(cases_dir / f"{inv.investigation_id}_stix.json")
1061
+ elif fmt == "misp":
1062
+ path = report.to_misp(cases_dir / f"{inv.investigation_id}_misp.json")
1063
+ elif fmt == "pdf":
1064
+ path = report.to_pdf(cases_dir / f"{inv.investigation_id}.pdf")
1065
+ elif fmt == "markdown":
1066
+ path = cases_dir / f"{inv.investigation_id}.md"
1067
+ path.write_text(report.to_markdown())
1068
+ else:
1069
+ return JSONResponse(status_code=400, content={"error": f"Unknown format: {fmt}"})
1070
+
1071
+ return {
1072
+ "format": fmt,
1073
+ "path": str(path),
1074
+ "size_bytes": path.stat().st_size if path.exists() else 0,
1075
+ "findings": len(all_findings),
1076
+ "verifiability": f"{report._verifiability():.0%}",
1077
+ }
1078
+
1079
+
1080
+ # ── Search ───────────────────────────────────────────────────
1081
+
1082
+ @app.get("/api/search")
1083
+ async def search_investigations(
1084
+ q: str = Query("", description="Search query"),
1085
+ agent: str = Query("", description="Filter by agent role"),
1086
+ tier: str = Query("", description="Filter by evidence tier"),
1087
+ limit: int = Query(20, description="Max results"),
1088
+ ):
1089
+ """Full-text search across all investigations."""
1090
+ if not q:
1091
+ return {"results": [], "total": 0, "query": q}
1092
+
1093
+ from src.watson.search import get_search
1094
+ s = get_search()
1095
+ results = s.search(q, limit=limit, agent_filter=agent, tier_filter=tier)
1096
+
1097
+ return {
1098
+ "query": q,
1099
+ "total": len(results),
1100
+ "results": results,
1101
+ "filters": {"agent": agent, "tier": tier},
1102
+ }
1103
+
1104
+
1105
+ @app.get("/api/search/investigations")
1106
+ async def search_inv_list(
1107
+ q: str = Query("", description="Search investigations"),
1108
+ limit: int = Query(10),
1109
+ ):
1110
+ """Search investigation-level metadata."""
1111
+ if not q:
1112
+ return {"results": [], "total": 0}
1113
+
1114
+ from src.watson.search import get_search
1115
+ s = get_search()
1116
+ results = s.search_investigations(q, limit=limit)
1117
+ return {"query": q, "total": len(results), "results": results}
1118
+
1119
+
1120
+ @app.get("/api/search/stats")
1121
+ async def search_stats():
1122
+ """Get search index statistics."""
1123
+ from src.watson.search import get_search
1124
+ return get_search().get_stats()
1125
+
1126
+
1127
+ # ═══════════════════════════════════════════════════════════════
1128
+ # API Key Settings — user-managed keys for tools
1129
+ # ═══════════════════════════════════════════════════════════════
1130
+
1131
+ from pydantic import BaseModel
1132
+
1133
+
1134
+ class SetKeyRequest(BaseModel):
1135
+ slug: str
1136
+ value: str
1137
+
1138
+
1139
+ class SaveRequest(BaseModel):
1140
+ consent_publish: bool = False
1141
+
1142
+
1143
+ @app.post("/api/cases/{case_id}/save")
1144
+ async def save_case(case_id: str, req: SaveRequest = SaveRequest()):
1145
+ """Save a pending investigation to disk + knowledge graph + optionally MCP.
1146
+
1147
+ With auto-save, the case is already on disk. This endpoint handles:
1148
+ - pending: save + graph + optional MCP publish
1149
+ - already saved: just MCP publish if requested
1150
+ """
1151
+ from src.watson.orchestration import get_engine
1152
+ from pathlib import Path
1153
+ engine = get_engine()
1154
+
1155
+ report = None
1156
+ already_saved = False
1157
+ matches: list = [] # populated when case found on disk
1158
+
1159
+ if hasattr(engine, '_pending_reports') and case_id in engine._pending_reports:
1160
+ report = engine._pending_reports[case_id]
1161
+ else:
1162
+ # Check if already saved to disk (auto-save mode)
1163
+ # _save_case writes "{case_id}_{date}.md" — glob for it
1164
+ cases_dir = Path.home() / "watson-cases"
1165
+ matches = list(cases_dir.glob(f"{case_id}_*.md")) if cases_dir.exists() else []
1166
+ if matches:
1167
+ already_saved = True
1168
+ else:
1169
+ raise HTTPException(404,
1170
+ f"Case {case_id} not found. It may not exist or the server was restarted. "
1171
+ f"Run the investigation again.")
1172
+
1173
+ if report is not None:
1174
+ # Normal flow: save pending report
1175
+ engine._save_case(report)
1176
+ engine._update_graph(report)
1177
+ del engine._pending_reports[case_id]
1178
+
1179
+ # Publish to MCP only with explicit consent
1180
+ published = False
1181
+ publish_error = None
1182
+ if req.consent_publish:
1183
+ if report:
1184
+ ok, err = _publish_to_mcp(report)
1185
+ elif already_saved:
1186
+ # Reconstruct from disk for publishing
1187
+ # _save_case writes "{case_id}_{date}.md" — use first glob match
1188
+ case_path = matches[0]
1189
+ text = case_path.read_text()
1190
+ from src.watson.orchestration.engine import OrchestrationEngine
1191
+ entities = OrchestrationEngine._extract_entities_from_text(text)
1192
+ if entities:
1193
+ import httpx
1194
+ import re as _re
1195
+ target = ""
1196
+ tm = _re.search(r"\*\*Target:\*\*\s*(.+)", text)
1197
+ if tm: target = tm.group(1).strip()
1198
+ payload = {
1199
+ "case_id": case_id,
1200
+ "target": target,
1201
+ "target_type": "person",
1202
+ "findings_count": 0,
1203
+ "confirmed_count": 0,
1204
+ "verifiability": "",
1205
+ "date": "",
1206
+ "entities": [{
1207
+ "value": e["value"], "type": e["type"],
1208
+ "source": "", "tier": "PROBABLE", "case_id": case_id,
1209
+ } for e in entities],
1210
+ }
1211
+ resp = httpx.post(
1212
+ f"{MCP_SERVER_URL}/api/ingest", json=payload, timeout=10,
1213
+ headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {},
1214
+ )
1215
+ ok = resp.status_code == 200
1216
+ err = None if ok else f"MCP server returned {resp.status_code}"
1217
+ else:
1218
+ ok, err = False, "No entities to publish"
1219
+ else:
1220
+ ok, err = False, "No report data available"
1221
+ published = ok
1222
+ publish_error = err
1223
+
1224
+ return {
1225
+ "case_id": case_id,
1226
+ "target": report.query if report else "saved case",
1227
+ "findings": len(report.findings) if report else 0,
1228
+ "verifiability": f"{report.verifiability_score:.0%}" if report else "",
1229
+ "status": "saved" if not already_saved else "already_saved",
1230
+ "published": published,
1231
+ "publish_error": publish_error,
1232
+ }
1233
+
1234
+
1235
+ def _publish_to_mcp(report):
1236
+ """Publish case entities to the MCP knowledge graph server with full case data.
1237
+
1238
+ Uses the engine's entity extraction to get real typed entities, not finding titles.
1239
+ Returns (success: bool, error: str | None).
1240
+ """
1241
+ try:
1242
+ import httpx
1243
+ # Use the engine's entity extraction — not finding titles
1244
+ from src.watson.orchestration.engine import OrchestrationEngine
1245
+
1246
+ # Build combined text from all finding titles and bodies
1247
+ text = " ".join(f"{f.title} {f.body}" for f in report.findings)
1248
+ entities = OrchestrationEngine._extract_entities_from_text(text)
1249
+ if not entities:
1250
+ return False, "No typed entities to publish"
1251
+
1252
+ payload = {
1253
+ "case_id": report.case_id,
1254
+ "target": report.query,
1255
+ "target_type": getattr(report, "target_type", "person"),
1256
+ "findings_count": len(report.findings),
1257
+ "confirmed_count": sum(1 for f in report.findings if f.tier == "CONFIRMED"),
1258
+ "verifiability": f"{report.verifiability_score:.0%}",
1259
+ "date": getattr(report, "timestamp", ""),
1260
+ "entities": [{
1261
+ "value": e.get("value", ""),
1262
+ "type": e.get("type", "unknown"),
1263
+ "source": e.get("source", ""),
1264
+ "tier": e.get("tier", "PROBABLE"),
1265
+ "case_id": report.case_id,
1266
+ } for e in entities],
1267
+ }
1268
+
1269
+ resp = httpx.post(
1270
+ f"{MCP_SERVER_URL}/api/ingest", json=payload, timeout=10,
1271
+ headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {},
1272
+ )
1273
+ if resp.status_code == 200:
1274
+ data = resp.json()
1275
+ ingested = data.get("entities_ingested", 0)
1276
+ rejected = data.get("entities_rejected", 0)
1277
+ logger.info("mcp_published", extra={
1278
+ "case_id": report.case_id,
1279
+ "ingested": ingested,
1280
+ "rejected": rejected,
1281
+ })
1282
+ return True, None
1283
+ elif resp.status_code == 401:
1284
+ return False, "Invalid MCP API key"
1285
+ elif resp.status_code == 503:
1286
+ return False, "MCP server not configured (no API key set)"
1287
+ else:
1288
+ return False, f"MCP server returned {resp.status_code}"
1289
+ except Exception as e:
1290
+ logger.warning("mcp_publish_error: %s", e)
1291
+ return False, str(e)
1292
+
1293
+
1294
+ @app.post("/api/cases/{case_id}/publish")
1295
+ async def publish_case(case_id: str):
1296
+ """Publish an already-saved case to the MCP community graph.
1297
+
1298
+ Uses the engine's entity extraction for properly-typed entities.
1299
+ """
1300
+ from pathlib import Path
1301
+ case_path = Path.home() / "watson-cases" / f"{case_id}.md"
1302
+ if not case_path.exists():
1303
+ raise HTTPException(404, f"Case {case_id} not found in archives")
1304
+
1305
+ # Reconstruct metadata and extract typed entities
1306
+ import re
1307
+ text = case_path.read_text()
1308
+ target = ""
1309
+ target_type = ""
1310
+ tm = re.search(r"\*\*Target:\*\*\s*(.+)", text)
1311
+ ttm = re.search(r"\*\*Target Type:\*\*\s*(.+)", text)
1312
+ fm = re.search(r"\*\*Findings:\*\*\s*(\d+)", text)
1313
+ cm = re.search(r"(\d+)\s+CONFIRMED", text)
1314
+ vm = re.search(r"\*\*Verifiability:\*\*\s*(.+)", text)
1315
+ dm = re.search(r"\*\*Date:\*\*\s*(.+)", text)
1316
+
1317
+ if tm: target = tm.group(1).strip()
1318
+ if ttm: target_type = ttm.group(1).strip()
1319
+ findings_count = int(fm.group(1)) if fm else 0
1320
+ confirmed_count = int(cm.group(1)) if cm else 0
1321
+ verifiability = vm.group(1).strip() if vm else ""
1322
+ date_str = dm.group(1).strip() if dm else ""
1323
+
1324
+ # Use engine's entity extraction — produces properly-typed entities
1325
+ from src.watson.orchestration.engine import OrchestrationEngine
1326
+ entities = OrchestrationEngine._extract_entities_from_text(text)
1327
+
1328
+ if not entities:
1329
+ return {"status": "no_entities", "case_id": case_id}
1330
+
1331
+ if not MCP_API_KEY and "localhost" not in MCP_SERVER_URL:
1332
+ raise HTTPException(400, "No MCP API key configured. Set MCP_API_KEY in env or run onboarding.")
1333
+
1334
+ try:
1335
+ import httpx
1336
+ payload = {
1337
+ "case_id": case_id,
1338
+ "target": target,
1339
+ "target_type": target_type,
1340
+ "findings_count": findings_count,
1341
+ "confirmed_count": confirmed_count,
1342
+ "verifiability": verifiability,
1343
+ "date": date_str,
1344
+ "entities": [{
1345
+ "value": e["value"],
1346
+ "type": e["type"],
1347
+ "source": "",
1348
+ "tier": "PROBABLE",
1349
+ "case_id": case_id,
1350
+ } for e in entities],
1351
+ }
1352
+ resp = httpx.post(f"{MCP_SERVER_URL}/api/ingest", json=payload, timeout=10,
1353
+ headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {})
1354
+ if resp.status_code == 200:
1355
+ data = resp.json()
1356
+ return {
1357
+ "status": "published",
1358
+ "case_id": case_id,
1359
+ "entities_sent": len(entities),
1360
+ "entities_ingested": data.get("entities_ingested", 0),
1361
+ "entities_rejected": data.get("entities_rejected", 0),
1362
+ }
1363
+ elif resp.status_code == 401:
1364
+ raise HTTPException(401, "Invalid MCP API key")
1365
+ elif resp.status_code == 503:
1366
+ raise HTTPException(503, "MCP server not configured")
1367
+ raise HTTPException(502, f"MCP server returned {resp.status_code}")
1368
+ except httpx.ConnectError:
1369
+ raise HTTPException(503, f"MCP server not reachable at {MCP_SERVER_URL}")
1370
+
1371
+
1372
+ @app.get("/api/graph/status")
1373
+ async def graph_status():
1374
+ """Check connection to the MCP community knowledge graph.
1375
+
1376
+ Returns connection status, whether a key is configured, and graph stats.
1377
+ Frontend uses this to show the connection indicator next to the publish checkbox.
1378
+ """
1379
+ is_local = "localhost" in MCP_SERVER_URL or "127.0.0.1" in MCP_SERVER_URL
1380
+ has_key = bool(MCP_API_KEY)
1381
+
1382
+ status = {
1383
+ "mcp_url": MCP_SERVER_URL,
1384
+ "configured": has_key or is_local,
1385
+ "reason": "",
1386
+ "stats": None,
1387
+ }
1388
+
1389
+ if is_local:
1390
+ status["reason"] = "Local graph — always available"
1391
+ elif not has_key:
1392
+ status["reason"] = "No MCP API key configured"
1393
+ return status
1394
+
1395
+ try:
1396
+ import httpx
1397
+ resp = httpx.get(
1398
+ f"{MCP_SERVER_URL}/api/stats",
1399
+ timeout=3,
1400
+ headers={"X-API-Key": MCP_API_KEY},
1401
+ )
1402
+ if resp.status_code == 200:
1403
+ status["stats"] = resp.json()
1404
+ status["connected"] = True
1405
+ else:
1406
+ status["connected"] = False
1407
+ status["reason"] = f"MCP server returned {resp.status_code}"
1408
+ except Exception as e:
1409
+ status["connected"] = False
1410
+ status["reason"] = str(e)
1411
+
1412
+ return status
1413
+
1414
+
1415
+ @app.get("/api/public/search")
1416
+ async def search_public_intel(q: str):
1417
+ """Search the community knowledge graph for prior intelligence on a target."""
1418
+ try:
1419
+ import httpx
1420
+ resp = httpx.get(f"{MCP_SERVER_URL}/api/search", params={"q": q, "limit": 10}, timeout=5)
1421
+ if resp.status_code == 200:
1422
+ data = resp.json()
1423
+ return {
1424
+ "query": q,
1425
+ "results": data.get("results", []),
1426
+ "count": data.get("count", 0),
1427
+ }
1428
+ return {"query": q, "results": [], "count": 0, "error": f"MCP returned {resp.status_code}"}
1429
+ except Exception as e:
1430
+ return {"query": q, "results": [], "count": 0, "mcp_offline": True}
1431
+
1432
+
1433
+ @app.get("/api/public/stats")
1434
+ async def public_intel_stats():
1435
+ """Get community knowledge graph stats."""
1436
+ try:
1437
+ import httpx
1438
+ resp = httpx.get(f"{MCP_SERVER_URL}/api/stats", timeout=5)
1439
+ if resp.status_code == 200:
1440
+ return resp.json()
1441
+ return {"error": f"MCP returned {resp.status_code}"}
1442
+ except Exception as e:
1443
+ return {"error": str(e), "mcp_offline": True}
1444
+
1445
+
1446
+ @app.get("/api/cases")
1447
+ async def list_cases(limit: int = 20, offset: int = 0):
1448
+ """List all saved investigation cases."""
1449
+ from pathlib import Path
1450
+ import re
1451
+
1452
+ cases_dir = Path.home() / "watson-cases"
1453
+ cases = []
1454
+ for f in sorted(cases_dir.glob("CASE-*.md"), key=lambda x: x.stat().st_mtime, reverse=True):
1455
+ if not f.name.startswith("CASE-"):
1456
+ continue
1457
+ text = f.read_text()
1458
+ target = ""
1459
+ target_type = ""
1460
+ findings = 0
1461
+ confirmed = 0
1462
+ verifiability = ""
1463
+ date_match = re.search(r"\*\*Date:\*\*\s*(.+)", text)
1464
+ target_match = re.search(r"\*\*Target:\*\*\s*(.+)", text)
1465
+ type_match = re.search(r"\*\*Target Type:\*\*\s*(.+)", text)
1466
+ findings_match = re.search(r"\*\*Findings:\*\*\s*(\d+)", text)
1467
+ confirmed_match = re.search(r"(\d+)\s+CONFIRMED", text)
1468
+ verif_match = re.search(r"\*\*Verifiability:\*\*\s*(.+)", text)
1469
+
1470
+ if target_match: target = target_match.group(1).strip()
1471
+ if type_match: target_type = type_match.group(1).strip()
1472
+ if findings_match: findings = int(findings_match.group(1))
1473
+ if confirmed_match: confirmed = int(confirmed_match.group(1))
1474
+ if verif_match: verifiability = verif_match.group(1).strip()
1475
+ date_str = date_match.group(1).strip() if date_match else ""
1476
+
1477
+ cases.append({
1478
+ "id": f.stem,
1479
+ "target": target,
1480
+ "target_type": target_type,
1481
+ "date": date_str,
1482
+ "findings": findings,
1483
+ "confirmed": confirmed,
1484
+ "verifiability": verifiability,
1485
+ "size": f.stat().st_size,
1486
+ })
1487
+
1488
+ return {
1489
+ "cases": cases[offset:offset+limit],
1490
+ "total": len(cases),
1491
+ }
1492
+
1493
+
1494
+ @app.get("/api/cases/{case_id}")
1495
+ async def get_case(case_id: str):
1496
+ """Get a specific case markdown."""
1497
+ from pathlib import Path
1498
+ case_path = Path.home() / "watson-cases" / f"{case_id}.md"
1499
+ if case_path.exists():
1500
+ from fastapi.responses import PlainTextResponse
1501
+ return PlainTextResponse(case_path.read_text(), media_type="text/markdown")
1502
+ raise HTTPException(404, f"Case {case_id} not found")
1503
+
1504
+
1505
+ @app.get("/api/settings/keys")
1506
+ async def get_api_keys():
1507
+ """List all configurable API keys with their status (values masked)."""
1508
+ from watson.api_keys import list_keys
1509
+ return {"keys": list_keys()}
1510
+
1511
+
1512
+ @app.post("/api/settings/keys")
1513
+ async def set_api_key(req: SetKeyRequest):
1514
+ """Save an API key for a tool."""
1515
+ from watson.api_keys import set_key
1516
+ set_key(req.slug, req.value)
1517
+ return {"status": "ok", "slug": req.slug, "configured": bool(req.value)}
1518
+
1519
+
1520
+ @app.delete("/api/settings/keys/{slug}")
1521
+ async def delete_api_key(slug: str):
1522
+ """Remove an API key."""
1523
+ from watson.api_keys import delete_key
1524
+ delete_key(slug)
1525
+ return {"status": "ok", "slug": slug, "configured": False}
1526
+
1527
+
1528
+ # ── Enterprise Exports ──────────────────────────────────────────
1529
+
1530
+ @app.get("/api/export/stix/{case_id}")
1531
+ async def export_stix_endpoint(case_id: str):
1532
+ """Export a completed investigation as STIX 2.1 JSON bundle."""
1533
+ from pathlib import Path
1534
+ case_path = Path.home() / "watson-cases" / f"{case_id}.md"
1535
+ if not case_path.exists():
1536
+ # Try case_id with date suffix (CASE-XXX_YYYY-MM-DD)
1537
+ matches = list((Path.home() / "watson-cases").glob(f"{case_id}*.md"))
1538
+ if matches:
1539
+ case_path = matches[0]
1540
+ else:
1541
+ raise HTTPException(404, f"Case {case_id} not found")
1542
+
1543
+ try:
1544
+ from watson.serializers.stix import export_stix
1545
+ # Parse minimal info from case filename
1546
+ import re
1547
+ content = case_path.read_text()
1548
+ query_match = re.search(r'\*\*Target:\*\*\s*(.+?)$', content, re.MULTILINE)
1549
+ target_match = re.search(r'\*\*Target Type:\*\*\s*(.+?)$', content, re.MULTILINE)
1550
+ query = query_match.group(1).strip() if query_match else case_id
1551
+ target_type = target_match.group(1).strip() if target_match else "unknown"
1552
+
1553
+ # Build minimal findings from markdown
1554
+ findings = _parse_findings_from_markdown(content)
1555
+
1556
+ bundle, _ = export_stix(
1557
+ query=query,
1558
+ case_id=case_id,
1559
+ target_type=target_type,
1560
+ findings=findings,
1561
+ )
1562
+ return JSONResponse(content=bundle, media_type="application/json")
1563
+ except Exception as e:
1564
+ raise HTTPException(500, f"STIX export failed: {e}")
1565
+
1566
+
1567
+ @app.get("/api/opsec/stats")
1568
+ async def opsec_stats():
1569
+ """OpSec proxy firewall statistics."""
1570
+ try:
1571
+ from watson.opsec import get_opsec_client
1572
+ client = get_opsec_client()
1573
+ return client.stats()
1574
+ except Exception as e:
1575
+ return {"status": "unavailable", "error": str(e)}
1576
+
1577
+
1578
+ # ── SPA fallback — serve React app for all non-API routes ──────────
1579
+
1580
+ @app.get("/{full_path:path}", response_class=HTMLResponse)
1581
+ async def spa_fallback(full_path: str):
1582
+ """Serve the React SPA for all client-side routes.
1583
+
1584
+ Every URL that isn't an explicit API endpoint or static file
1585
+ returns index.html so React Router can handle it client-side.
1586
+ """
1587
+ # Don't intercept API routes (shouldn't reach here if registered first, but safety)
1588
+ if full_path.startswith("api/"):
1589
+ raise HTTPException(404)
1590
+ index = STATIC_DIR / "index.html"
1591
+ if index.exists():
1592
+ return HTMLResponse(content=index.read_text(), headers=_NO_CACHE)
1593
+ return HTMLResponse(
1594
+ "<h1>Watson — run <code>cd frontend && npm run build</code></h1>",
1595
+ status_code=404,
1596
+ )
1597
+
1598
+
1599
+ def _parse_findings_from_markdown(content: str) -> list:
1600
+ """Parse findings from a Watson markdown report for STIX export."""
1601
+ import re
1602
+ findings = []
1603
+
1604
+ # Find "Key Findings" section
1605
+ key_section = re.search(r'## Key Findings\s*\n(.*?)(?=\n## |\Z)', content, re.DOTALL)
1606
+ if not key_section:
1607
+ return findings
1608
+
1609
+ # Parse individual finding lines (bullet points with tier icons)
1610
+ finding_pattern = re.compile(
1611
+ r'-\s*(?:🟢|🟡|🟠|🔴|⚪)\s*\*\*(.+?)\*\*\s*\[(.+?)\]\s*\((\d+)% confidence\)\s*\n'
1612
+ r'\s*(.+?)(?=\n\s*(?:Source:|$))(?:\n\s*Source:\s*(.+?))?(?=\n- |\n## |\Z)',
1613
+ re.DOTALL,
1614
+ )
1615
+
1616
+ for m in finding_pattern.finditer(key_section.group(1)):
1617
+ title = m.group(1).strip()
1618
+ src_tier = m.group(2).strip()
1619
+ confidence_str = m.group(3)
1620
+ desc = m.group(4).strip()
1621
+ source_url = m.group(5).strip() if m.group(5) else ""
1622
+
1623
+ # Map source tier to Watson tier
1624
+ tier_map = {"PRIMARY": "CONFIRMED", "SECONDARY": "PROBABLE",
1625
+ "TERTIARY": "POSSIBLE"}
1626
+ tier = tier_map.get(src_tier, "POSSIBLE")
1627
+
1628
+ findings.append(SimpleFinding(
1629
+ title=title,
1630
+ description=desc[:500],
1631
+ tier=tier,
1632
+ source_url=source_url,
1633
+ source_type="osint",
1634
+ confidence=int(confidence_str) / 100,
1635
+ ))
1636
+
1637
+ return findings
1638
+
1639
+
1640
+ class SimpleFinding:
1641
+ """Minimal finding object for STIX export from markdown."""
1642
+ def __init__(self, title, description, tier, source_url, source_type, confidence):
1643
+ self.id = str(abs(hash(title)) % (10**12))
1644
+ self.title = title
1645
+ self.description = description
1646
+ self.tier = tier
1647
+ self.source_url = source_url
1648
+ self.source_type = source_type
1649
+ self.confidence = confidence
1650
+ self.entities = []