osintengine 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osintengine-1.0.2.dist-info/METADATA +311 -0
- osintengine-1.0.2.dist-info/RECORD +103 -0
- osintengine-1.0.2.dist-info/WHEEL +5 -0
- osintengine-1.0.2.dist-info/entry_points.txt +2 -0
- osintengine-1.0.2.dist-info/licenses/LICENSE +202 -0
- osintengine-1.0.2.dist-info/top_level.txt +2 -0
- src/watson/__init__.py +10 -0
- src/watson/agent/__init__.py +96 -0
- src/watson/agents/__init__.py +101 -0
- src/watson/agents/protocol.py +196 -0
- src/watson/auth/__init__.py +68 -0
- src/watson/auth/store.py +145 -0
- src/watson/conversation.py +68 -0
- src/watson/core/__init__.py +0 -0
- src/watson/core/models.py +96 -0
- src/watson/ethics.py +205 -0
- src/watson/exports.py +182 -0
- src/watson/graph/__init__.py +69 -0
- src/watson/graph/entities.py +207 -0
- src/watson/graph/graph.py +489 -0
- src/watson/graph/osint_framework.py +266 -0
- src/watson/graph/relationships.py +116 -0
- src/watson/graph/transforms.py +787 -0
- src/watson/infra/__init__.py +43 -0
- src/watson/infra/cache.py +171 -0
- src/watson/infra/ratelimit.py +125 -0
- src/watson/infra/resilience.py +239 -0
- src/watson/infra/retry.py +186 -0
- src/watson/metrics.py +128 -0
- src/watson/opsec/__init__.py +290 -0
- src/watson/orchestration/__init__.py +23 -0
- src/watson/orchestration/engine.py +5587 -0
- src/watson/orchestration/executor.py +96 -0
- src/watson/orchestration/intent.py +139 -0
- src/watson/orchestration/intent_classifier.py +45 -0
- src/watson/orchestration/llm_config.py +160 -0
- src/watson/orchestration/resolution.py +555 -0
- src/watson/orchestration/scraper.py +94 -0
- src/watson/orchestration/synthesis.py +726 -0
- src/watson/orchestration/target_profile.py +748 -0
- src/watson/persistence/__init__.py +18 -0
- src/watson/persistence/models.py +161 -0
- src/watson/persistence/store.py +230 -0
- src/watson/pipeline/__init__.py +16 -0
- src/watson/pipeline/pre_synthesis.py +621 -0
- src/watson/search.py +103 -0
- src/watson/serializers/stix.py +451 -0
- src/watson/tools/__init__.py +31 -0
- src/watson/tools/base.py +97 -0
- src/watson/tools/blockchain.py +359 -0
- src/watson/tools/captcha.py +276 -0
- src/watson/tools/conflict.py +107 -0
- src/watson/tools/corporate.py +287 -0
- src/watson/tools/darkweb.py +144 -0
- src/watson/tools/geolocation.py +347 -0
- src/watson/tools/image_video.py +108 -0
- src/watson/tools/marinetraffic.py +142 -0
- src/watson/tools/people.py +476 -0
- src/watson/tools/registry.py +56 -0
- src/watson/tools/satellite.py +118 -0
- src/watson/tools/scraper.py +745 -0
- src/watson/tools/shodan.py +129 -0
- src/watson/tools/social_media.py +160 -0
- src/watson/tools/websites.py +291 -0
- src/watson/tools/wikidata.py +398 -0
- src/watson/utils/__init__.py +1 -0
- src/watson/utils/helpers.py +49 -0
- src/watson/utils/http.py +119 -0
- src/watson/verification/__init__.py +245 -0
- watson/__init__.py +6 -0
- watson/agents/__init__.py +20 -0
- watson/agents/base.py +103 -0
- watson/agents/direct.py +225 -0
- watson/agents/hermes.py +275 -0
- watson/agents/hermes_mcp.py +292 -0
- watson/api_keys.py +176 -0
- watson/browser_scraper.py +481 -0
- watson/cli.py +836 -0
- watson/crossref.py +175 -0
- watson/ethics.py +20 -0
- watson/graph.py +406 -0
- watson/mcp_server.py +601 -0
- watson/memory.py +552 -0
- watson/neo4j_graph.py +124 -0
- watson/opsec/__init__.py +307 -0
- watson/reporter.py +537 -0
- watson/scheduler.py +486 -0
- watson/serializers/stix.py +451 -0
- watson/terminal.py +410 -0
- watson/toolkit.py +252 -0
- watson/toolkit_api.py +347 -0
- watson/toolkit_automation.py +95 -0
- watson/toolkit_registry.py +345 -0
- watson/verification/__init__.py +251 -0
- watson/web/__init__.py +1 -0
- watson/web/app.py +1650 -0
- watson/web/middleware.py +301 -0
- watson/web/static/assets/index-B7hPOc0z.js +278 -0
- watson/web/static/assets/index-CJ6FP8Mp.css +1 -0
- watson/web/static/assets/watson-logo-Dk9tawHb.png +0 -0
- watson/web/static/index.html +14 -0
- watson/web/templates/chat.html +2 -0
- watson/web/templates/investigation-map.html +429 -0
watson/web/app.py
ADDED
|
@@ -0,0 +1,1650 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Watson Web — FastAPI application (Phase 1 rewrite).
|
|
3
|
+
|
|
4
|
+
Routes:
|
|
5
|
+
GET / — Chat UI
|
|
6
|
+
GET /health — Health check
|
|
7
|
+
POST /api/chat — Tool-calling chat (streaming SSE)
|
|
8
|
+
POST /api/agent/investigate — Agent investigation
|
|
9
|
+
GET /api/agent/stream/<id> — Investigation SSE stream
|
|
10
|
+
POST /api/agent/chat/stream — Chat SSE (with tools)
|
|
11
|
+
+ memory, scheduler, bellingcat, export, knowledge, terminal
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import threading
|
|
17
|
+
import asyncio
|
|
18
|
+
import json
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import sys
|
|
23
|
+
import time
|
|
24
|
+
import uuid
|
|
25
|
+
from datetime import datetime, timezone
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Optional
|
|
28
|
+
|
|
29
|
+
# Ensure both src/ and project root are on path for imports
|
|
30
|
+
# src/watson/* → src.watson.agent, src.watson.auth, ...
|
|
31
|
+
# watson/* → watson.reporter, watson.web.middleware, ...
|
|
32
|
+
_src = Path(__file__).resolve().parent.parent.parent / "src"
|
|
33
|
+
_root = Path(__file__).resolve().parent.parent.parent
|
|
34
|
+
for p in (str(_root), str(_src)):
|
|
35
|
+
if p not in sys.path:
|
|
36
|
+
sys.path.insert(0, p)
|
|
37
|
+
|
|
38
|
+
from fastapi import FastAPI, HTTPException, Query, Request
|
|
39
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
40
|
+
from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, Response
|
|
41
|
+
from fastapi.staticfiles import StaticFiles
|
|
42
|
+
from pydantic import BaseModel
|
|
43
|
+
|
|
44
|
+
# ── App ─────────────────────────────────────────────────────────
|
|
45
|
+
|
|
46
|
+
app = FastAPI(
|
|
47
|
+
title="Watson OSINT",
|
|
48
|
+
description="The OSINT investigation engine. Evidence-based. Graph-native.",
|
|
49
|
+
version="0.3.0",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
@app.on_event("startup")
|
|
53
|
+
async def _bind_sse_loop():
|
|
54
|
+
"""Bind the event loop to SSEManager so events can be enqueued
|
|
55
|
+
from worker threads via call_soon_threadsafe."""
|
|
56
|
+
sse.set_loop(asyncio.get_running_loop())
|
|
57
|
+
|
|
58
|
+
app.add_middleware(
|
|
59
|
+
CORSMiddleware,
|
|
60
|
+
allow_origins=["*"],
|
|
61
|
+
allow_methods=["*"],
|
|
62
|
+
allow_headers=["*"],
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# Auth middleware
|
|
66
|
+
from src.watson.auth import AuthMiddleware
|
|
67
|
+
app.add_middleware(AuthMiddleware)
|
|
68
|
+
|
|
69
|
+
# ── Health + Metrics ─────────────────────────────────────────────
|
|
70
|
+
|
|
71
|
+
from src.watson.infra.cache import all_cache_stats
|
|
72
|
+
from src.watson.infra.retry import _circuits
|
|
73
|
+
from src.watson.persistence import get_store
|
|
74
|
+
from src.watson.metrics import prometheus_endpoint, track_investigation
|
|
75
|
+
|
|
76
|
+
@app.get("/api/metrics")
|
|
77
|
+
async def metrics():
|
|
78
|
+
"""Prometheus-compatible metrics endpoint."""
|
|
79
|
+
return Response(content=prometheus_endpoint(), media_type="text/plain")
|
|
80
|
+
|
|
81
|
+
@app.get("/api/health")
|
|
82
|
+
async def health():
|
|
83
|
+
"""Health check with system status."""
|
|
84
|
+
store = get_store()
|
|
85
|
+
stats = store.get_stats()
|
|
86
|
+
cache_stats = all_cache_stats()
|
|
87
|
+
open_circuits = sum(1 for cb in _circuits.values() if cb.is_open)
|
|
88
|
+
return {
|
|
89
|
+
"status": "ok",
|
|
90
|
+
"version": "0.3.0-enterprise",
|
|
91
|
+
"investigations": stats,
|
|
92
|
+
"caches": {k: {"size": v["size"], "hit_rate": f"{v.get('hits',0)/max(v.get('hits',0)+v.get('misses',1),1):.1%}"}
|
|
93
|
+
for k, v in cache_stats.items()},
|
|
94
|
+
"circuit_breakers_open": open_circuits,
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
BASE_DIR = Path(__file__).resolve().parent
|
|
98
|
+
TEMPLATES_DIR = BASE_DIR / "templates"
|
|
99
|
+
STATIC_DIR = BASE_DIR / "static"
|
|
100
|
+
if STATIC_DIR.exists():
|
|
101
|
+
app.mount("/assets", StaticFiles(directory=str(STATIC_DIR / "assets")), name="assets")
|
|
102
|
+
app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
|
103
|
+
|
|
104
|
+
# MCP community graph server URL — configurable for shared instances
|
|
105
|
+
# Priority: env var > config file > default
|
|
106
|
+
def _load_mcp_config() -> dict:
|
|
107
|
+
"""Load MCP settings from config file (env overrides)."""
|
|
108
|
+
mcp = {
|
|
109
|
+
"url": os.environ.get("WATSON_MCP_URL", ""),
|
|
110
|
+
"key": os.environ.get("MCP_API_KEY", ""),
|
|
111
|
+
}
|
|
112
|
+
try:
|
|
113
|
+
cfg_path = Path.home() / ".watson" / "config.json"
|
|
114
|
+
if cfg_path.exists():
|
|
115
|
+
cfg = json.loads(cfg_path.read_text())
|
|
116
|
+
if not mcp["url"]:
|
|
117
|
+
mcp["url"] = cfg.get("mcp_url", "")
|
|
118
|
+
if not mcp["key"]:
|
|
119
|
+
mcp["key"] = cfg.get("mcp_api_key", "")
|
|
120
|
+
except Exception:
|
|
121
|
+
pass
|
|
122
|
+
if not mcp["url"]:
|
|
123
|
+
mcp["url"] = "http://localhost:8700"
|
|
124
|
+
return mcp
|
|
125
|
+
|
|
126
|
+
_mcp = _load_mcp_config()
|
|
127
|
+
MCP_SERVER_URL = _mcp["url"]
|
|
128
|
+
MCP_API_KEY = _mcp["key"]
|
|
129
|
+
|
|
130
|
+
# ── Env loading (must be first — MCP server subprocess needs these vars) ──
|
|
131
|
+
|
|
132
|
+
def _load_env():
|
|
133
|
+
env_path = os.path.expanduser("~/.hermes/.env")
|
|
134
|
+
if not os.path.exists(env_path):
|
|
135
|
+
return
|
|
136
|
+
with open(env_path) as f:
|
|
137
|
+
for line in f:
|
|
138
|
+
if "=" in line and not line.startswith("#"):
|
|
139
|
+
k, v = line.strip().split("=", 1)
|
|
140
|
+
k, v = k.strip(), v.strip().strip('"').strip("'")
|
|
141
|
+
if k and v and k not in os.environ:
|
|
142
|
+
os.environ[k] = v
|
|
143
|
+
|
|
144
|
+
_load_env()
|
|
145
|
+
|
|
146
|
+
# ── Auto-start local MCP server ──────────────────────────────────
|
|
147
|
+
|
|
148
|
+
def _start_mcp_server():
|
|
149
|
+
"""Start the MCP knowledge graph server as a subprocess if URL is local."""
|
|
150
|
+
if "localhost" not in MCP_SERVER_URL and "127.0.0.1" not in MCP_SERVER_URL:
|
|
151
|
+
return # Remote MCP — assume it's already running
|
|
152
|
+
|
|
153
|
+
mcp_port = MCP_SERVER_URL.rsplit(":", 1)[-1]
|
|
154
|
+
import subprocess as _sp
|
|
155
|
+
import sys as _sys
|
|
156
|
+
try:
|
|
157
|
+
proc = _sp.Popen(
|
|
158
|
+
[_sys.executable, "-m", "uvicorn", "watson.mcp_server:mcp",
|
|
159
|
+
"--host", "0.0.0.0", "--port", mcp_port],
|
|
160
|
+
cwd=str(Path(__file__).resolve().parents[2]), # project root
|
|
161
|
+
env={**os.environ, "PYTHONPATH": f".:{os.environ.get('PYTHONPATH', '')}"},
|
|
162
|
+
stdout=_sp.DEVNULL, stderr=_sp.DEVNULL,
|
|
163
|
+
)
|
|
164
|
+
logging.getLogger("watson").info("mcp_server_started", extra={"port": mcp_port, "pid": proc.pid})
|
|
165
|
+
except Exception as e:
|
|
166
|
+
logging.getLogger("watson").warning("mcp_server_start_failed: %s", e)
|
|
167
|
+
|
|
168
|
+
_start_mcp_server()
|
|
169
|
+
|
|
170
|
+
# ── Enterprise middleware ─────────────────────────────────────────
|
|
171
|
+
|
|
172
|
+
from watson.web.middleware import init_app, register_task
|
|
173
|
+
|
|
174
|
+
init_app(app) # Auth, rate limiting, structured logging, tracing, graceful shutdown
|
|
175
|
+
|
|
176
|
+
logger = logging.getLogger("watson")
|
|
177
|
+
|
|
178
|
+
import datetime as _dt
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _json_default(obj):
|
|
182
|
+
"""JSON default handler: datetime → ISO, everything else → truncated str."""
|
|
183
|
+
if isinstance(obj, _dt.datetime):
|
|
184
|
+
return obj.isoformat()
|
|
185
|
+
if isinstance(obj, _dt.date):
|
|
186
|
+
return obj.isoformat()
|
|
187
|
+
# Safety net — never let str() produce huge blobs
|
|
188
|
+
s = str(obj)
|
|
189
|
+
return s[:200] if len(s) > 200 else s
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
_JSON_MAX_BYTES = 500_000 # 500 KB — drop events bigger than this
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _safe_serialize(ev_type: str, ev_data: dict) -> tuple[bool, str]:
|
|
196
|
+
"""Serialize event data to SSE-safe JSON. Returns (ok, sse_string).
|
|
197
|
+
|
|
198
|
+
ok=False means serialization failed outright (exception during json.dumps).
|
|
199
|
+
ok=True with data starting with _TRUNCATED marker means the payload was
|
|
200
|
+
oversized and has been replaced with a stub.
|
|
201
|
+
"""
|
|
202
|
+
try:
|
|
203
|
+
j = json.dumps(ev_data, default=_json_default)
|
|
204
|
+
if len(j) > _JSON_MAX_BYTES:
|
|
205
|
+
logger.warning(
|
|
206
|
+
"sse_event_too_large: type=%s size=%d — truncating", ev_type, len(j)
|
|
207
|
+
)
|
|
208
|
+
j = json.dumps({
|
|
209
|
+
"_truncated": True,
|
|
210
|
+
"_original_type": ev_type,
|
|
211
|
+
"_original_size": len(j),
|
|
212
|
+
"message": f"Event payload too large ({len(j)} bytes). "
|
|
213
|
+
f"Full data available in saved case report.",
|
|
214
|
+
})
|
|
215
|
+
return True, f"event: {ev_type}\ndata: {j}\n\n"
|
|
216
|
+
except Exception as e:
|
|
217
|
+
logger.warning("sse_serialize_failed: type=%s error=%s", ev_type, e)
|
|
218
|
+
return False, f"event: error\ndata: {json.dumps({'message': 'Serialization failed'})}\n\n"
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# ── SSE helper (thread-safe) ─────────────────────────────────────
|
|
222
|
+
|
|
223
|
+
class SSEManager:
|
|
224
|
+
"""Per-client SSE event queues — thread-safe via call_soon_threadsafe.
|
|
225
|
+
|
|
226
|
+
asyncio.Queue produces events correctly when enqueued from the event loop
|
|
227
|
+
thread. Investigations run in a worker thread (asyncio.to_thread), so we
|
|
228
|
+
use loop.call_soon_threadsafe() to safely enqueue from any thread.
|
|
229
|
+
|
|
230
|
+
Previous approaches that failed:
|
|
231
|
+
- asyncio.Queue + put_nowait(): race condition corrupts the queue
|
|
232
|
+
- queue.Queue + run_in_executor(get): leaked threads poison event delivery
|
|
233
|
+
(stale blocked threads capture events meant for the active consumer)
|
|
234
|
+
"""
|
|
235
|
+
def __init__(self):
|
|
236
|
+
self._queues: dict[str, asyncio.Queue] = {}
|
|
237
|
+
self._lock = threading.Lock()
|
|
238
|
+
self._loop: asyncio.AbstractEventLoop | None = None
|
|
239
|
+
|
|
240
|
+
def set_loop(self, loop: asyncio.AbstractEventLoop):
|
|
241
|
+
self._loop = loop
|
|
242
|
+
|
|
243
|
+
def create(self, client_id: str) -> asyncio.Queue:
|
|
244
|
+
q = asyncio.Queue()
|
|
245
|
+
with self._lock:
|
|
246
|
+
self._queues[client_id] = q
|
|
247
|
+
return q
|
|
248
|
+
|
|
249
|
+
def send(self, client_id: str, event: str, data: dict):
|
|
250
|
+
with self._lock:
|
|
251
|
+
q = self._queues.get(client_id)
|
|
252
|
+
if q and self._loop:
|
|
253
|
+
# ── Pre-serialize to catch bad data before it hits the stream ──
|
|
254
|
+
ok, _pre = _safe_serialize(event, data)
|
|
255
|
+
if ok:
|
|
256
|
+
# call_soon_threadsafe queues a callback to run put_nowait
|
|
257
|
+
# in the event loop thread — the only safe way to touch asyncio.Queue
|
|
258
|
+
# from a worker thread.
|
|
259
|
+
self._loop.call_soon_threadsafe(
|
|
260
|
+
lambda q=q, item=(event, data): q.put_nowait(item)
|
|
261
|
+
)
|
|
262
|
+
else:
|
|
263
|
+
logger.warning("sse_dropped: type=%s — unserializable", event)
|
|
264
|
+
|
|
265
|
+
def remove(self, client_id: str):
|
|
266
|
+
with self._lock:
|
|
267
|
+
self._queues.pop(client_id, None)
|
|
268
|
+
|
|
269
|
+
sse = SSEManager()
|
|
270
|
+
|
|
271
|
+
# ── Models ───────────────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
class InvestigateRequest(BaseModel):
|
|
274
|
+
query: str
|
|
275
|
+
client_id: Optional[str] = None
|
|
276
|
+
image_path: Optional[str] = None
|
|
277
|
+
depth: int = 2
|
|
278
|
+
context: str = ""
|
|
279
|
+
mode: str = "deep_investigation" # "background_check" | "due_diligence" | "deep_investigation"
|
|
280
|
+
publish_to_graph: bool = False # Per-case consent: publish findings to community knowledge graph
|
|
281
|
+
|
|
282
|
+
@property
|
|
283
|
+
def safe_depth(self) -> int:
|
|
284
|
+
return max(1, min(self.depth, 5)) # Clamp 1-5
|
|
285
|
+
|
|
286
|
+
class ChatRequest(BaseModel):
|
|
287
|
+
message: str
|
|
288
|
+
findings: list = []
|
|
289
|
+
history: list = []
|
|
290
|
+
last_query: str = ""
|
|
291
|
+
|
|
292
|
+
# ── Routes ───────────────────────────────────────────────────────
|
|
293
|
+
|
|
294
|
+
_NO_CACHE = {
|
|
295
|
+
"Cache-Control": "no-cache, no-store, must-revalidate",
|
|
296
|
+
"Pragma": "no-cache",
|
|
297
|
+
"Expires": "0",
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
@app.get("/", response_class=HTMLResponse)
|
|
301
|
+
async def root():
|
|
302
|
+
"""Serve the React app (built to static/) or fall back to old HTML."""
|
|
303
|
+
index = STATIC_DIR / "index.html"
|
|
304
|
+
if index.exists():
|
|
305
|
+
return HTMLResponse(content=index.read_text(), headers=_NO_CACHE)
|
|
306
|
+
# Fallback to old template
|
|
307
|
+
path = TEMPLATES_DIR / "investigation-map.html"
|
|
308
|
+
if path.exists():
|
|
309
|
+
return HTMLResponse(content=path.read_text(), headers=_NO_CACHE)
|
|
310
|
+
return HTMLResponse("<h1>Watson — run <code>cd frontend && npm run build</code></h1>", status_code=404)
|
|
311
|
+
|
|
312
|
+
@app.get("/chat", response_class=HTMLResponse)
|
|
313
|
+
async def chat():
|
|
314
|
+
"""Serve the React app at /chat too."""
|
|
315
|
+
return await root()
|
|
316
|
+
return HTMLResponse(content=path.read_text(), headers=_NO_CACHE)
|
|
317
|
+
|
|
318
|
+
@app.get("/health")
|
|
319
|
+
async def health():
|
|
320
|
+
"""Health check with dependency verification."""
|
|
321
|
+
deps = {}
|
|
322
|
+
|
|
323
|
+
# Check DeepSeek API
|
|
324
|
+
api_key = os.environ.get("DEEPSEEK_API_KEY", "")
|
|
325
|
+
deps["deepseek_api"] = "configured" if api_key and api_key != "***" else "missing"
|
|
326
|
+
|
|
327
|
+
# Check memory DB
|
|
328
|
+
try:
|
|
329
|
+
from watson.memory import memory as mem
|
|
330
|
+
stats = mem.stats()
|
|
331
|
+
deps["memory_db"] = {"status": "ok", "investigations": stats.get("investigations", 0)}
|
|
332
|
+
except Exception:
|
|
333
|
+
deps["memory_db"] = {"status": "degraded"}
|
|
334
|
+
|
|
335
|
+
# Check knowledge graph
|
|
336
|
+
try:
|
|
337
|
+
from watson.neo4j_graph import NEO4J_AVAILABLE
|
|
338
|
+
deps["neo4j"] = "available" if NEO4J_AVAILABLE else "fallback_json"
|
|
339
|
+
except Exception:
|
|
340
|
+
deps["neo4j"] = "unknown"
|
|
341
|
+
|
|
342
|
+
all_ok = all(
|
|
343
|
+
v not in ("missing",) and (isinstance(v, dict) and v.get("status") != "degraded") or isinstance(v, str) and v not in ("missing",)
|
|
344
|
+
for v in deps.values()
|
|
345
|
+
) or True # Don't fail health for missing configs in dev
|
|
346
|
+
|
|
347
|
+
return {
|
|
348
|
+
"status": "ok",
|
|
349
|
+
"version": "1.0.0",
|
|
350
|
+
"server": "FastAPI/uvicorn",
|
|
351
|
+
"dependencies": deps,
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
@app.get("/api/tools")
|
|
355
|
+
async def api_tools():
|
|
356
|
+
return {
|
|
357
|
+
"total": 12,
|
|
358
|
+
"categories": [
|
|
359
|
+
{"category": "toolkit", "count": 1, "tools": [{"name": "osint-toolkit", "description": "338 OSINT investigation tools", "free": True}]},
|
|
360
|
+
{"category": "corporate", "count": 1, "tools": [{"name": "corporate-finance", "description": "Company registries", "free": True}]},
|
|
361
|
+
{"category": "websites", "count": 2, "tools": [{"name": "websites-domains", "description": "Domain investigation", "free": True}, {"name": "browser-automation", "description": "Headless browser", "free": True}]},
|
|
362
|
+
{"category": "social_media", "count": 1, "tools": [{"name": "social-media", "description": "Social media search", "free": True}]},
|
|
363
|
+
{"category": "people", "count": 2, "tools": [{"name": "people-search", "description": "Person lookup", "free": True}, {"name": "scraper", "description": "Wiki/OpenSanctions", "free": True}]},
|
|
364
|
+
],
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
# ── Agent Investigation ──────────────────────────────────────────
|
|
368
|
+
|
|
369
|
+
@app.post("/api/agent/investigate")
|
|
370
|
+
async def agent_investigate(req: InvestigateRequest):
|
|
371
|
+
"""Start a Watson v1 investigation. Returns client_id for SSE stream."""
|
|
372
|
+
import gc
|
|
373
|
+
gc.collect()
|
|
374
|
+
|
|
375
|
+
from src.watson.orchestration import get_engine
|
|
376
|
+
|
|
377
|
+
client_id = f"agent-{uuid.uuid4().hex[:8]}"
|
|
378
|
+
q = sse.create(client_id)
|
|
379
|
+
|
|
380
|
+
async def run():
|
|
381
|
+
def push(event_type, data):
|
|
382
|
+
sse.send(client_id, event_type, data)
|
|
383
|
+
# ── Mode-based hard timeout — prevents hung investigations from blocking server ──
|
|
384
|
+
_mode_timeouts = {
|
|
385
|
+
"background_check": 120,
|
|
386
|
+
"due_diligence": 420,
|
|
387
|
+
"deep_investigation": 900,
|
|
388
|
+
"twin_connection": 300,
|
|
389
|
+
}
|
|
390
|
+
_hard_timeout = _mode_timeouts.get(req.mode, 600)
|
|
391
|
+
try:
|
|
392
|
+
engine = get_engine()
|
|
393
|
+
# Register interrupt queue for interactive steering
|
|
394
|
+
engine.register_interrupt_queue(client_id)
|
|
395
|
+
# ── Run investigation in a dedicated thread so CPU-bound loops
|
|
396
|
+
# don't block the uvicorn event loop. The asyncio.wait_for timeout
|
|
397
|
+
# will fire even if the thread hangs. ──
|
|
398
|
+
def _run_sync():
|
|
399
|
+
loop = asyncio.new_event_loop()
|
|
400
|
+
asyncio.set_event_loop(loop)
|
|
401
|
+
try:
|
|
402
|
+
return loop.run_until_complete(engine.investigate(
|
|
403
|
+
query=req.query,
|
|
404
|
+
focus=req.context,
|
|
405
|
+
on_event=push,
|
|
406
|
+
mode=req.mode,
|
|
407
|
+
save_mode="auto",
|
|
408
|
+
))
|
|
409
|
+
finally:
|
|
410
|
+
loop.close()
|
|
411
|
+
result = await asyncio.wait_for(
|
|
412
|
+
asyncio.to_thread(_run_sync),
|
|
413
|
+
timeout=_hard_timeout,
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
# Send final report event with markdown
|
|
417
|
+
sse.send(client_id, "report", {
|
|
418
|
+
"case_id": result["case_id"],
|
|
419
|
+
"findings_count": result["findings_count"],
|
|
420
|
+
"confirmed": result["confirmed_count"],
|
|
421
|
+
"verifiability": f"{result['verifiability_score']:.0%}",
|
|
422
|
+
"markdown": result.get("markdown", ""),
|
|
423
|
+
"published_to_graph": req.publish_to_graph,
|
|
424
|
+
})
|
|
425
|
+
|
|
426
|
+
# ── Per-case consent: publish to community knowledge graph ──
|
|
427
|
+
if req.publish_to_graph and result.get("findings"):
|
|
428
|
+
publish_error = None
|
|
429
|
+
try:
|
|
430
|
+
# Check if key is available for remote MCP
|
|
431
|
+
is_local = "localhost" in MCP_SERVER_URL or "127.0.0.1" in MCP_SERVER_URL
|
|
432
|
+
if not is_local and not MCP_API_KEY:
|
|
433
|
+
publish_error = "No MCP API key configured. Set it in onboarding or with: export MCP_API_KEY=your-key"
|
|
434
|
+
else:
|
|
435
|
+
# Use the engine's report if available, otherwise publish manually
|
|
436
|
+
engine_report = getattr(result, '_report', None)
|
|
437
|
+
if engine_report:
|
|
438
|
+
_pub_result, _pub_err = _publish_to_mcp(engine_report)
|
|
439
|
+
if _pub_err:
|
|
440
|
+
publish_error = _pub_err
|
|
441
|
+
else:
|
|
442
|
+
# Fallback: publish entities from findings directly
|
|
443
|
+
import httpx
|
|
444
|
+
entities = []
|
|
445
|
+
for f in result["findings"]:
|
|
446
|
+
if hasattr(f, 'tier') and f.tier in ("CONFIRMED", "PROBABLE"):
|
|
447
|
+
for ent in getattr(f, 'entities', []):
|
|
448
|
+
# Normalize entities — they come in as strings, dicts, or objects
|
|
449
|
+
if isinstance(ent, str):
|
|
450
|
+
val, etype = ent.strip(), "unknown"
|
|
451
|
+
elif isinstance(ent, dict):
|
|
452
|
+
val = ent.get("value", ent.get("canonical", ""))
|
|
453
|
+
etype = ent.get("type", "unknown")
|
|
454
|
+
elif hasattr(ent, 'value'):
|
|
455
|
+
val = getattr(ent, 'value', '') or getattr(ent, 'canonical', '')
|
|
456
|
+
etype = getattr(ent, 'type', 'unknown')
|
|
457
|
+
else:
|
|
458
|
+
val, etype = str(ent), "unknown"
|
|
459
|
+
# Skip empty values — MCP validator rejects them
|
|
460
|
+
if not val or not val.strip():
|
|
461
|
+
continue
|
|
462
|
+
entities.append({
|
|
463
|
+
"value": val.strip()[:500],
|
|
464
|
+
"type": etype or "unknown",
|
|
465
|
+
"source": ent.get("source", "") if isinstance(ent, dict) else "",
|
|
466
|
+
"tier": f.tier if hasattr(f, 'tier') else "PROBABLE",
|
|
467
|
+
"case_id": result["case_id"],
|
|
468
|
+
})
|
|
469
|
+
if entities:
|
|
470
|
+
payload = {
|
|
471
|
+
"case_id": result["case_id"],
|
|
472
|
+
"target": req.query,
|
|
473
|
+
"target_type": "",
|
|
474
|
+
"findings_count": len(result["findings"]),
|
|
475
|
+
"confirmed_count": result.get("confirmed_count", 0),
|
|
476
|
+
"verifiability": "",
|
|
477
|
+
"date": "",
|
|
478
|
+
"entities": entities,
|
|
479
|
+
}
|
|
480
|
+
resp = httpx.post(
|
|
481
|
+
f"{MCP_SERVER_URL}/api/ingest",
|
|
482
|
+
json=payload, timeout=10,
|
|
483
|
+
headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {},
|
|
484
|
+
)
|
|
485
|
+
if resp.status_code != 200:
|
|
486
|
+
publish_error = f"MCP server returned {resp.status_code}"
|
|
487
|
+
except Exception as e:
|
|
488
|
+
publish_error = str(e)
|
|
489
|
+
|
|
490
|
+
if publish_error:
|
|
491
|
+
sse.send(client_id, "graph_error", {
|
|
492
|
+
"message": f"Graph publish failed: {publish_error}",
|
|
493
|
+
"case_id": result["case_id"],
|
|
494
|
+
})
|
|
495
|
+
else:
|
|
496
|
+
sse.send(client_id, "graph_published", {
|
|
497
|
+
"case_id": result["case_id"],
|
|
498
|
+
"url": MCP_SERVER_URL,
|
|
499
|
+
})
|
|
500
|
+
|
|
501
|
+
logger.info("investigation_complete", extra={
|
|
502
|
+
"client_id": client_id, "query": req.query[:80],
|
|
503
|
+
"findings": result["findings_count"],
|
|
504
|
+
"case": result["case_id"],
|
|
505
|
+
})
|
|
506
|
+
except asyncio.TimeoutError:
|
|
507
|
+
logger.error("investigation_timeout", extra={
|
|
508
|
+
"client_id": client_id, "query": req.query[:80],
|
|
509
|
+
"mode": req.mode,
|
|
510
|
+
})
|
|
511
|
+
sse.send(client_id, "error", {
|
|
512
|
+
"message": f"Investigation timed out after {_hard_timeout}s — try a shallower mode or narrower query"
|
|
513
|
+
})
|
|
514
|
+
except Exception as e:
|
|
515
|
+
logger.error("investigation_failed", extra={
|
|
516
|
+
"client_id": client_id, "query": req.query[:80], "error": str(e),
|
|
517
|
+
})
|
|
518
|
+
sse.send(client_id, "error", {"message": str(e)})
|
|
519
|
+
finally:
|
|
520
|
+
await asyncio.sleep(0.5)
|
|
521
|
+
sse.send(client_id, "_close", {})
|
|
522
|
+
# Cleanup interrupt queue
|
|
523
|
+
from src.watson.orchestration import get_engine as _get_eng
|
|
524
|
+
_eng = _get_eng()
|
|
525
|
+
_eng.remove_interrupt_queue(client_id)
|
|
526
|
+
|
|
527
|
+
task = asyncio.create_task(run())
|
|
528
|
+
register_task(task)
|
|
529
|
+
return {"client_id": client_id, "status": "started"}
|
|
530
|
+
|
|
531
|
+
@app.get("/api/agent/stream/{client_id}")
|
|
532
|
+
async def agent_stream(client_id: str):
|
|
533
|
+
"""SSE stream for an investigation in progress."""
|
|
534
|
+
q = sse._queues.get(client_id)
|
|
535
|
+
if not q:
|
|
536
|
+
async def error_gen():
|
|
537
|
+
yield f"event: error\ndata: {json.dumps({'message': 'Stream not found'})}\n\n"
|
|
538
|
+
yield "event: done\ndata: {}\n\n"
|
|
539
|
+
return StreamingResponse(error_gen(), media_type="text/event-stream")
|
|
540
|
+
|
|
541
|
+
async def generate():
|
|
542
|
+
try:
|
|
543
|
+
while True:
|
|
544
|
+
try:
|
|
545
|
+
ev_type, ev_data = await asyncio.wait_for(q.get(), timeout=30.0)
|
|
546
|
+
_ok, sse_str = _safe_serialize(ev_type, ev_data)
|
|
547
|
+
yield sse_str
|
|
548
|
+
if ev_type == "_close":
|
|
549
|
+
break
|
|
550
|
+
except asyncio.TimeoutError:
|
|
551
|
+
yield ": heartbeat\n\n"
|
|
552
|
+
finally:
|
|
553
|
+
sse.remove(client_id)
|
|
554
|
+
|
|
555
|
+
return StreamingResponse(
|
|
556
|
+
generate(),
|
|
557
|
+
media_type="text/event-stream",
|
|
558
|
+
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
|
559
|
+
)
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
# ── Interactive Investigation Steering ──────────────────────────
|
|
563
|
+
|
|
564
|
+
class InterruptRequest(BaseModel):
|
|
565
|
+
action: str = "context" # "context", "stop", "skip_phase"
|
|
566
|
+
text: str = ""
|
|
567
|
+
|
|
568
|
+
@app.post("/api/agent/investigate/{client_id}/interrupt")
|
|
569
|
+
async def agent_interrupt(client_id: str, req: InterruptRequest):
|
|
570
|
+
"""Send a steering command to a running investigation."""
|
|
571
|
+
from src.watson.orchestration import get_engine
|
|
572
|
+
engine = get_engine()
|
|
573
|
+
ok = engine.send_interrupt(client_id, {"action": req.action, "text": req.text})
|
|
574
|
+
if ok:
|
|
575
|
+
return {"status": "sent", "client_id": client_id}
|
|
576
|
+
return JSONResponse(
|
|
577
|
+
status_code=404,
|
|
578
|
+
content={"status": "not_found", "message": f"No active investigation: {client_id}"}
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
# ── Agent Chat with Tools (SSE) ──────────────────────────────────
|
|
583
|
+
|
|
584
|
+
WATSON_SOUL = (
|
|
585
|
+
"You are Watson, an autonomous OSINT investigator.\n\n"
|
|
586
|
+
"ABSOLUTE RULES — VIOLATION MEANS FAILURE:\n"
|
|
587
|
+
"- NEVER fabricate specific identifiers: no invented SAR numbers, VINs, wallet addresses, "
|
|
588
|
+
"transaction hashes, case IDs, or document numbers.\n"
|
|
589
|
+
"- NEVER invent reports or documents you cannot link to a public URL.\n"
|
|
590
|
+
"- NEVER fabricate dollar amounts from inaccessible databases.\n"
|
|
591
|
+
"- MARK UNCERTAINTY: [CONFIRMED], [PLAUSIBLE BUT UNVERIFIED], [HYPOTHETICAL — NOT REAL].\n"
|
|
592
|
+
"- When OSINT data runs out, STOP and state the boundary.\n\n"
|
|
593
|
+
"CAPABILITIES: web_search, fetch_url, whois_lookup, dns_lookup, crt_sh_search, "
|
|
594
|
+
"wayback_machine, etherscan_lookup, opencorporates_search, news_search, "
|
|
595
|
+
"run_terminal_command, investigate_target.\n\n"
|
|
596
|
+
"TONE: Sharp, direct, evidence-based. Brief like an intelligence analyst.\n"
|
|
597
|
+
"Use bullet points. Cite sources. Flag gaps.\n\n"
|
|
598
|
+
"CRITICAL — AFTER EVERY ANSWER:\n"
|
|
599
|
+
"If the user has CURRENT FINDINGS from an active investigation, you MUST end your response "
|
|
600
|
+
"with a 'Follow-up leads' section suggesting 2-3 concrete next investigation steps based on:\n"
|
|
601
|
+
"- Entities discovered but not yet explored (domains, people, companies in the findings)\n"
|
|
602
|
+
"- Gaps in the evidence (missing WHOIS details, unverified registrars, unresearched nameservers)\n"
|
|
603
|
+
"- Cross-references that deserve deeper investigation\n"
|
|
604
|
+
"Format each lead as: '🔍 Investigate [entity/value] — [why it matters]'\n"
|
|
605
|
+
"This is mandatory — the investigation is incomplete without follow-up leads.\n"
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
@app.post("/api/agent/chat/stream")
|
|
609
|
+
async def agent_chat_stream(req: ChatRequest):
|
|
610
|
+
"""Streaming chat — delegates to investigation engine for chat-style queries."""
|
|
611
|
+
async def respond():
|
|
612
|
+
yield f"event: token\ndata: {json.dumps({'token': 'Chat mode coming soon. Use /api/agent/investigate for full OSINT investigations.'})}\n\n"
|
|
613
|
+
yield "event: done\ndata: {}\n\n"
|
|
614
|
+
return StreamingResponse(respond(), media_type="text/event-stream")
|
|
615
|
+
|
|
616
|
+
# ── Agent Tools ──────────────────────────────────────────────────
|
|
617
|
+
|
|
618
|
+
@app.get("/api/agent/detect-intent")
|
|
619
|
+
async def detect_intent(msg: str = Query("", description="Message to classify")):
|
|
620
|
+
"""Determine whether a message is an investigation request or a chat question.
|
|
621
|
+
Public endpoint — no auth required."""
|
|
622
|
+
msg = msg.strip()
|
|
623
|
+
if not msg:
|
|
624
|
+
return {"intent": "chat", "confidence": 1.0}
|
|
625
|
+
|
|
626
|
+
# TECHNICAL TARGETS — clearly investigable, auto-dispatch
|
|
627
|
+
if re.search(r"\.(com|org|net|io|gov|edu|uk|de|fr|ru|cn|jp|ai|dev|onion)\b", msg, re.IGNORECASE):
|
|
628
|
+
return {"intent": "investigate", "confidence": 0.95, "reason": "domain_detected"}
|
|
629
|
+
if re.search(r"@[\w.-]+\.[a-z]{2,}", msg):
|
|
630
|
+
return {"intent": "investigate", "confidence": 0.95, "reason": "email_detected"}
|
|
631
|
+
if re.search(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", msg):
|
|
632
|
+
return {"intent": "investigate", "confidence": 0.95, "reason": "ip_detected"}
|
|
633
|
+
if re.search(r"0x[a-fA-F0-9]{40}", msg):
|
|
634
|
+
return {"intent": "investigate", "confidence": 0.95, "reason": "crypto_detected"}
|
|
635
|
+
|
|
636
|
+
# EXPLICIT INVESTIGATION COMMANDS
|
|
637
|
+
if re.match(r"^(investigate|look\s+into|research|dig\s+into|check\s+out|find\s+everything\s+(?:about|on))\s+", msg, re.IGNORECASE):
|
|
638
|
+
return {"intent": "investigate", "confidence": 0.85, "reason": "explicit_command"}
|
|
639
|
+
|
|
640
|
+
# OSINT-INTENT PHRASES — these signal "investigate X", not a chat question.
|
|
641
|
+
# "Amazon due diligence", "background check on Acme", "Tesla controversies",
|
|
642
|
+
# "risks of investing in X", "Acme lawsuits/sanctions/scandal".
|
|
643
|
+
osint_intent = re.search(
|
|
644
|
+
r"(?i)\b(due\s+diligence|background\s+check|kyc|aml|"
|
|
645
|
+
r"controvers(?:y|ies)|lawsuit?s?|litigation|sanction(?:s|ed)?|"
|
|
646
|
+
r"scandal|fraud|investigation|red\s+flags?|reputation|"
|
|
647
|
+
r"risk\s+(?:assessment|profile|exposure)|adverse\s+media|"
|
|
648
|
+
r"compliance|regulatory\s+action|antitrust|wrongdoing)\b",
|
|
649
|
+
msg,
|
|
650
|
+
)
|
|
651
|
+
if osint_intent:
|
|
652
|
+
return {"intent": "investigate", "confidence": 0.88,
|
|
653
|
+
"reason": f"osint_intent:{osint_intent.group(1).lower()}"}
|
|
654
|
+
|
|
655
|
+
# OSINT-QUESTION PATTERNS — questions that demand investigation, not chat.
|
|
656
|
+
# "What is the corporate structure of X", "Who owns Y", "How did Z become involved"
|
|
657
|
+
# These look like "what/who/how" questions but are OSINT targets.
|
|
658
|
+
osint_question = re.search(
|
|
659
|
+
r"(?i)\b("
|
|
660
|
+
r"corporate\s+structure|beneficial\s+owner(?:s|ship)?|"
|
|
661
|
+
r"who\s+owns?|who\s+controls?|"
|
|
662
|
+
r"ownership\s+(?:structure|chain|network)|"
|
|
663
|
+
r"shell\s+compan|subsidiary|parent\s+compan|"
|
|
664
|
+
r"supply\s+chain|shipping\s+(?:compan|fleet|network)|"
|
|
665
|
+
r"how\s+did\s+.+\s+become\s+involved|"
|
|
666
|
+
r"what\s+(?:companies|entities|firms)\s+(?:are|own|control|operate)"
|
|
667
|
+
r")\b",
|
|
668
|
+
msg,
|
|
669
|
+
)
|
|
670
|
+
if osint_question:
|
|
671
|
+
return {"intent": "investigate", "confidence": 0.82,
|
|
672
|
+
"reason": f"osint_question:{osint_question.group(1).lower()[:40]}"}
|
|
673
|
+
|
|
674
|
+
# CHAT PATTERNS — questions about knowledge, not OSINT targets
|
|
675
|
+
chat_patterns = [
|
|
676
|
+
r"^(what|who|where|when|why|how)\s+(is|are|was|were|do|does|did|can|could|would|should|shall|will|may|might)",
|
|
677
|
+
r"\?$", # ends with question mark
|
|
678
|
+
r"^(tell|explain|describe|show|list|summarize|compare|define|elaborate)",
|
|
679
|
+
r"^(hello|hi|hey|help|thanks|thank|what's up|howdy)",
|
|
680
|
+
r"^(what|who|where|when|why|how)\s+(about|to|can|could|do)",
|
|
681
|
+
r"^(can|will|would)\s+you\s+(tell|explain|show|help|list)",
|
|
682
|
+
r"^(i\s+(?:have|need|want)\s+(?:a|some|to)\s+(?:question|ask|know|understand))",
|
|
683
|
+
]
|
|
684
|
+
for pattern in chat_patterns:
|
|
685
|
+
if re.search(pattern, msg, re.IGNORECASE):
|
|
686
|
+
return {"intent": "chat", "confidence": 0.90, "reason": "conversational_pattern"}
|
|
687
|
+
|
|
688
|
+
# ── LLM CLASSIFIER FALLBACK ──
|
|
689
|
+
# If none of the fast-path regexes matched, ask the LLM. This replaces
|
|
690
|
+
# the old "default to chat" which trapped investigable targets like
|
|
691
|
+
# "Area 51" and article headlines in a 5-turn conversation loop.
|
|
692
|
+
try:
|
|
693
|
+
from src.watson.orchestration.intent_classifier import classify_intent
|
|
694
|
+
from src.watson.orchestration.llm_config import call_llm
|
|
695
|
+
|
|
696
|
+
intent, confidence, reason = await classify_intent(msg, call_llm)
|
|
697
|
+
return {"intent": intent, "confidence": confidence, "reason": reason}
|
|
698
|
+
except Exception as e:
|
|
699
|
+
logger.warning("intent_classifier_error: %s", e)
|
|
700
|
+
|
|
701
|
+
# Absolute last resort — short non-question messages are probably targets
|
|
702
|
+
word_count = len(msg.split())
|
|
703
|
+
if "?" not in msg and word_count <= 15 and any(c.isalpha() for c in msg):
|
|
704
|
+
return {"intent": "investigate", "confidence": 0.50,
|
|
705
|
+
"reason": "fallback_short_topic"}
|
|
706
|
+
return {"intent": "chat", "confidence": 0.50, "reason": "fallback_ambiguous"}
|
|
707
|
+
|
|
708
|
+
@app.post("/api/agent/terminal")
|
|
709
|
+
async def agent_terminal(req: Request):
|
|
710
|
+
"""Terminal command execution — admin key required, allowlist enforced."""
|
|
711
|
+
import subprocess
|
|
712
|
+
|
|
713
|
+
# Admin auth is handled by middleware — if we get here, it passed
|
|
714
|
+
data = await req.json()
|
|
715
|
+
cmd = data.get("command", "").strip()
|
|
716
|
+
if not cmd:
|
|
717
|
+
return JSONResponse({"error": "No command"}, status_code=400)
|
|
718
|
+
|
|
719
|
+
# Command allowlist — only safe OSINT tools
|
|
720
|
+
ALLOWED = {"whois", "dig", "nslookup", "curl", "wget", "traceroute", "ping",
|
|
721
|
+
"openssl", "python3", "git", "date", "echo", "cat", "head", "tail"}
|
|
722
|
+
cmd_root = cmd.split()[0].split("/")[-1] if cmd.split() else ""
|
|
723
|
+
if cmd_root not in ALLOWED and not cmd.startswith(("python3 -c", "git ", "curl ", "echo ")):
|
|
724
|
+
logger.warning("terminal_blocked", extra={"command": cmd[:100], "root": cmd_root})
|
|
725
|
+
return JSONResponse(
|
|
726
|
+
{"error": f"Command '{cmd_root}' not in allowlist. Allowed: {', '.join(sorted(ALLOWED))}"},
|
|
727
|
+
status_code=403,
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
try:
|
|
731
|
+
result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=30)
|
|
732
|
+
logger.info("terminal_executed", extra={"command": cmd[:100], "exit_code": result.returncode})
|
|
733
|
+
return {"stdout": result.stdout.strip()[:10000], "stderr": result.stderr.strip()[:5000], "exit_code": result.returncode}
|
|
734
|
+
except subprocess.TimeoutExpired:
|
|
735
|
+
return {"error": "Timed out (30s)", "exit_code": -1}
|
|
736
|
+
except Exception as e:
|
|
737
|
+
logger.error("terminal_failed", extra={"command": cmd[:100], "error": str(e)})
|
|
738
|
+
return {"error": str(e), "exit_code": -1}
|
|
739
|
+
|
|
740
|
+
# ── Memory Endpoints ─────────────────────────────────────────────
|
|
741
|
+
|
|
742
|
+
@app.get("/api/memory/search")
|
|
743
|
+
async def memory_search(q: str = Query(...), limit: int = 10):
|
|
744
|
+
try:
|
|
745
|
+
from watson.memory import memory as mem
|
|
746
|
+
return {"results": mem.search(q, limit=limit)}
|
|
747
|
+
except Exception as e:
|
|
748
|
+
logger.warning("memory_search_failed", extra={"query": q[:100], "error": str(e)})
|
|
749
|
+
return {"results": []}
|
|
750
|
+
|
|
751
|
+
@app.get("/api/memory/recent")
|
|
752
|
+
async def memory_recent(limit: int = 20):
|
|
753
|
+
try:
|
|
754
|
+
from watson.memory import memory as mem
|
|
755
|
+
return {"investigations": mem.list_recent(limit=limit)}
|
|
756
|
+
except Exception as e:
|
|
757
|
+
logger.warning("memory_recent_failed", extra={"error": str(e)})
|
|
758
|
+
return {"investigations": []}
|
|
759
|
+
|
|
760
|
+
@app.get("/api/memory/stats")
|
|
761
|
+
async def memory_stats():
|
|
762
|
+
try:
|
|
763
|
+
from watson.memory import memory as mem
|
|
764
|
+
return mem.stats()
|
|
765
|
+
except Exception as e:
|
|
766
|
+
logger.warning("memory_stats_failed", extra={"error": str(e)})
|
|
767
|
+
return {"investigations": 0, "entities": 0, "findings": 0}
|
|
768
|
+
|
|
769
|
+
# ── Scheduler ────────────────────────────────────────────────────
|
|
770
|
+
|
|
771
|
+
@app.get("/api/scheduler/jobs")
|
|
772
|
+
async def scheduler_list():
|
|
773
|
+
try:
|
|
774
|
+
from watson.core.scheduler import Scheduler
|
|
775
|
+
sched = Scheduler()
|
|
776
|
+
return {"jobs": sched.list_jobs(), "total": len(sched.list_jobs())}
|
|
777
|
+
except Exception as e:
|
|
778
|
+
logger.warning("scheduler_list_failed", extra={"error": str(e)})
|
|
779
|
+
return {"jobs": [], "total": 0}
|
|
780
|
+
|
|
781
|
+
# ── Toolkit Registry ─────────────────────────────────────────────
|
|
782
|
+
|
|
783
|
+
@app.get("/api/bellingcat/summary")
|
|
784
|
+
async def bellingcat_summary():
|
|
785
|
+
try:
|
|
786
|
+
from watson.toolkit_registry import BellingcatRegistry
|
|
787
|
+
reg = BellingcatRegistry()
|
|
788
|
+
return reg.summary()
|
|
789
|
+
except Exception as e:
|
|
790
|
+
logger.warning("bellingcat_summary_failed", extra={"error": str(e)})
|
|
791
|
+
return {"total_tools": 0, "categories": 0}
|
|
792
|
+
|
|
793
|
+
# ── Intelligence Ledger ──────────────────────────────────────────
|
|
794
|
+
|
|
795
|
+
@app.get("/api/ledger/stats")
|
|
796
|
+
async def ledger_stats():
|
|
797
|
+
"""Cross-case intelligence accumulation statistics."""
|
|
798
|
+
try:
|
|
799
|
+
from src.watson.ethics import get_ledger
|
|
800
|
+
ledger = get_ledger()
|
|
801
|
+
return ledger.get_stats()
|
|
802
|
+
except ImportError:
|
|
803
|
+
try:
|
|
804
|
+
from watson.ethics import get_ledger
|
|
805
|
+
ledger = get_ledger()
|
|
806
|
+
return ledger.get_stats()
|
|
807
|
+
except Exception:
|
|
808
|
+
return {"total_investigations": 0, "status": "ledger_unavailable"}
|
|
809
|
+
|
|
810
|
+
@app.get("/api/ledger/entity")
|
|
811
|
+
async def ledger_entity(q: str = Query(..., description="Entity to query")):
|
|
812
|
+
"""Query prior intelligence on a specific entity."""
|
|
813
|
+
try:
|
|
814
|
+
from src.watson.ethics import get_ledger
|
|
815
|
+
ledger = get_ledger()
|
|
816
|
+
result = ledger.get_entity_intel(q)
|
|
817
|
+
if result:
|
|
818
|
+
return {"found": True, "entity": q, "intel": result}
|
|
819
|
+
return {"found": False, "entity": q}
|
|
820
|
+
except ImportError:
|
|
821
|
+
try:
|
|
822
|
+
from watson.ethics import get_ledger
|
|
823
|
+
ledger = get_ledger()
|
|
824
|
+
result = ledger.get_entity_intel(q)
|
|
825
|
+
return {"found": bool(result), "entity": q, "intel": result}
|
|
826
|
+
except Exception:
|
|
827
|
+
return {"found": False, "entity": q, "status": "ledger_unavailable"}
|
|
828
|
+
|
|
829
|
+
# ── Orchestrator (multi-agent, enterprise) ─────────────────────────
|
|
830
|
+
|
|
831
|
+
@app.post("/api/agent/orchestrate")
|
|
832
|
+
async def agent_orchestrate(req: InvestigateRequest):
|
|
833
|
+
"""Multi-turn investigation with persistence, retry, and adversarial resilience."""
|
|
834
|
+
import gc
|
|
835
|
+
gc.collect()
|
|
836
|
+
|
|
837
|
+
from src.watson.orchestration import get_engine
|
|
838
|
+
from src.watson.metrics import investigations_total, findings_total, findings_confirmed
|
|
839
|
+
|
|
840
|
+
engine = get_engine(max_hops=5)
|
|
841
|
+
client_id = f"orch-{uuid.uuid4().hex[:8]}"
|
|
842
|
+
q = sse.create(client_id)
|
|
843
|
+
|
|
844
|
+
async def run():
|
|
845
|
+
def push(event_type, data):
|
|
846
|
+
sse.send(client_id, event_type, data)
|
|
847
|
+
try:
|
|
848
|
+
inv = await engine.investigate(
|
|
849
|
+
query=req.query,
|
|
850
|
+
context=req.context or "",
|
|
851
|
+
on_event=push,
|
|
852
|
+
image_path=req.image_path or "",
|
|
853
|
+
)
|
|
854
|
+
|
|
855
|
+
findings_total.inc(inv.total_findings)
|
|
856
|
+
findings_confirmed.inc(inv.confirmed_count)
|
|
857
|
+
|
|
858
|
+
# Generate report
|
|
859
|
+
import json as _json
|
|
860
|
+
cross_refs = _json.loads(inv.cross_references) if inv.cross_references else []
|
|
861
|
+
|
|
862
|
+
sse.send(client_id, "report", {
|
|
863
|
+
"investigation_id": inv.investigation_id,
|
|
864
|
+
"status": inv.status.value,
|
|
865
|
+
"findings_count": inv.total_findings,
|
|
866
|
+
"hops": inv.total_hops,
|
|
867
|
+
"confirmed": inv.confirmed_count,
|
|
868
|
+
"cross_references": len(cross_refs),
|
|
869
|
+
"created_at": inv.created_at,
|
|
870
|
+
})
|
|
871
|
+
sse.send(client_id, "cross_references", {"patterns": cross_refs[:5]})
|
|
872
|
+
|
|
873
|
+
logger.info("orchestration_complete", extra={
|
|
874
|
+
"investigation_id": inv.investigation_id,
|
|
875
|
+
"query": req.query[:80],
|
|
876
|
+
"findings": inv.total_findings,
|
|
877
|
+
"hops": inv.total_hops,
|
|
878
|
+
"confirmed": inv.confirmed_count,
|
|
879
|
+
})
|
|
880
|
+
except Exception as e:
|
|
881
|
+
logger.error("orchestration_failed", extra={
|
|
882
|
+
"client_id": client_id, "query": req.query[:80], "error": str(e),
|
|
883
|
+
})
|
|
884
|
+
sse.send(client_id, "error", {"message": str(e)})
|
|
885
|
+
finally:
|
|
886
|
+
await asyncio.sleep(0.5)
|
|
887
|
+
sse.send(client_id, "_close", {})
|
|
888
|
+
# Cleanup interrupt queue
|
|
889
|
+
from src.watson.orchestration import get_engine as _get_eng
|
|
890
|
+
_eng = _get_eng()
|
|
891
|
+
_eng.remove_interrupt_queue(client_id)
|
|
892
|
+
|
|
893
|
+
task = asyncio.create_task(run())
|
|
894
|
+
register_task(task)
|
|
895
|
+
return {"client_id": client_id, "status": "started", "mode": "multi-agent"}
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
@app.get("/api/agent/agents")
|
|
899
|
+
async def list_agents():
|
|
900
|
+
"""List all available specialized agents with capabilities."""
|
|
901
|
+
from src.watson.agents import get_all_agents
|
|
902
|
+
agents = get_all_agents()
|
|
903
|
+
return {
|
|
904
|
+
"agents": [
|
|
905
|
+
{
|
|
906
|
+
"role": a.role.value,
|
|
907
|
+
"description": a.description,
|
|
908
|
+
"capabilities": a.capabilities,
|
|
909
|
+
"tool_count": a.tool_count,
|
|
910
|
+
}
|
|
911
|
+
for a in agents.values()
|
|
912
|
+
],
|
|
913
|
+
"total": len(agents),
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
# ── Legacy compat ────────────────────────────────────────────────
|
|
918
|
+
|
|
919
|
+
@app.post("/api/chat")
|
|
920
|
+
async def legacy_chat(req: ChatRequest):
|
|
921
|
+
"""Legacy alias → forwards to streaming chat."""
|
|
922
|
+
return await agent_chat_stream(req)
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
# ═══════════════════════════════════════════════════════════════
|
|
926
|
+
# PHASE B — Auth, Exports, Search, Workspaces
|
|
927
|
+
# ═══════════════════════════════════════════════════════════════
|
|
928
|
+
|
|
929
|
+
# ── Auth ─────────────────────────────────────────────────────
|
|
930
|
+
|
|
931
|
+
@app.post("/api/auth/register")
|
|
932
|
+
async def auth_register(req: Request):
|
|
933
|
+
"""Register a new workspace + admin user."""
|
|
934
|
+
body = await req.json()
|
|
935
|
+
email = body.get("email", "").strip()
|
|
936
|
+
workspace_name = body.get("workspace", "").strip()
|
|
937
|
+
|
|
938
|
+
if not email or "@" not in email:
|
|
939
|
+
return JSONResponse(status_code=400, content={"error": "Valid email required"})
|
|
940
|
+
if not workspace_name:
|
|
941
|
+
return JSONResponse(status_code=400, content={"error": "Workspace name required"})
|
|
942
|
+
|
|
943
|
+
try:
|
|
944
|
+
from src.watson.auth.store import get_auth_store
|
|
945
|
+
store = get_auth_store()
|
|
946
|
+
ws, user = store.create_workspace(workspace_name, email)
|
|
947
|
+
api_key = store.generate_api_key(user.user_id)
|
|
948
|
+
token = store.create_token(user)
|
|
949
|
+
|
|
950
|
+
return {
|
|
951
|
+
"user_id": user.user_id,
|
|
952
|
+
"workspace_id": ws.workspace_id,
|
|
953
|
+
"workspace_name": ws.name,
|
|
954
|
+
"api_key": api_key,
|
|
955
|
+
"token": token.token,
|
|
956
|
+
"expires_at": token.expires_at,
|
|
957
|
+
}
|
|
958
|
+
except Exception as e:
|
|
959
|
+
return JSONResponse(status_code=409, content={"error": str(e)})
|
|
960
|
+
|
|
961
|
+
@app.post("/api/auth/login")
|
|
962
|
+
async def auth_login(req: Request):
|
|
963
|
+
"""Login with API key or email + workspace."""
|
|
964
|
+
body = await req.json()
|
|
965
|
+
api_key = body.get("api_key", "").strip()
|
|
966
|
+
|
|
967
|
+
try:
|
|
968
|
+
from src.watson.auth.store import get_auth_store
|
|
969
|
+
store = get_auth_store()
|
|
970
|
+
|
|
971
|
+
if api_key:
|
|
972
|
+
user = store.validate_api_key(api_key)
|
|
973
|
+
else:
|
|
974
|
+
email = body.get("email", "").strip()
|
|
975
|
+
workspace_id = body.get("workspace_id", "").strip()
|
|
976
|
+
user = store.get_user_by_email(email, workspace_id)
|
|
977
|
+
|
|
978
|
+
if not user:
|
|
979
|
+
return JSONResponse(status_code=401, content={"error": "Invalid credentials"})
|
|
980
|
+
|
|
981
|
+
token = store.create_token(user)
|
|
982
|
+
return {
|
|
983
|
+
"user_id": user.user_id,
|
|
984
|
+
"workspace_id": user.workspace_id,
|
|
985
|
+
"role": user.role.value,
|
|
986
|
+
"token": token.token,
|
|
987
|
+
"expires_at": token.expires_at,
|
|
988
|
+
}
|
|
989
|
+
except Exception as e:
|
|
990
|
+
return JSONResponse(status_code=500, content={"error": str(e)})
|
|
991
|
+
|
|
992
|
+
|
|
993
|
+
# ── Workspaces ───────────────────────────────────────────────
|
|
994
|
+
|
|
995
|
+
@app.get("/api/workspace/{workspace_id}")
|
|
996
|
+
async def workspace_get(workspace_id: str, req: Request):
|
|
997
|
+
"""Get workspace details."""
|
|
998
|
+
from src.watson.auth.store import get_auth_store
|
|
999
|
+
store = get_auth_store()
|
|
1000
|
+
ws = store.get_workspace(workspace_id)
|
|
1001
|
+
if not ws:
|
|
1002
|
+
return JSONResponse(status_code=404, content={"error": "Workspace not found"})
|
|
1003
|
+
users = store.list_users(workspace_id)
|
|
1004
|
+
return {
|
|
1005
|
+
"workspace": {
|
|
1006
|
+
"id": ws.workspace_id, "name": ws.name,
|
|
1007
|
+
"created_at": ws.created_at,
|
|
1008
|
+
},
|
|
1009
|
+
"users": [{"id": u.user_id, "email": u.email, "role": u.role.value} for u in users],
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
|
|
1013
|
+
# ── Exports ──────────────────────────────────────────────────
|
|
1014
|
+
|
|
1015
|
+
class ExportRequest(BaseModel):
|
|
1016
|
+
investigation_id: str
|
|
1017
|
+
format: str = "json" # json, stix, misp, pdf, markdown
|
|
1018
|
+
|
|
1019
|
+
|
|
1020
|
+
@app.post("/api/export")
|
|
1021
|
+
async def export_investigation(export: ExportRequest):
|
|
1022
|
+
"""Export investigation in requested format."""
|
|
1023
|
+
from src.watson.persistence import get_store
|
|
1024
|
+
from src.watson.exports import BellingcatReport
|
|
1025
|
+
|
|
1026
|
+
store = get_store()
|
|
1027
|
+
inv, steps = store.get_full_investigation(export.investigation_id)
|
|
1028
|
+
if inv is None:
|
|
1029
|
+
return JSONResponse(status_code=404, content={"error": "Investigation not found"})
|
|
1030
|
+
|
|
1031
|
+
# Collect findings from all steps
|
|
1032
|
+
all_findings = []
|
|
1033
|
+
for s in steps:
|
|
1034
|
+
if s.findings_json:
|
|
1035
|
+
try:
|
|
1036
|
+
all_findings.extend(json.loads(s.findings_json))
|
|
1037
|
+
except json.JSONDecodeError:
|
|
1038
|
+
pass
|
|
1039
|
+
|
|
1040
|
+
cross_refs = json.loads(inv.cross_references) if inv.cross_references else []
|
|
1041
|
+
|
|
1042
|
+
report = BellingcatReport(
|
|
1043
|
+
query=inv.original_query,
|
|
1044
|
+
investigation_id=inv.investigation_id,
|
|
1045
|
+
target_type=inv.target_type,
|
|
1046
|
+
target_value=inv.target_value,
|
|
1047
|
+
)
|
|
1048
|
+
report.add_findings(all_findings)
|
|
1049
|
+
report.add_cross_references(cross_refs)
|
|
1050
|
+
report.hops = inv.total_hops
|
|
1051
|
+
|
|
1052
|
+
# Save to cases directory
|
|
1053
|
+
cases_dir = Path("cases/exports")
|
|
1054
|
+
cases_dir.mkdir(parents=True, exist_ok=True)
|
|
1055
|
+
|
|
1056
|
+
fmt = export.format.lower()
|
|
1057
|
+
if fmt == "json":
|
|
1058
|
+
path = report.to_json(cases_dir / f"{inv.investigation_id}.json")
|
|
1059
|
+
elif fmt == "stix":
|
|
1060
|
+
path = report.to_stix(cases_dir / f"{inv.investigation_id}_stix.json")
|
|
1061
|
+
elif fmt == "misp":
|
|
1062
|
+
path = report.to_misp(cases_dir / f"{inv.investigation_id}_misp.json")
|
|
1063
|
+
elif fmt == "pdf":
|
|
1064
|
+
path = report.to_pdf(cases_dir / f"{inv.investigation_id}.pdf")
|
|
1065
|
+
elif fmt == "markdown":
|
|
1066
|
+
path = cases_dir / f"{inv.investigation_id}.md"
|
|
1067
|
+
path.write_text(report.to_markdown())
|
|
1068
|
+
else:
|
|
1069
|
+
return JSONResponse(status_code=400, content={"error": f"Unknown format: {fmt}"})
|
|
1070
|
+
|
|
1071
|
+
return {
|
|
1072
|
+
"format": fmt,
|
|
1073
|
+
"path": str(path),
|
|
1074
|
+
"size_bytes": path.stat().st_size if path.exists() else 0,
|
|
1075
|
+
"findings": len(all_findings),
|
|
1076
|
+
"verifiability": f"{report._verifiability():.0%}",
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
# ── Search ───────────────────────────────────────────────────
|
|
1081
|
+
|
|
1082
|
+
@app.get("/api/search")
|
|
1083
|
+
async def search_investigations(
|
|
1084
|
+
q: str = Query("", description="Search query"),
|
|
1085
|
+
agent: str = Query("", description="Filter by agent role"),
|
|
1086
|
+
tier: str = Query("", description="Filter by evidence tier"),
|
|
1087
|
+
limit: int = Query(20, description="Max results"),
|
|
1088
|
+
):
|
|
1089
|
+
"""Full-text search across all investigations."""
|
|
1090
|
+
if not q:
|
|
1091
|
+
return {"results": [], "total": 0, "query": q}
|
|
1092
|
+
|
|
1093
|
+
from src.watson.search import get_search
|
|
1094
|
+
s = get_search()
|
|
1095
|
+
results = s.search(q, limit=limit, agent_filter=agent, tier_filter=tier)
|
|
1096
|
+
|
|
1097
|
+
return {
|
|
1098
|
+
"query": q,
|
|
1099
|
+
"total": len(results),
|
|
1100
|
+
"results": results,
|
|
1101
|
+
"filters": {"agent": agent, "tier": tier},
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
@app.get("/api/search/investigations")
|
|
1106
|
+
async def search_inv_list(
|
|
1107
|
+
q: str = Query("", description="Search investigations"),
|
|
1108
|
+
limit: int = Query(10),
|
|
1109
|
+
):
|
|
1110
|
+
"""Search investigation-level metadata."""
|
|
1111
|
+
if not q:
|
|
1112
|
+
return {"results": [], "total": 0}
|
|
1113
|
+
|
|
1114
|
+
from src.watson.search import get_search
|
|
1115
|
+
s = get_search()
|
|
1116
|
+
results = s.search_investigations(q, limit=limit)
|
|
1117
|
+
return {"query": q, "total": len(results), "results": results}
|
|
1118
|
+
|
|
1119
|
+
|
|
1120
|
+
@app.get("/api/search/stats")
|
|
1121
|
+
async def search_stats():
|
|
1122
|
+
"""Get search index statistics."""
|
|
1123
|
+
from src.watson.search import get_search
|
|
1124
|
+
return get_search().get_stats()
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
# ═══════════════════════════════════════════════════════════════
|
|
1128
|
+
# API Key Settings — user-managed keys for tools
|
|
1129
|
+
# ═══════════════════════════════════════════════════════════════
|
|
1130
|
+
|
|
1131
|
+
from pydantic import BaseModel
|
|
1132
|
+
|
|
1133
|
+
|
|
1134
|
+
class SetKeyRequest(BaseModel):
|
|
1135
|
+
slug: str
|
|
1136
|
+
value: str
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
class SaveRequest(BaseModel):
|
|
1140
|
+
consent_publish: bool = False
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
@app.post("/api/cases/{case_id}/save")
|
|
1144
|
+
async def save_case(case_id: str, req: SaveRequest = SaveRequest()):
|
|
1145
|
+
"""Save a pending investigation to disk + knowledge graph + optionally MCP.
|
|
1146
|
+
|
|
1147
|
+
With auto-save, the case is already on disk. This endpoint handles:
|
|
1148
|
+
- pending: save + graph + optional MCP publish
|
|
1149
|
+
- already saved: just MCP publish if requested
|
|
1150
|
+
"""
|
|
1151
|
+
from src.watson.orchestration import get_engine
|
|
1152
|
+
from pathlib import Path
|
|
1153
|
+
engine = get_engine()
|
|
1154
|
+
|
|
1155
|
+
report = None
|
|
1156
|
+
already_saved = False
|
|
1157
|
+
matches: list = [] # populated when case found on disk
|
|
1158
|
+
|
|
1159
|
+
if hasattr(engine, '_pending_reports') and case_id in engine._pending_reports:
|
|
1160
|
+
report = engine._pending_reports[case_id]
|
|
1161
|
+
else:
|
|
1162
|
+
# Check if already saved to disk (auto-save mode)
|
|
1163
|
+
# _save_case writes "{case_id}_{date}.md" — glob for it
|
|
1164
|
+
cases_dir = Path.home() / "watson-cases"
|
|
1165
|
+
matches = list(cases_dir.glob(f"{case_id}_*.md")) if cases_dir.exists() else []
|
|
1166
|
+
if matches:
|
|
1167
|
+
already_saved = True
|
|
1168
|
+
else:
|
|
1169
|
+
raise HTTPException(404,
|
|
1170
|
+
f"Case {case_id} not found. It may not exist or the server was restarted. "
|
|
1171
|
+
f"Run the investigation again.")
|
|
1172
|
+
|
|
1173
|
+
if report is not None:
|
|
1174
|
+
# Normal flow: save pending report
|
|
1175
|
+
engine._save_case(report)
|
|
1176
|
+
engine._update_graph(report)
|
|
1177
|
+
del engine._pending_reports[case_id]
|
|
1178
|
+
|
|
1179
|
+
# Publish to MCP only with explicit consent
|
|
1180
|
+
published = False
|
|
1181
|
+
publish_error = None
|
|
1182
|
+
if req.consent_publish:
|
|
1183
|
+
if report:
|
|
1184
|
+
ok, err = _publish_to_mcp(report)
|
|
1185
|
+
elif already_saved:
|
|
1186
|
+
# Reconstruct from disk for publishing
|
|
1187
|
+
# _save_case writes "{case_id}_{date}.md" — use first glob match
|
|
1188
|
+
case_path = matches[0]
|
|
1189
|
+
text = case_path.read_text()
|
|
1190
|
+
from src.watson.orchestration.engine import OrchestrationEngine
|
|
1191
|
+
entities = OrchestrationEngine._extract_entities_from_text(text)
|
|
1192
|
+
if entities:
|
|
1193
|
+
import httpx
|
|
1194
|
+
import re as _re
|
|
1195
|
+
target = ""
|
|
1196
|
+
tm = _re.search(r"\*\*Target:\*\*\s*(.+)", text)
|
|
1197
|
+
if tm: target = tm.group(1).strip()
|
|
1198
|
+
payload = {
|
|
1199
|
+
"case_id": case_id,
|
|
1200
|
+
"target": target,
|
|
1201
|
+
"target_type": "person",
|
|
1202
|
+
"findings_count": 0,
|
|
1203
|
+
"confirmed_count": 0,
|
|
1204
|
+
"verifiability": "",
|
|
1205
|
+
"date": "",
|
|
1206
|
+
"entities": [{
|
|
1207
|
+
"value": e["value"], "type": e["type"],
|
|
1208
|
+
"source": "", "tier": "PROBABLE", "case_id": case_id,
|
|
1209
|
+
} for e in entities],
|
|
1210
|
+
}
|
|
1211
|
+
resp = httpx.post(
|
|
1212
|
+
f"{MCP_SERVER_URL}/api/ingest", json=payload, timeout=10,
|
|
1213
|
+
headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {},
|
|
1214
|
+
)
|
|
1215
|
+
ok = resp.status_code == 200
|
|
1216
|
+
err = None if ok else f"MCP server returned {resp.status_code}"
|
|
1217
|
+
else:
|
|
1218
|
+
ok, err = False, "No entities to publish"
|
|
1219
|
+
else:
|
|
1220
|
+
ok, err = False, "No report data available"
|
|
1221
|
+
published = ok
|
|
1222
|
+
publish_error = err
|
|
1223
|
+
|
|
1224
|
+
return {
|
|
1225
|
+
"case_id": case_id,
|
|
1226
|
+
"target": report.query if report else "saved case",
|
|
1227
|
+
"findings": len(report.findings) if report else 0,
|
|
1228
|
+
"verifiability": f"{report.verifiability_score:.0%}" if report else "",
|
|
1229
|
+
"status": "saved" if not already_saved else "already_saved",
|
|
1230
|
+
"published": published,
|
|
1231
|
+
"publish_error": publish_error,
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
def _publish_to_mcp(report):
|
|
1236
|
+
"""Publish case entities to the MCP knowledge graph server with full case data.
|
|
1237
|
+
|
|
1238
|
+
Uses the engine's entity extraction to get real typed entities, not finding titles.
|
|
1239
|
+
Returns (success: bool, error: str | None).
|
|
1240
|
+
"""
|
|
1241
|
+
try:
|
|
1242
|
+
import httpx
|
|
1243
|
+
# Use the engine's entity extraction — not finding titles
|
|
1244
|
+
from src.watson.orchestration.engine import OrchestrationEngine
|
|
1245
|
+
|
|
1246
|
+
# Build combined text from all finding titles and bodies
|
|
1247
|
+
text = " ".join(f"{f.title} {f.body}" for f in report.findings)
|
|
1248
|
+
entities = OrchestrationEngine._extract_entities_from_text(text)
|
|
1249
|
+
if not entities:
|
|
1250
|
+
return False, "No typed entities to publish"
|
|
1251
|
+
|
|
1252
|
+
payload = {
|
|
1253
|
+
"case_id": report.case_id,
|
|
1254
|
+
"target": report.query,
|
|
1255
|
+
"target_type": getattr(report, "target_type", "person"),
|
|
1256
|
+
"findings_count": len(report.findings),
|
|
1257
|
+
"confirmed_count": sum(1 for f in report.findings if f.tier == "CONFIRMED"),
|
|
1258
|
+
"verifiability": f"{report.verifiability_score:.0%}",
|
|
1259
|
+
"date": getattr(report, "timestamp", ""),
|
|
1260
|
+
"entities": [{
|
|
1261
|
+
"value": e.get("value", ""),
|
|
1262
|
+
"type": e.get("type", "unknown"),
|
|
1263
|
+
"source": e.get("source", ""),
|
|
1264
|
+
"tier": e.get("tier", "PROBABLE"),
|
|
1265
|
+
"case_id": report.case_id,
|
|
1266
|
+
} for e in entities],
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
resp = httpx.post(
|
|
1270
|
+
f"{MCP_SERVER_URL}/api/ingest", json=payload, timeout=10,
|
|
1271
|
+
headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {},
|
|
1272
|
+
)
|
|
1273
|
+
if resp.status_code == 200:
|
|
1274
|
+
data = resp.json()
|
|
1275
|
+
ingested = data.get("entities_ingested", 0)
|
|
1276
|
+
rejected = data.get("entities_rejected", 0)
|
|
1277
|
+
logger.info("mcp_published", extra={
|
|
1278
|
+
"case_id": report.case_id,
|
|
1279
|
+
"ingested": ingested,
|
|
1280
|
+
"rejected": rejected,
|
|
1281
|
+
})
|
|
1282
|
+
return True, None
|
|
1283
|
+
elif resp.status_code == 401:
|
|
1284
|
+
return False, "Invalid MCP API key"
|
|
1285
|
+
elif resp.status_code == 503:
|
|
1286
|
+
return False, "MCP server not configured (no API key set)"
|
|
1287
|
+
else:
|
|
1288
|
+
return False, f"MCP server returned {resp.status_code}"
|
|
1289
|
+
except Exception as e:
|
|
1290
|
+
logger.warning("mcp_publish_error: %s", e)
|
|
1291
|
+
return False, str(e)
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
@app.post("/api/cases/{case_id}/publish")
|
|
1295
|
+
async def publish_case(case_id: str):
|
|
1296
|
+
"""Publish an already-saved case to the MCP community graph.
|
|
1297
|
+
|
|
1298
|
+
Uses the engine's entity extraction for properly-typed entities.
|
|
1299
|
+
"""
|
|
1300
|
+
from pathlib import Path
|
|
1301
|
+
case_path = Path.home() / "watson-cases" / f"{case_id}.md"
|
|
1302
|
+
if not case_path.exists():
|
|
1303
|
+
raise HTTPException(404, f"Case {case_id} not found in archives")
|
|
1304
|
+
|
|
1305
|
+
# Reconstruct metadata and extract typed entities
|
|
1306
|
+
import re
|
|
1307
|
+
text = case_path.read_text()
|
|
1308
|
+
target = ""
|
|
1309
|
+
target_type = ""
|
|
1310
|
+
tm = re.search(r"\*\*Target:\*\*\s*(.+)", text)
|
|
1311
|
+
ttm = re.search(r"\*\*Target Type:\*\*\s*(.+)", text)
|
|
1312
|
+
fm = re.search(r"\*\*Findings:\*\*\s*(\d+)", text)
|
|
1313
|
+
cm = re.search(r"(\d+)\s+CONFIRMED", text)
|
|
1314
|
+
vm = re.search(r"\*\*Verifiability:\*\*\s*(.+)", text)
|
|
1315
|
+
dm = re.search(r"\*\*Date:\*\*\s*(.+)", text)
|
|
1316
|
+
|
|
1317
|
+
if tm: target = tm.group(1).strip()
|
|
1318
|
+
if ttm: target_type = ttm.group(1).strip()
|
|
1319
|
+
findings_count = int(fm.group(1)) if fm else 0
|
|
1320
|
+
confirmed_count = int(cm.group(1)) if cm else 0
|
|
1321
|
+
verifiability = vm.group(1).strip() if vm else ""
|
|
1322
|
+
date_str = dm.group(1).strip() if dm else ""
|
|
1323
|
+
|
|
1324
|
+
# Use engine's entity extraction — produces properly-typed entities
|
|
1325
|
+
from src.watson.orchestration.engine import OrchestrationEngine
|
|
1326
|
+
entities = OrchestrationEngine._extract_entities_from_text(text)
|
|
1327
|
+
|
|
1328
|
+
if not entities:
|
|
1329
|
+
return {"status": "no_entities", "case_id": case_id}
|
|
1330
|
+
|
|
1331
|
+
if not MCP_API_KEY and "localhost" not in MCP_SERVER_URL:
|
|
1332
|
+
raise HTTPException(400, "No MCP API key configured. Set MCP_API_KEY in env or run onboarding.")
|
|
1333
|
+
|
|
1334
|
+
try:
|
|
1335
|
+
import httpx
|
|
1336
|
+
payload = {
|
|
1337
|
+
"case_id": case_id,
|
|
1338
|
+
"target": target,
|
|
1339
|
+
"target_type": target_type,
|
|
1340
|
+
"findings_count": findings_count,
|
|
1341
|
+
"confirmed_count": confirmed_count,
|
|
1342
|
+
"verifiability": verifiability,
|
|
1343
|
+
"date": date_str,
|
|
1344
|
+
"entities": [{
|
|
1345
|
+
"value": e["value"],
|
|
1346
|
+
"type": e["type"],
|
|
1347
|
+
"source": "",
|
|
1348
|
+
"tier": "PROBABLE",
|
|
1349
|
+
"case_id": case_id,
|
|
1350
|
+
} for e in entities],
|
|
1351
|
+
}
|
|
1352
|
+
resp = httpx.post(f"{MCP_SERVER_URL}/api/ingest", json=payload, timeout=10,
|
|
1353
|
+
headers={"X-API-Key": MCP_API_KEY} if MCP_API_KEY else {})
|
|
1354
|
+
if resp.status_code == 200:
|
|
1355
|
+
data = resp.json()
|
|
1356
|
+
return {
|
|
1357
|
+
"status": "published",
|
|
1358
|
+
"case_id": case_id,
|
|
1359
|
+
"entities_sent": len(entities),
|
|
1360
|
+
"entities_ingested": data.get("entities_ingested", 0),
|
|
1361
|
+
"entities_rejected": data.get("entities_rejected", 0),
|
|
1362
|
+
}
|
|
1363
|
+
elif resp.status_code == 401:
|
|
1364
|
+
raise HTTPException(401, "Invalid MCP API key")
|
|
1365
|
+
elif resp.status_code == 503:
|
|
1366
|
+
raise HTTPException(503, "MCP server not configured")
|
|
1367
|
+
raise HTTPException(502, f"MCP server returned {resp.status_code}")
|
|
1368
|
+
except httpx.ConnectError:
|
|
1369
|
+
raise HTTPException(503, f"MCP server not reachable at {MCP_SERVER_URL}")
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
@app.get("/api/graph/status")
|
|
1373
|
+
async def graph_status():
|
|
1374
|
+
"""Check connection to the MCP community knowledge graph.
|
|
1375
|
+
|
|
1376
|
+
Returns connection status, whether a key is configured, and graph stats.
|
|
1377
|
+
Frontend uses this to show the connection indicator next to the publish checkbox.
|
|
1378
|
+
"""
|
|
1379
|
+
is_local = "localhost" in MCP_SERVER_URL or "127.0.0.1" in MCP_SERVER_URL
|
|
1380
|
+
has_key = bool(MCP_API_KEY)
|
|
1381
|
+
|
|
1382
|
+
status = {
|
|
1383
|
+
"mcp_url": MCP_SERVER_URL,
|
|
1384
|
+
"configured": has_key or is_local,
|
|
1385
|
+
"reason": "",
|
|
1386
|
+
"stats": None,
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
if is_local:
|
|
1390
|
+
status["reason"] = "Local graph — always available"
|
|
1391
|
+
elif not has_key:
|
|
1392
|
+
status["reason"] = "No MCP API key configured"
|
|
1393
|
+
return status
|
|
1394
|
+
|
|
1395
|
+
try:
|
|
1396
|
+
import httpx
|
|
1397
|
+
resp = httpx.get(
|
|
1398
|
+
f"{MCP_SERVER_URL}/api/stats",
|
|
1399
|
+
timeout=3,
|
|
1400
|
+
headers={"X-API-Key": MCP_API_KEY},
|
|
1401
|
+
)
|
|
1402
|
+
if resp.status_code == 200:
|
|
1403
|
+
status["stats"] = resp.json()
|
|
1404
|
+
status["connected"] = True
|
|
1405
|
+
else:
|
|
1406
|
+
status["connected"] = False
|
|
1407
|
+
status["reason"] = f"MCP server returned {resp.status_code}"
|
|
1408
|
+
except Exception as e:
|
|
1409
|
+
status["connected"] = False
|
|
1410
|
+
status["reason"] = str(e)
|
|
1411
|
+
|
|
1412
|
+
return status
|
|
1413
|
+
|
|
1414
|
+
|
|
1415
|
+
@app.get("/api/public/search")
|
|
1416
|
+
async def search_public_intel(q: str):
|
|
1417
|
+
"""Search the community knowledge graph for prior intelligence on a target."""
|
|
1418
|
+
try:
|
|
1419
|
+
import httpx
|
|
1420
|
+
resp = httpx.get(f"{MCP_SERVER_URL}/api/search", params={"q": q, "limit": 10}, timeout=5)
|
|
1421
|
+
if resp.status_code == 200:
|
|
1422
|
+
data = resp.json()
|
|
1423
|
+
return {
|
|
1424
|
+
"query": q,
|
|
1425
|
+
"results": data.get("results", []),
|
|
1426
|
+
"count": data.get("count", 0),
|
|
1427
|
+
}
|
|
1428
|
+
return {"query": q, "results": [], "count": 0, "error": f"MCP returned {resp.status_code}"}
|
|
1429
|
+
except Exception as e:
|
|
1430
|
+
return {"query": q, "results": [], "count": 0, "mcp_offline": True}
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
@app.get("/api/public/stats")
|
|
1434
|
+
async def public_intel_stats():
|
|
1435
|
+
"""Get community knowledge graph stats."""
|
|
1436
|
+
try:
|
|
1437
|
+
import httpx
|
|
1438
|
+
resp = httpx.get(f"{MCP_SERVER_URL}/api/stats", timeout=5)
|
|
1439
|
+
if resp.status_code == 200:
|
|
1440
|
+
return resp.json()
|
|
1441
|
+
return {"error": f"MCP returned {resp.status_code}"}
|
|
1442
|
+
except Exception as e:
|
|
1443
|
+
return {"error": str(e), "mcp_offline": True}
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
@app.get("/api/cases")
|
|
1447
|
+
async def list_cases(limit: int = 20, offset: int = 0):
|
|
1448
|
+
"""List all saved investigation cases."""
|
|
1449
|
+
from pathlib import Path
|
|
1450
|
+
import re
|
|
1451
|
+
|
|
1452
|
+
cases_dir = Path.home() / "watson-cases"
|
|
1453
|
+
cases = []
|
|
1454
|
+
for f in sorted(cases_dir.glob("CASE-*.md"), key=lambda x: x.stat().st_mtime, reverse=True):
|
|
1455
|
+
if not f.name.startswith("CASE-"):
|
|
1456
|
+
continue
|
|
1457
|
+
text = f.read_text()
|
|
1458
|
+
target = ""
|
|
1459
|
+
target_type = ""
|
|
1460
|
+
findings = 0
|
|
1461
|
+
confirmed = 0
|
|
1462
|
+
verifiability = ""
|
|
1463
|
+
date_match = re.search(r"\*\*Date:\*\*\s*(.+)", text)
|
|
1464
|
+
target_match = re.search(r"\*\*Target:\*\*\s*(.+)", text)
|
|
1465
|
+
type_match = re.search(r"\*\*Target Type:\*\*\s*(.+)", text)
|
|
1466
|
+
findings_match = re.search(r"\*\*Findings:\*\*\s*(\d+)", text)
|
|
1467
|
+
confirmed_match = re.search(r"(\d+)\s+CONFIRMED", text)
|
|
1468
|
+
verif_match = re.search(r"\*\*Verifiability:\*\*\s*(.+)", text)
|
|
1469
|
+
|
|
1470
|
+
if target_match: target = target_match.group(1).strip()
|
|
1471
|
+
if type_match: target_type = type_match.group(1).strip()
|
|
1472
|
+
if findings_match: findings = int(findings_match.group(1))
|
|
1473
|
+
if confirmed_match: confirmed = int(confirmed_match.group(1))
|
|
1474
|
+
if verif_match: verifiability = verif_match.group(1).strip()
|
|
1475
|
+
date_str = date_match.group(1).strip() if date_match else ""
|
|
1476
|
+
|
|
1477
|
+
cases.append({
|
|
1478
|
+
"id": f.stem,
|
|
1479
|
+
"target": target,
|
|
1480
|
+
"target_type": target_type,
|
|
1481
|
+
"date": date_str,
|
|
1482
|
+
"findings": findings,
|
|
1483
|
+
"confirmed": confirmed,
|
|
1484
|
+
"verifiability": verifiability,
|
|
1485
|
+
"size": f.stat().st_size,
|
|
1486
|
+
})
|
|
1487
|
+
|
|
1488
|
+
return {
|
|
1489
|
+
"cases": cases[offset:offset+limit],
|
|
1490
|
+
"total": len(cases),
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
@app.get("/api/cases/{case_id}")
|
|
1495
|
+
async def get_case(case_id: str):
|
|
1496
|
+
"""Get a specific case markdown."""
|
|
1497
|
+
from pathlib import Path
|
|
1498
|
+
case_path = Path.home() / "watson-cases" / f"{case_id}.md"
|
|
1499
|
+
if case_path.exists():
|
|
1500
|
+
from fastapi.responses import PlainTextResponse
|
|
1501
|
+
return PlainTextResponse(case_path.read_text(), media_type="text/markdown")
|
|
1502
|
+
raise HTTPException(404, f"Case {case_id} not found")
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
@app.get("/api/settings/keys")
|
|
1506
|
+
async def get_api_keys():
|
|
1507
|
+
"""List all configurable API keys with their status (values masked)."""
|
|
1508
|
+
from watson.api_keys import list_keys
|
|
1509
|
+
return {"keys": list_keys()}
|
|
1510
|
+
|
|
1511
|
+
|
|
1512
|
+
@app.post("/api/settings/keys")
|
|
1513
|
+
async def set_api_key(req: SetKeyRequest):
|
|
1514
|
+
"""Save an API key for a tool."""
|
|
1515
|
+
from watson.api_keys import set_key
|
|
1516
|
+
set_key(req.slug, req.value)
|
|
1517
|
+
return {"status": "ok", "slug": req.slug, "configured": bool(req.value)}
|
|
1518
|
+
|
|
1519
|
+
|
|
1520
|
+
@app.delete("/api/settings/keys/{slug}")
|
|
1521
|
+
async def delete_api_key(slug: str):
|
|
1522
|
+
"""Remove an API key."""
|
|
1523
|
+
from watson.api_keys import delete_key
|
|
1524
|
+
delete_key(slug)
|
|
1525
|
+
return {"status": "ok", "slug": slug, "configured": False}
|
|
1526
|
+
|
|
1527
|
+
|
|
1528
|
+
# ── Enterprise Exports ──────────────────────────────────────────
|
|
1529
|
+
|
|
1530
|
+
@app.get("/api/export/stix/{case_id}")
|
|
1531
|
+
async def export_stix_endpoint(case_id: str):
|
|
1532
|
+
"""Export a completed investigation as STIX 2.1 JSON bundle."""
|
|
1533
|
+
from pathlib import Path
|
|
1534
|
+
case_path = Path.home() / "watson-cases" / f"{case_id}.md"
|
|
1535
|
+
if not case_path.exists():
|
|
1536
|
+
# Try case_id with date suffix (CASE-XXX_YYYY-MM-DD)
|
|
1537
|
+
matches = list((Path.home() / "watson-cases").glob(f"{case_id}*.md"))
|
|
1538
|
+
if matches:
|
|
1539
|
+
case_path = matches[0]
|
|
1540
|
+
else:
|
|
1541
|
+
raise HTTPException(404, f"Case {case_id} not found")
|
|
1542
|
+
|
|
1543
|
+
try:
|
|
1544
|
+
from watson.serializers.stix import export_stix
|
|
1545
|
+
# Parse minimal info from case filename
|
|
1546
|
+
import re
|
|
1547
|
+
content = case_path.read_text()
|
|
1548
|
+
query_match = re.search(r'\*\*Target:\*\*\s*(.+?)$', content, re.MULTILINE)
|
|
1549
|
+
target_match = re.search(r'\*\*Target Type:\*\*\s*(.+?)$', content, re.MULTILINE)
|
|
1550
|
+
query = query_match.group(1).strip() if query_match else case_id
|
|
1551
|
+
target_type = target_match.group(1).strip() if target_match else "unknown"
|
|
1552
|
+
|
|
1553
|
+
# Build minimal findings from markdown
|
|
1554
|
+
findings = _parse_findings_from_markdown(content)
|
|
1555
|
+
|
|
1556
|
+
bundle, _ = export_stix(
|
|
1557
|
+
query=query,
|
|
1558
|
+
case_id=case_id,
|
|
1559
|
+
target_type=target_type,
|
|
1560
|
+
findings=findings,
|
|
1561
|
+
)
|
|
1562
|
+
return JSONResponse(content=bundle, media_type="application/json")
|
|
1563
|
+
except Exception as e:
|
|
1564
|
+
raise HTTPException(500, f"STIX export failed: {e}")
|
|
1565
|
+
|
|
1566
|
+
|
|
1567
|
+
@app.get("/api/opsec/stats")
|
|
1568
|
+
async def opsec_stats():
|
|
1569
|
+
"""OpSec proxy firewall statistics."""
|
|
1570
|
+
try:
|
|
1571
|
+
from watson.opsec import get_opsec_client
|
|
1572
|
+
client = get_opsec_client()
|
|
1573
|
+
return client.stats()
|
|
1574
|
+
except Exception as e:
|
|
1575
|
+
return {"status": "unavailable", "error": str(e)}
|
|
1576
|
+
|
|
1577
|
+
|
|
1578
|
+
# ── SPA fallback — serve React app for all non-API routes ──────────
|
|
1579
|
+
|
|
1580
|
+
@app.get("/{full_path:path}", response_class=HTMLResponse)
|
|
1581
|
+
async def spa_fallback(full_path: str):
|
|
1582
|
+
"""Serve the React SPA for all client-side routes.
|
|
1583
|
+
|
|
1584
|
+
Every URL that isn't an explicit API endpoint or static file
|
|
1585
|
+
returns index.html so React Router can handle it client-side.
|
|
1586
|
+
"""
|
|
1587
|
+
# Don't intercept API routes (shouldn't reach here if registered first, but safety)
|
|
1588
|
+
if full_path.startswith("api/"):
|
|
1589
|
+
raise HTTPException(404)
|
|
1590
|
+
index = STATIC_DIR / "index.html"
|
|
1591
|
+
if index.exists():
|
|
1592
|
+
return HTMLResponse(content=index.read_text(), headers=_NO_CACHE)
|
|
1593
|
+
return HTMLResponse(
|
|
1594
|
+
"<h1>Watson — run <code>cd frontend && npm run build</code></h1>",
|
|
1595
|
+
status_code=404,
|
|
1596
|
+
)
|
|
1597
|
+
|
|
1598
|
+
|
|
1599
|
+
def _parse_findings_from_markdown(content: str) -> list:
|
|
1600
|
+
"""Parse findings from a Watson markdown report for STIX export."""
|
|
1601
|
+
import re
|
|
1602
|
+
findings = []
|
|
1603
|
+
|
|
1604
|
+
# Find "Key Findings" section
|
|
1605
|
+
key_section = re.search(r'## Key Findings\s*\n(.*?)(?=\n## |\Z)', content, re.DOTALL)
|
|
1606
|
+
if not key_section:
|
|
1607
|
+
return findings
|
|
1608
|
+
|
|
1609
|
+
# Parse individual finding lines (bullet points with tier icons)
|
|
1610
|
+
finding_pattern = re.compile(
|
|
1611
|
+
r'-\s*(?:🟢|🟡|🟠|🔴|⚪)\s*\*\*(.+?)\*\*\s*\[(.+?)\]\s*\((\d+)% confidence\)\s*\n'
|
|
1612
|
+
r'\s*(.+?)(?=\n\s*(?:Source:|$))(?:\n\s*Source:\s*(.+?))?(?=\n- |\n## |\Z)',
|
|
1613
|
+
re.DOTALL,
|
|
1614
|
+
)
|
|
1615
|
+
|
|
1616
|
+
for m in finding_pattern.finditer(key_section.group(1)):
|
|
1617
|
+
title = m.group(1).strip()
|
|
1618
|
+
src_tier = m.group(2).strip()
|
|
1619
|
+
confidence_str = m.group(3)
|
|
1620
|
+
desc = m.group(4).strip()
|
|
1621
|
+
source_url = m.group(5).strip() if m.group(5) else ""
|
|
1622
|
+
|
|
1623
|
+
# Map source tier to Watson tier
|
|
1624
|
+
tier_map = {"PRIMARY": "CONFIRMED", "SECONDARY": "PROBABLE",
|
|
1625
|
+
"TERTIARY": "POSSIBLE"}
|
|
1626
|
+
tier = tier_map.get(src_tier, "POSSIBLE")
|
|
1627
|
+
|
|
1628
|
+
findings.append(SimpleFinding(
|
|
1629
|
+
title=title,
|
|
1630
|
+
description=desc[:500],
|
|
1631
|
+
tier=tier,
|
|
1632
|
+
source_url=source_url,
|
|
1633
|
+
source_type="osint",
|
|
1634
|
+
confidence=int(confidence_str) / 100,
|
|
1635
|
+
))
|
|
1636
|
+
|
|
1637
|
+
return findings
|
|
1638
|
+
|
|
1639
|
+
|
|
1640
|
+
class SimpleFinding:
|
|
1641
|
+
"""Minimal finding object for STIX export from markdown."""
|
|
1642
|
+
def __init__(self, title, description, tier, source_url, source_type, confidence):
|
|
1643
|
+
self.id = str(abs(hash(title)) % (10**12))
|
|
1644
|
+
self.title = title
|
|
1645
|
+
self.description = description
|
|
1646
|
+
self.tier = tier
|
|
1647
|
+
self.source_url = source_url
|
|
1648
|
+
self.source_type = source_type
|
|
1649
|
+
self.confidence = confidence
|
|
1650
|
+
self.entities = []
|