osintengine 1.0.2__py3-none-any.whl

src/watson/verification/__init__.py

@@ -0,0 +1,245 @@
+"""LLM Verification Layer — second-pass validation to filter false positives.
+
+After Phase 6 synthesis, each finding is re-examined by the LLM with a strict
+verification prompt. The LLM checks:
+  1. Does the source URL actually support the finding?
+  2. Is the tier (CONFIRMED/PROBABLE/POSSIBLE) appropriate?
+  3. Is anything hallucinated (fake entities, fabricated URLs, impossible claims)?
+  4. Should the finding be downgraded or dropped?
+
+Output: adjusted tier + verification_notes for each finding.
+
+Thresholds:
+  - Drop findings where verification_confidence < 30
+  - Downgrade findings where LLM flags overconfidence
+  - Keep findings that pass verification unchanged
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger("watson.verify")
+
+# ── Verification prompt ───────────────────────────────────────
+
+_VERIFY_SYSTEM = """You are a strict OSINT quality-assurance auditor. Your job is to validate
+intelligence findings and catch false positives, hallucinations, and overconfident claims.
+
+For each finding, check:
+1. Does the source URL actually exist and support the claim? If the URL looks fabricated
+   (random-looking domain, 404 patterns, impossible paths), flag it.
+2. Is the confidence tier appropriate? CONFIRMED needs multi-source corroboration.
+   PROBABLE needs a credible source. POSSIBLE is for weak signals.
+3. Are there any hallucinated elements? Invented person names, fake organizations,
+   impossible technical details, fabricated numbers/SAR IDs.
+4. Is the finding logically coherent? Does the description actually match the title?
+
+Respond with ONLY a JSON object. No markdown, no explanation outside the JSON."""
+
+_VERIFY_PROMPT = """Review these OSINT findings for quality. Return a JSON dict with:
+- "verifications": list of { "finding_id": str, "pass": bool, "adjusted_tier": str,
+  "verification_confidence": int (0-100), "issues": [str], "notes": str }
+
+Findings to verify:
+{findings_json}
+
+Rules:
+- CONFIRMED tier requires: real source URL + specific verifiable claim + no hallucination risk
+- PROBABLE tier requires: credible source URL + plausible claim
+- POSSIBLE tier: weak signal, correlation, or unverified mention
+- UNLIKELY/UNVERIFIED: should be dropped (pass=false)
+- If source URL doesn't look real, flag and downgrade
+- If finding contains invented names/numbers, drop it
+- If description is generic/vague, downgrade to POSSIBLE
+- If finding is about a different entity than the target, flag it
+
+Return ONLY valid JSON."""
+
+
+# ── Verification engine ───────────────────────────────────────
+
+class FindingVerifier:
+    """Second-pass LLM verification for OSINT findings."""
+
+    def __init__(self, call_llm=None):
+        """call_llm: async function (prompt, timeout, max_tokens) → str | None."""
+        self._call_llm = call_llm
+
+    async def verify(
+        self,
+        findings: list[Any],
+        query: str = "",
+        batch_size: int = 15,
+    ) -> list[dict]:
+        """Verify findings in batches. Returns list of verification results."""
+        if not findings or not self._call_llm:
+            return []
+
+        results: list[dict] = []
+
+        # Batch findings to avoid token overflow
+        for i in range(0, len(findings), batch_size):
+            batch = findings[i : i + batch_size]
+            batch_results = await self._verify_batch(batch, query)
+            results.extend(batch_results)
+
+        return results
+
+    async def _verify_batch(self, findings: list[Any], query: str) -> list[dict]:
+        """Verify a batch of findings."""
+        if not self._call_llm:
+            return []
+        # Serialize findings to compact JSON
+        findings_data = []
+        for f in findings:
+            fid = getattr(f, "id", str(hash(f)))
+            findings_data.append({
+                "id": fid,
+                "title": getattr(f, "title", "")[:200],
+                "description": getattr(f, "description", "")[:300],
+                "tier": getattr(f, "tier", "UNVERIFIED"),
+                "source_url": getattr(f, "source_url", "")[:200],
+                "source_type": getattr(f, "source_type", "osint"),
+                "confidence": getattr(f, "confidence", 0.5),
+            })
+
+        prompt = _VERIFY_PROMPT.format(
+            findings_json=json.dumps(findings_data, indent=2)
+        )
+
+        try:
+            raw = await self._call_llm(
+                prompt,
+                timeout=120,
+                max_tokens=4096,
+                system=_VERIFY_SYSTEM,
+            )
+            if not raw or not raw.strip():
+                logger.warning("verify: LLM returned empty — skipping verification")
+                return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
+                         "adjusted_tier": fd["tier"], "issues": [], "notes": "Verification skipped (LLM unavailable)"}
+                        for fd in findings_data]
+
+            parsed = self._parse_response(raw)
+            return self._merge_results(findings_data, parsed)
+
+        except Exception as e:
+            logger.warning("verify: batch failed: %s", e)
+            return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
+                     "adjusted_tier": fd["tier"], "issues": [],
+                     "notes": f"Verification error: {e}"}
+                    for fd in findings_data]
+
+    def _parse_response(self, raw: str) -> dict:
+        """Parse LLM verification response."""
+        # Strip markdown fences
+        text = raw.strip()
+        if text.startswith("```"):
+            text = re.sub(r"^```(?:json)?\s*", "", text)
+            text = re.sub(r"\s*```$", "", text)
+
+        # Try JSON parse
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            # Try to extract JSON object
+            m = re.search(r"\{.*\}", text, re.DOTALL)
+            if m:
+                try:
+                    return json.loads(m.group(0))
+                except json.JSONDecodeError:
+                    pass
+
+        logger.warning("verify: could not parse response: %s", raw[:200])
+        return {}
+
+    def _merge_results(self, findings_data: list[dict], parsed: dict) -> list[dict]:
+        """Merge parsed verifications with original finding data."""
+        verifications = parsed.get("verifications", [])
+        if not verifications:
+            # No structured verifications — pass everything
+            return [{"finding_id": fd["id"], "pass": True, "verification_confidence": 50,
+                     "adjusted_tier": fd["tier"], "issues": [], "notes": "No verification data"}
+                    for fd in findings_data]
+
+        # Build lookup
+        vmap: dict[str, dict] = {v.get("finding_id", ""): v for v in verifications}
+
+        results = []
+        for fd in findings_data:
+            fid = fd["id"]
+            v = vmap.get(fid, {})
+            results.append({
+                "finding_id": fid,
+                "pass": v.get("pass", True),
+                "verification_confidence": v.get("verification_confidence", 50),
+                "adjusted_tier": v.get("adjusted_tier", fd["tier"]),
+                "issues": v.get("issues", []),
+                "notes": v.get("notes", ""),
+            })
+        return results
+
+    # ── Apply verification results to findings ────────────────
+
+    @staticmethod
+    def apply(finding: Any, verification: dict) -> tuple[Any, bool]:
+        """Apply verification to a finding.
+
+        Returns (possibly_modified_finding, was_dropped).
+        """
+        if not verification:
+            return finding, False
+
+        vconf = verification.get("verification_confidence", 50)
+        passed = verification.get("pass", True)
+        issues = verification.get("issues", [])
+
+        # Drop findings with very low verification confidence
+        if vconf < 30:
+            logger.info("verify: dropping finding %s (confidence=%d, issues=%s)",
+                        getattr(finding, "id", "?"), vconf, issues)
+            return finding, True
+
+        # Downgrade tier if LLM suggests it
+        adjusted = verification.get("adjusted_tier", "")
+        if adjusted and hasattr(finding, "tier"):
+            tier_order = ["CONFIRMED", "PROBABLE", "POSSIBLE", "UNLIKELY", "UNVERIFIED"]
+            old_idx = tier_order.index(finding.tier) if finding.tier in tier_order else 4
+            new_idx = tier_order.index(adjusted) if adjusted in tier_order else old_idx
+            if new_idx > old_idx:
+                logger.info("verify: downgrading finding %s %s→%s",
+                            getattr(finding, "id", "?"), finding.tier, adjusted)
+                finding.tier = adjusted
+
+        # Attach verification notes
+        notes = verification.get("notes", "")
+        if notes and hasattr(finding, "description"):
+            finding.description = f"[Verified: {notes}] {finding.description}"
+
+        if issues and hasattr(finding, "description"):
+            finding.description = f"[⚠ Issues: {'; '.join(issues[:3])}] {finding.description}"
+
+        return finding, False
+
+
+# ── Factory ───────────────────────────────────────────────────
+
+def create_verifier(provider: str | None = None) -> FindingVerifier | None:
+    """Create a verifier using the configured LLM provider.
+
+    Returns None if no LLM is available.
+    """
+    try:
+        from watson.orchestration.llm_config import call_llm
+
+        async def _call(prompt, timeout=120, max_tokens=2000, system=""):
+            return await call_llm(prompt, timeout=timeout, max_tokens=max_tokens, system=system)
+
+        return FindingVerifier(call_llm=_call)
+    except ImportError:
+        logger.warning("verify: could not import llm_config — verification disabled")
+        return None

watson/__init__.py

@@ -0,0 +1,6 @@
+"""Watson — the OSINT investigation engine.
+
+Bellingcat-inspired. Graph-native. Agent-agnostic.
+"""
+
+__version__ = "0.1.0"

watson/agents/__init__.py

@@ -0,0 +1,20 @@
+"""Agent adapters for Watson — pluggable backends."""
+
+from .base import AgentAdapter, BrowseResult, InvestigationResult, SearchResult, TerminalResult, VisionResult
+from .hermes import HermesCLIAdapter
+from .hermes_mcp import HermesMCPAdapter
+
+# Legacy alias — kept for backward compatibility
+HermesAdapter = HermesCLIAdapter
+
+__all__ = [
+    "AgentAdapter",
+    "HermesAdapter",         # Legacy alias
+    "HermesCLIAdapter",      # CLI subprocess backend
+    "HermesMCPAdapter",      # MCP protocol backend
+    "SearchResult",
+    "BrowseResult",
+    "VisionResult",
+    "TerminalResult",
+    "InvestigationResult",
+]

watson/agents/base.py

@@ -0,0 +1,103 @@
+"""
+Agent adapter — abstract interface for pluggable agent backends.
+Watson is agent-agnostic: Hermes, OpenClaw, OpenHuman, or direct LLM.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class SearchResult:
+    title: str
+    url: str
+    snippet: str = ""
+    source: str = ""
+
+
+@dataclass
+class BrowseResult:
+    url: str
+    content: str
+    title: str = ""
+
+
+@dataclass
+class VisionResult:
+    description: str
+    objects: list[str] = field(default_factory=list)
+    text: str = ""
+
+
+@dataclass
+class TerminalResult:
+    output: str
+    exit_code: int = 0
+
+
+@dataclass
+class InvestigationResult:
+    """Structured result from an investigation angle."""
+    angle: str
+    findings: list[dict] = field(default_factory=list)
+    sources: list[str] = field(default_factory=list)
+    confidence: float = 0.0
+    raw: str = ""
+
+
+class AgentAdapter(ABC):
+    """Pluggable agent backend for Watson.
+
+    Watson's engine calls these abstract methods. Each adapter
+    implements them for its specific agent. Watson's methodology,
+    dispatch logic, graph, and reporting are independent of this layer.
+    """
+
+    name: str = "base"
+    description: str = ""
+
+    @abstractmethod
+    async def search(self, query: str, num_results: int = 10) -> list[SearchResult]:
+        """Web search."""
+        ...
+
+    @abstractmethod
+    async def browse(self, url: str) -> BrowseResult:
+        """Navigate to and extract content from a URL."""
+        ...
+
+    @abstractmethod
+    async def vision(self, image_path: str, question: str = "Describe this image in detail.") -> VisionResult:
+        """Analyze an image."""
+        ...
+
+    @abstractmethod
+    async def terminal(self, command: str, timeout: int = 30) -> TerminalResult:
+        """Execute a shell command."""
+        ...
+
+    async def investigate_angle(self, angle: str, query: str) -> InvestigationResult:
+        """Run a single investigation angle. Default: search + browse top results.
+
+        Adapters can override this for richer investigation (multi-tool dispatch).
+        """
+        results = await self.search(query, num_results=5)
+        sources = [r.url for r in results if r.url]
+        findings = [
+            {"title": r.title, "url": r.url, "snippet": r.snippet}
+            for r in results
+        ]
+
+        return InvestigationResult(
+            angle=angle,
+            findings=findings,
+            sources=sources,
+            confidence=0.5,
+        )
+
+    async def health_check(self) -> bool:
+        """Check if the agent backend is reachable."""
+        return True

watson/agents/direct.py

@@ -0,0 +1,225 @@
+"""
+Direct LLM adapter — any OpenAI-compatible API.
+Uses DuckDuckGo via duckduckgo_search library for real web search.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import re
+
+import aiohttp
+
+from .base import (
+    AgentAdapter,
+    BrowseResult,
+    InvestigationResult,
+    SearchResult,
+    TerminalResult,
+    VisionResult,
+)
+
+DEFAULT_API_BASE = "https://api.openai.com/v1"
+
+
+class DirectAdapter(AgentAdapter):
+    """LLM-agnostic adapter — any OpenAI-compatible API + DuckDuckGo search."""
+
+    name = "direct"
+    description = "Any OpenAI-compatible API + DuckDuckGo search"
+
+    def __init__(
+        self,
+        api_key: str = "",
+        model: str = "gpt-4o",
+        api_base: str | None = None,
+    ):
+        self.api_key = api_key or os.environ.get("WATSON_API_KEY", "")
+        self.model = model or os.environ.get("WATSON_MODEL", "gpt-4o")
+        self.api_base = (
+            api_base
+            or os.environ.get("WATSON_API_BASE")
+            or DEFAULT_API_BASE
+        )
+        self._ddgs = None
+
+    @property
+    def ddgs(self):
+        """Lazy-load DuckDuckGo search client."""
+        if self._ddgs is None:
+            from ddgs import DDGS
+            self._ddgs = DDGS()
+        return self._ddgs
+
+    async def _call_llm(
+        self, prompt: str, system: str = "", max_tokens: int = 4000
+    ) -> str:
+        """Call the LLM API."""
+        messages = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                f"{self.api_base}/chat/completions",
+                json={
+                    "model": self.model,
+                    "messages": messages,
+                    "temperature": 0.4,
+                    "max_tokens": max_tokens,
+                },
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f"Bearer {self.api_key}",
+                },
+                timeout=aiohttp.ClientTimeout(total=60),
+            ) as resp:
+                if resp.status != 200:
+                    text = await resp.text()
+                    raise RuntimeError(f"LLM API error ({resp.status}): {text}")
+                data = await resp.json()
+                return data["choices"][0]["message"]["content"]
+
+    async def _ddg_search(self, query: str, num_results: int = 10) -> list[SearchResult]:
+        """Real web search via DuckDuckGo."""
+        results: list[SearchResult] = []
+        try:
+            loop = asyncio.get_running_loop()
+            raw_results = await loop.run_in_executor(
+                None,
+                lambda: list(self.ddgs.text(query, max_results=num_results))
+            )
+            for r in raw_results:
+                results.append(SearchResult(
+                    title=r.get("title", ""),
+                    url=r.get("href", r.get("url", "")),
+                    snippet=r.get("body", r.get("snippet", "")),
+                    source="duckduckgo",
+                ))
+        except Exception:
+            pass
+        return results
+
+    async def search(self, query: str, num_results: int = 10) -> list[SearchResult]:
+        """Web search via DuckDuckGo only. No LLM fallback — only real data."""
+        return await self._ddg_search(query, num_results)
+
+    async def browse(self, url: str) -> BrowseResult:
+        """Fetch URL content via HTTP."""
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(
+                    url,
+                    headers={"User-Agent": "Mozilla/5.0 (compatible; Watson-OSINT/1.0)"},
+                    timeout=aiohttp.ClientTimeout(total=15),
+                ) as resp:
+                    text = await resp.text()
+                    # Strip HTML tags for plain text
+                    clean = re.sub(r"<[^>]+>", " ", text)
+                    clean = re.sub(r"\s+", " ", clean)
+                    return BrowseResult(
+                        url=url,
+                        content=clean[:5000],
+                        title=url,
+                    )
+        except Exception:
+            return BrowseResult(url=url, content=f"[Could not fetch {url}]", title=url)
+
+    async def vision(self, image_path: str, question: str = "Describe this image in detail.") -> VisionResult:
+        """LLM cannot analyze images."""
+        return VisionResult(
+            description=f"[Direct adapter cannot analyze images. Use Hermes for vision.]",
+        )
+
+    async def terminal(self, command: str, timeout: int = 30) -> TerminalResult:
+        """Execute commands via subprocess (available in Direct mode too)."""
+        try:
+            proc = await asyncio.create_subprocess_shell(
+                command,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+            output = stdout.decode("utf-8", errors="replace")
+            if stderr:
+                output += "\n" + stderr.decode("utf-8", errors="replace")
+            return TerminalResult(
+                output=output.strip() or "(no output)",
+                exit_code=proc.returncode or 0,
+            )
+        except asyncio.TimeoutError:
+            return TerminalResult(output="Command timed out", exit_code=1)
+        except Exception as e:
+            return TerminalResult(output=str(e), exit_code=1)
+
+    async def investigate_angle(self, angle: str, query: str) -> InvestigationResult:
+        """Analyze real search results for investigation findings.
+
+        Always performs DDG web search (no API key needed).
+        With LLM API key: enriches results with AI analysis.
+        Without: returns raw DDG results directly — never returns empty.
+        """
+        # Get real search results (no API key needed)
+        search_results = await self._ddg_search(query, num_results=5)
+        if not search_results:
+            return InvestigationResult(
+                angle=angle, confidence=0.0,
+                raw="No search results available for this angle."
+            )
+
+        # Without LLM, return raw DDG results as findings
+        if not self.api_key:
+            raw = "\n".join(
+                f"• {r.title}\n  {r.snippet[:200]}\n  {r.url}"
+                for r in search_results
+            )
+            return InvestigationResult(
+                angle=angle,
+                raw=raw,
+                findings=[
+                    {"title": r.title, "snippet": r.snippet, "url": r.url}
+                    for r in search_results
+                ],
+                sources=[r.url for r in search_results if r.url],
+                confidence=0.4,
+            )
+
+        # With LLM: enrich with AI analysis
+        context = "\n".join(
+            f"- {r.title}: {r.snippet} ({r.url})"
+            for r in search_results
+        )
+
+        try:
+            response = await self._call_llm(
+                prompt=(
+                    f"ANALYZE these search results for an OSINT investigation.\n\n"
+                    f"Angle: {angle}\n"
+                    f"Query: {query}\n\n"
+                    f"SEARCH RESULTS:\n{context}\n\n"
+                    f"Extract the key facts from these results. Return:\n"
+                    f"1. A 1-sentence summary of what was found\n"
+                    f"2. 3-5 specific findings, each with the source URL\n"
+                    f"3. Names, dates, and key facts mentioned\n"
+                    f"4. What's missing or needs deeper investigation\n\n"
+                    f"Be concise. Use ONLY information from the search results above."
+                ),
+                system=(
+                    "You analyze search results for OSINT investigations. "
+                    "Extract facts from provided results only. Do not invent or plan — analyze."
+                ),
+                max_tokens=2000,
+            )
+
+            urls = [r.url for r in search_results if r.url]
+            return InvestigationResult(
+                angle=angle,
+                raw=response,
+                sources=urls,
+                confidence=0.5 if len(response) > 200 else 0.3,
+            )
+        except Exception as e:
+            return InvestigationResult(angle=angle, confidence=0.0, raw=str(e))