osintengine 1.0.2__py3-none-any.whl

src/watson/orchestration/executor.py

@@ -0,0 +1,96 @@
+"""Tool executor — retry wrappers, circuit breakers, graceful degradation."""
+
+from __future__ import annotations
+import asyncio
+import logging
+import time
+from typing import Any, Callable
+
+logger = logging.getLogger("watson.executor")
+
+# ── Circuit breaker ───────────────────────────────────────────
+
+class CircuitBreaker:
+    """Prevent hammering a failing API."""
+    
+    def __init__(self, name: str, failure_threshold: int = 3, cooldown: float = 60.0):
+        self.name = name
+        self.failure_threshold = failure_threshold
+        self.cooldown = cooldown
+        self._failures = 0
+        self._last_failure = 0.0
+        self._open = False
+    
+    @property
+    def is_open(self) -> bool:
+        if not self._open:
+            return False
+        if time.time() - self._last_failure > self.cooldown:
+            self._open = False
+            self._failures = 0
+            return False
+        return True
+    
+    def record_failure(self):
+        self._failures += 1
+        self._last_failure = time.time()
+        if self._failures >= self.failure_threshold:
+            self._open = True
+            logger.warning("circuit_breaker_open: %s", self.name)
+    
+    def record_success(self):
+        self._failures = 0
+        self._open = False
+
+
+# Global circuit breakers
+_circuits: dict[str, CircuitBreaker] = {}
+
+
+def get_circuit(name: str) -> CircuitBreaker:
+    if name not in _circuits:
+        _circuits[name] = CircuitBreaker(name)
+    return _circuits[name]
+
+
+# ── Retry wrapper ─────────────────────────────────────────────
+
+async def with_retry(
+    fn: Callable,
+    *args,
+    tool_name: str = "unknown",
+    max_retries: int = 2,
+    backoff: float = 2.0,
+    **kwargs,
+) -> Any:
+    """Call fn with retry + circuit breaker. Returns result or error finding."""
+    circuit = get_circuit(tool_name)
+    
+    if circuit.is_open:
+        return _error_finding(tool_name, "Circuit breaker open — API unavailable")
+    
+    last_error = None
+    for attempt in range(max_retries + 1):
+        try:
+            result = await fn(*args, **kwargs) if asyncio.iscoroutinefunction(fn) else fn(*args, **kwargs)
+            circuit.record_success()
+            return result
+        except Exception as e:
+            last_error = e
+            logger.warning("tool_attempt_failed: %s attempt %d: %s", tool_name, attempt + 1, e)
+            if attempt < max_retries:
+                await asyncio.sleep(backoff * (attempt + 1))
+    
+    circuit.record_failure()
+    return _error_finding(tool_name, str(last_error)[:200])
+
+
+def _error_finding(tool_name: str, error_msg: str) -> dict:
+    """Return a structured error finding — NOT None, so the pipeline knows the tool ran."""
+    return {
+        "title": f"⚠️ {tool_name}: API error",
+        "description": f"Tool '{tool_name}' failed: {error_msg}",
+        "source_type": "error",
+        "confidence": 0.0,
+        "evidence": [],
+    }

src/watson/orchestration/intent.py

@@ -0,0 +1,139 @@
+"""Intent classification — understand what the user is investigating."""
+
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Optional
+
+# ── Data model ────────────────────────────────────────────────
+
+@dataclass
+class Intent:
+    """What Watson understood about the investigation target."""
+    focus: str                          # natural-language focus area
+    target_type: str                    # person | company | domain | topic | wallet
+    confidence: float                   # classification confidence 0-1
+    entities: list[str] = field(default_factory=list)
+    categories: list[str] = field(default_factory=list)
+    raw_query: str = ""
+
+# ── LLM prompt for classification ─────────────────────────────
+
+_INTENT_PROMPT = """You are an OSINT target classifier. Given a search query, determine what is being investigated.
+
+Return STRICT JSON with these fields:
+{
+  "target_type": "person|company|domain|topic|wallet|ip|email",
+  "focus": "short description of what aspect to investigate (1 sentence)",
+  "confidence": 0.0-1.0,
+  "categories": ["list of tool categories to use: social, corporate, dark, crypto, geo, media, recon, vision"],
+  "entities": ["list of named entities found in the query"]
+}
+
+ENTITY TYPE RULES:
+- Classify each entity independently based on context
+- People have names, DOBs, nationalities, aliases, professions
+- Companies have suffixes (Inc, LLC, Ltd, Corp, GmbH, SA, AG, Group, Holdings) or are known orgs
+- Entities with version numbers (e.g., "LockBit 3.0") are NEVER typed as person
+- Email addresses, crypto wallets, IPs, domains have their own types
+- If ambiguous, prefer the more specific type
+
+Query: {query}
+
+JSON:"""
+
+# ── Deterministic fallback — no LLM needed for obvious cases ──
+
+def _deterministic_classify(query: str) -> Optional[Intent]:
+    """Quick classification without LLM for obvious cases."""
+    q = query.strip().lower()
+    
+    # Email
+    if "@" in q and "." in q.split("@")[-1]:
+        return Intent(focus=f"Email investigation of {q}", target_type="email",
+                      confidence=0.95, entities=[query.strip()],
+                      categories=["social", "recon"], raw_query=query)
+    
+    # Crypto wallet
+    if q.startswith("0x") and len(q) >= 40:
+        return Intent(focus=f"Cryptocurrency wallet {q[:10]}...", target_type="wallet",
+                      confidence=0.95, entities=[q], categories=["crypto", "dark"],
+                      raw_query=query)
+    
+    # IP address
+    parts = q.split(".")
+    if len(parts) == 4 and all(p.isdigit() and 0 <= int(p) <= 255 for p in parts):
+        return Intent(focus=f"IP address investigation of {q}", target_type="ip",
+                      confidence=0.95, entities=[q], categories=["geo", "dark"],
+                      raw_query=query)
+    
+    # Domain
+    if "." in q and not " " in q and not q.startswith("http"):
+        if any(q.endswith(t) for t in [".com", ".org", ".net", ".io", ".gov", ".ru", ".cn", ".de"]):
+            return Intent(focus=f"Domain investigation of {q}", target_type="domain",
+                          confidence=0.85, entities=[q],
+                          categories=["recon", "dark", "corporate"],
+                          raw_query=query)
+    
+    return None
+
+# ── LLM fallback classifier ──────────────────────────────────
+
+async def _llm_classify(query: str, call_llm) -> Intent:
+    """Use LLM to classify the query intent."""
+    prompt = _INTENT_PROMPT.format(query=query[:500])
+    
+    try:
+        raw = await call_llm(prompt, timeout=30)
+    except Exception:
+        raw = None
+    
+    if not raw:
+        return Intent(focus=query, target_type="topic", confidence=0.3,
+                      entities=[], categories=["recon", "social"],
+                      raw_query=query)
+    
+    import json, re
+    # Strip markdown fences
+    raw = raw.strip()
+    if raw.startswith("```"):
+        raw = re.sub(r"^```(?:json)?\s*", "", raw)
+        raw = re.sub(r"\s*```$", "", raw)
+    
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        m = re.search(r"\{.*\}", raw, re.DOTALL)
+        if m:
+            try:
+                parsed = json.loads(m.group(0))
+            except json.JSONDecodeError:
+                parsed = {}
+        else:
+            parsed = {}
+    
+    target_type = str(parsed.get("target_type", "topic")).lower()
+    if target_type not in ("person", "company", "domain", "topic", "wallet", "ip", "email"):
+        target_type = "topic"
+    
+    return Intent(
+        focus=str(parsed.get("focus", query)),
+        target_type=target_type,
+        confidence=float(parsed.get("confidence", 0.5)),
+        entities=[str(e) for e in parsed.get("entities", []) if str(e).strip()],
+        categories=[str(c) for c in parsed.get("categories", ["recon", "social"])],
+        raw_query=query,
+    )
+
+# ── Main entry point ─────────────────────────────────────────
+
+async def classify_intent(query: str, call_llm=None) -> Intent:
+    """Classify investigation intent. Uses deterministic rules first, LLM as fallback."""
+    det = _deterministic_classify(query)
+    if det:
+        return det
+    
+    if call_llm:
+        return await _llm_classify(query, call_llm)
+    
+    return Intent(focus=query, target_type="topic", confidence=0.4,
+                  entities=[], categories=["recon", "social"], raw_query=query)

src/watson/orchestration/intent_classifier.py

@@ -0,0 +1,45 @@
+"""Intent classifier — determines if a message is an investigation or chat."""
+
+from __future__ import annotations
+
+from typing import Callable, Coroutine, Tuple
+
+
+async def classify_intent(
+    msg: str,
+    call_llm: Callable[..., Coroutine],
+) -> Tuple[str, float, str]:
+    """Classify a message as 'investigate' or 'chat' using LLM fallback.
+
+    Args:
+        msg: The user message to classify.
+        call_llm: Async function to call the LLM.
+
+    Returns:
+        Tuple of (intent: str, confidence: float, reason: str)
+    """
+    prompt = (
+        "Classify this message as 'investigate' or 'chat'.\n"
+        "'investigate' means the user wants to investigate a person, domain, company, "
+        "crypto address, IP, email, or similar target.\n"
+        "'chat' means it's a conversational question about OSINT, methodology, or general knowledge.\n\n"
+        f"Message: \"{msg[:500]}\"\n\n"
+        "Respond with only one word: investigate or chat"
+    )
+
+    try:
+        result = await call_llm(
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=10,
+            temperature=0.0,
+        )
+        answer = result.strip().lower()
+        if "investigate" in answer:
+            return "investigate", 0.65, "llm_classifier"
+        elif "chat" in answer:
+            return "chat", 0.65, "llm_classifier"
+        else:
+            # Default: ambiguous → treat as chat for safety
+            return "chat", 0.5, "llm_ambiguous"
+    except Exception:
+        return "chat", 0.5, "llm_unavailable"

src/watson/orchestration/llm_config.py

@@ -0,0 +1,160 @@
+"""LLM configuration — call any OpenAI-compatible API."""
+
+from __future__ import annotations
+import os, json, logging, re
+
+logger = logging.getLogger("watson.llm")
+
+# ── Provider config ──────────────────────────────────────────
+
+_PROVIDER_CONFIG = {
+    "deepseek": {
+        "base_url": "https://api.deepseek.com/v1",
+        "api_key_env": "DEEPSEEK_API_KEY",
+        "default_model": os.environ.get("DEEPSEEK_MODEL", "deepseek-v4-pro"),
+    },
+    "openai": {
+        "base_url": "https://api.openai.com/v1",
+        "api_key_env": "OPENAI_API_KEY",
+        "default_model": "gpt-4o",
+    },
+    "anthropic": {
+        "base_url": "https://api.anthropic.com/v1",
+        "api_key_env": "ANTHROPIC_API_KEY",
+        "default_model": "claude-sonnet-4-20250514",
+    },
+    "hermes": {
+        "base_url": os.environ.get("HERMES_API_BASE", "http://localhost:8080/v1"),
+        "api_key_env": "HERMES_API_KEY",
+        "default_model": "hermes",
+    },
+    "openrouter": {
+        "base_url": "https://openrouter.ai/api/v1",
+        "api_key_env": "OPENROUTER_API_KEY",
+        "default_model": "deepseek/deepseek-chat",
+    },
+}
+
+# ── Main call function ───────────────────────────────────────
+
+def _auto_detect_provider() -> str:
+    """Auto-detect which LLM provider has an API key configured.
+    
+    Checks environment variables and the Watson key store.
+    Returns the first available provider name, or '' if none found.
+    """
+    # Check env vars for each provider
+    for provider_name, cfg in _PROVIDER_CONFIG.items():
+        if os.environ.get(cfg["api_key_env"], ""):
+            return provider_name
+    
+    # Check the Watson UI key store
+    try:
+        from watson.api_keys import list_keys
+        stored = list_keys()
+        for entry in stored:
+            if entry.get("configured") and entry.get("slug") in _PROVIDER_CONFIG:
+                return entry["slug"]
+    except (ImportError, Exception):
+        pass
+    
+    return ""
+
+async def call_llm(
+    prompt: str,
+    timeout: int = 60,
+    max_tokens: int = 2048,
+    model: str | None = None,
+    provider: str | None = None,
+    system: str = "",
+) -> str | None:
+    """Call an LLM via OpenAI-compatible API. Returns text or None."""
+    
+    # Resolve provider: explicit > env var > auto-detect
+    if not provider:
+        provider = os.environ.get("WATSON_LLM_PROVIDER", "")
+    if not provider:
+        provider = _auto_detect_provider()
+    if not provider:
+        logger.warning(
+            "llm: no LLM provider configured. Set one of: "
+            "DEEPSEEK_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY, "
+            "OPENROUTER_API_KEY, or WATSON_LLM_PROVIDER. "
+            "See .env.example for setup instructions."
+        )
+        return None
+    
+    cfg = _PROVIDER_CONFIG.get(provider)
+    if not cfg:
+        logger.warning("llm: unknown provider '%s' — known: %s", provider, list(_PROVIDER_CONFIG.keys()))
+        return None
+    
+    api_key = os.environ.get(cfg["api_key_env"], "")
+    
+    # Fallback: check the Watson key store (user-configured via UI)
+    if not api_key:
+        try:
+            from watson.api_keys import get_key
+            api_key = get_key(provider) or ""
+        except ImportError:
+            pass
+    
+    if not api_key:
+        logger.warning("llm: no API key for provider %s (env: %s)", provider, cfg["api_key_env"])
+        return None
+    
+    if not model:
+        model = cfg["default_model"]
+    
+    url = f"{cfg['base_url']}/chat/completions"
+    
+    messages = []
+    if system:
+        messages.append({"role": "system", "content": system})
+    messages.append({"role": "user", "content": prompt})
+    
+    payload = {
+        "model": model,
+        "messages": messages,
+        "max_tokens": max_tokens,
+        "temperature": 0.3,
+    }
+    
+    try:
+        import httpx
+        async with httpx.AsyncClient(timeout=httpx.Timeout(timeout, connect=15.0)) as client:
+            resp = await client.post(
+                url,
+                json=payload,
+                headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                msg = data["choices"][0]["message"]
+                content = msg.get("content", "") or ""
+                # Reasoning models (deepseek-v4-pro, R1) put thinking in reasoning_content
+                # and the final answer in content. If content is empty (all tokens spent
+                # on reasoning), fall back to reasoning_content.
+                if not content.strip():
+                    reasoning = msg.get("reasoning_content", "") or ""
+                    if reasoning.strip():
+                        logger.debug("llm: using reasoning_content (content was empty)")
+                        return reasoning
+                    # Both empty — log response for debugging
+                    logger.warning("llm: empty content AND empty reasoning_content — finish_reason=%s model=%s",
+                                   data["choices"][0].get("finish_reason", "?"), model)
+                    # Check if there's any text anywhere in the response
+                    all_keys = list(msg.keys())
+                    logger.debug("llm: message keys: %s", all_keys)
+                    for k in all_keys:
+                        v = msg.get(k, "")
+                        if isinstance(v, str) and v.strip():
+                            logger.info("llm: found text in key '%s', using that", k)
+                            return v
+                return content
+            logger.warning("llm: HTTP %d from %s — body: %s", resp.status_code, provider,
+                           resp.text[:300] if resp.text else "(empty)")
+            return None
+    except Exception as e:
+        logger.warning("llm: request failed: %s", e)
+        return None