osintengine 1.0.2__py3-none-any.whl

src/watson/agent/__init__.py

@@ -0,0 +1,96 @@
+"""OSINT Agent — top-level investigation agent.
+
+Delegates to the orchestration engine for the full pipeline:
+classify → burst → resolve → cross-ref → synthesize → continue.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, Callable, Dict, List, Optional
+
+
+class OSINTAgent:
+    """High-level OSINT investigation agent with full reasoning pipeline."""
+
+    def __init__(self, depth: int = 2, api_keys: Optional[Dict[str, str]] = None):
+        self.depth = max(1, min(depth, 5))
+        self.api_keys = api_keys or {}
+
+    async def investigate(
+        self,
+        query: str,
+        context: str = "",
+        on_event: Optional[Callable[[str, Any], None]] = None,
+        image_path: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """Run a full investigation through the orchestration engine."""
+        
+        client_id = f"agent-{uuid.uuid4().hex[:8]}"
+        events: List[Dict[str, Any]] = []
+        
+        def collect_event(event_type: str, data: Any):
+            events.append({"event": event_type, "data": data})
+            if on_event:
+                on_event(event_type, data)
+        
+        try:
+            from src.watson.orchestration import get_engine
+            
+            engine = get_engine(depth=self.depth)
+            brief = await engine.investigate(
+                query=query,
+                focus=context,
+                on_event=collect_event,
+                client_id=client_id,
+                depth=self.depth,
+            )
+            
+            if brief:
+                events.append({"event": "brief", "data": brief})
+                
+        except Exception as e:
+            # Fallback: use the orchestration engine directly
+            try:
+                from src.watson.orchestration import get_engine
+                
+                engine = get_engine()
+                result = await engine.investigate(query)
+                
+                if result and result.get("findings"):
+                    for f in result["findings"]:
+                        events.append({"event": "finding", "data": {
+                            "title": f.title,
+                            "description": f.description[:500],
+                            "source_type": f.source_type,
+                            "confidence": f.confidence,
+                            "tier": "CONFIRMED" if f.confidence >= 0.9 else
+                                   ("PROBABLE" if f.confidence >= 0.7 else "POSSIBLE"),
+                        }})
+                        if on_event:
+                            on_event("finding", events[-1]["data"])
+            except Exception as e2:
+                events.append({"event": "error", "data": {
+                    "message": f"Investigation failed: {e2}",
+                    "query": query,
+                }})
+        
+        if not events:
+            events.append({"event": "finding", "data": {
+                "title": f"Investigation: {query}",
+                "description": f"Basic investigation of {query}",
+                "source_type": "web_search",
+                "confidence": 0.3,
+                "tier": "POSSIBLE",
+            }})
+        
+        return events
+
+    def get_graph(self) -> Dict[str, int]:
+        """Return knowledge graph statistics."""
+        try:
+            from watson.graph import KnowledgeGraph
+            g = KnowledgeGraph()
+            return g.stats() if hasattr(g, "stats") else {"nodes": 0, "edges": 0}
+        except Exception:
+            return {"nodes": 0, "edges": 0}

src/watson/agents/__init__.py

@@ -0,0 +1,101 @@
+"""Agents registry — list all available OSINT sub-agents.
+
+Provides get_all_agents() for the /api/agent/agents endpoint.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, List
+
+
+class AgentRole(str, Enum):
+    COORDINATOR = "coordinator"
+    DOMAIN_RESEARCHER = "domain_researcher"
+    SOCIAL_MEDIA_ANALYST = "social_media_analyst"
+    CORPORATE_ANALYST = "corporate_analyst"
+    CRYPTO_ANALYST = "crypto_analyst"
+    GEOLOCATION_ANALYST = "geolocation_analyst"
+    IMAGE_ANALYST = "image_analyst"
+    PEOPLE_SEARCHER = "people_searcher"
+    DARKWEB_SCOUT = "darkweb_scout"
+    CROSS_REFERENCER = "cross_referencer"
+
+
+@dataclass
+class AgentInfo:
+    role: AgentRole
+    description: str
+    capabilities: List[str] = field(default_factory=list)
+    tool_count: int = 0
+
+
+# Registry of all specialized agents
+_AGENTS: Dict[str, AgentInfo] = {
+    "coordinator": AgentInfo(
+        role=AgentRole.COORDINATOR,
+        description="Orchestrates investigations, dispatches sub-agents, and synthesizes findings",
+        capabilities=["multi_agent_orchestration", "finding_synthesis", "gap_analysis"],
+        tool_count=5,
+    ),
+    "domain_researcher": AgentInfo(
+        role=AgentRole.DOMAIN_RESEARCHER,
+        description="Domain & DNS investigation — WHOIS, certificate transparency, DNS records",
+        capabilities=["whois_lookup", "dns_lookup", "crt_sh", "wayback_machine"],
+        tool_count=4,
+    ),
+    "social_media_analyst": AgentInfo(
+        role=AgentRole.SOCIAL_MEDIA_ANALYST,
+        description="Social media investigation — profiles, posts, network analysis",
+        capabilities=["social_search", "profile_analysis", "network_mapping"],
+        tool_count=3,
+    ),
+    "corporate_analyst": AgentInfo(
+        role=AgentRole.CORPORATE_ANALYST,
+        description="Corporate investigation — company registries, directors, financial records",
+        capabilities=["opencorporates_search", "sec_filings", "sanctions_check"],
+        tool_count=3,
+    ),
+    "crypto_analyst": AgentInfo(
+        role=AgentRole.CRYPTO_ANALYST,
+        description="Cryptocurrency investigation — blockchain analysis, wallet tracing",
+        capabilities=["etherscan_lookup", "wallet_tracing", "transaction_analysis"],
+        tool_count=3,
+    ),
+    "geolocation_analyst": AgentInfo(
+        role=AgentRole.GEOLOCATION_ANALYST,
+        description="Geolocation investigation — satellite imagery, location verification",
+        capabilities=["satellite_analysis", "geolocation", "reverse_image_search"],
+        tool_count=3,
+    ),
+    "image_analyst": AgentInfo(
+        role=AgentRole.IMAGE_ANALYST,
+        description="Image & video investigation — EXIF analysis, deepfake detection, visual forensics",
+        capabilities=["exif_analysis", "reverse_image_search", "deepfake_detection"],
+        tool_count=3,
+    ),
+    "people_searcher": AgentInfo(
+        role=AgentRole.PEOPLE_SEARCHER,
+        description="People investigation — public records, professional profiles, background checks",
+        capabilities=["people_search", "professional_profiles", "public_records"],
+        tool_count=3,
+    ),
+    "darkweb_scout": AgentInfo(
+        role=AgentRole.DARKWEB_SCOUT,
+        description="Dark web investigation — .onion sites, forums, marketplaces",
+        capabilities=["darkweb_search", "onion_scan", "forum_monitoring"],
+        tool_count=3,
+    ),
+    "cross_referencer": AgentInfo(
+        role=AgentRole.CROSS_REFERENCER,
+        description="Cross-references findings across agents and external sources",
+        capabilities=["cross_reference", "entity_linking", "pattern_detection"],
+        tool_count=2,
+    ),
+}
+
+
+def get_all_agents() -> Dict[str, AgentInfo]:
+    """Return all registered agents."""
+    return _AGENTS

src/watson/agents/protocol.py

@@ -0,0 +1,196 @@
+"""Shared protocol — data models used across Watson's investigation pipeline.
+
+This module is the canonical source of Finding, AgentRole, SourceClass, and
+AgentContext. All orchestration, resolution, and synthesis modules import from
+here to avoid circular dependencies and divergent copies.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any, Optional
+
+
+class AgentRole(str, Enum):
+    """Roles for specialized investigation agents."""
+    ORCHESTRATOR = "orchestrator"
+    RECON = "recon"
+    SOCIAL = "social"
+    CORPORATE = "corporate"
+    CRYPTO = "crypto"
+    GEO = "geo"
+    MEDIA = "media"
+    PEOPLE = "people"
+    DARK = "dark"
+
+    # Aliases for backward compat with agents/__init__.py
+    COORDINATOR = "orchestrator"
+    DOMAIN_RESEARCHER = "recon"
+    SOCIAL_MEDIA_ANALYST = "social"
+    CORPORATE_ANALYST = "corporate"
+    CRYPTO_ANALYST = "crypto"
+    GEOLOCATION_ANALYST = "geo"
+    IMAGE_ANALYST = "media"
+    PEOPLE_SEARCHER = "people"
+    DARKWEB_SCOUT = "dark"
+    CROSS_REFERENCER = "orchestrator"
+
+
+class SourceClass(str, Enum):
+    """Source classification tiers (Bazzell-standard)."""
+    PRIMARY = "primary"        # Court records, gov registries, OFAC sanctions
+    SECONDARY = "secondary"    # News articles, corporate registries, breach data
+    TERTIARY = "tertiary"       # Wikipedia, social media, self-reported
+    UNVERIFIED = "unverified"   # Anonymous sources, forum posts, single mentions
+
+
+@dataclass
+class Finding:
+    """Intelligence finding with source tracking and confidence.
+
+    This is the canonical Finding used across the entire pipeline.
+    The orchestration engine has its own Finding (in engine.py) but
+    both are duck-type compatible — resolution and synthesis use getattr().
+    """
+    id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
+    title: str = ""
+    description: str = ""
+    agent: AgentRole = AgentRole.ORCHESTRATOR
+    source_class: SourceClass = SourceClass.SECONDARY
+    source_type: str = "web_search"
+    source_url: str = ""
+    source_tier: str = "SECONDARY"
+    confidence: float = 0.5
+    entities: list[dict] = field(default_factory=list)
+    evidence: list[str] = field(default_factory=list)
+    phase: str = ""
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+    @property
+    def tier(self) -> str:
+        if self.confidence >= 0.90:
+            return "CONFIRMED"
+        if self.confidence >= 0.70:
+            return "PROBABLE"
+        if self.confidence >= 0.40:
+            return "POSSIBLE"
+        if self.confidence >= 0.10:
+            return "UNLIKELY"
+        return "UNSUBSTANTIATED"
+
+    @property
+    def agent_value(self) -> str:
+        return self.agent.value if isinstance(self.agent, AgentRole) else str(self.agent)
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "title": self.title,
+            "description": self.description,
+            "agent": self.agent_value,
+            "source_class": self.source_class.value if isinstance(self.source_class, SourceClass) else str(self.source_class),
+            "source_type": self.source_type,
+            "source_url": self.source_url,
+            "source_tier": self.source_tier,
+            "confidence": self.confidence,
+            "tier": self.tier,
+            "entities": self.entities,
+            "phase": self.phase,
+            "timestamp": self.timestamp,
+        }
+
+
+@dataclass
+class AgentContext:
+    """Context passed to investigation agents."""
+    query: str = ""
+    target_type: str = "unknown"
+    target_value: str = ""
+    focus: str = ""
+    depth: int = 2
+    findings: list[Finding] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+# ── Target classification ────────────────────────────────────
+
+def classify_target(query: str) -> tuple[str, str]:
+    """Quick deterministic target classification.
+    
+    Returns (target_type, target_value).
+    Used by tests and as a fast-path before the LLM classifier.
+    """
+    import re
+    q = query.strip()
+    
+    # Email
+    if re.match(r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', q):
+        return "email", q.lower()
+    
+    # Ethereum wallet
+    if re.match(r'^0x[a-fA-F0-9]{40}$', q):
+        return "crypto", q
+    
+    # Bitcoin address
+    if re.match(r'^[13][a-km-zA-HJ-NP-Z1-9]{25,34}$', q):
+        return "crypto", q
+    
+    # IPv4
+    if re.match(r'^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$', q):
+        return "ip", q
+    
+    # .onion
+    if '.onion' in q:
+        return "onion", q
+    
+    # GPS coordinates
+    if re.match(r'^-?\d{1,3}\.\d+,\s*-?\d{1,3}\.\d+$', q):
+        return "gps", q
+    
+    # Flight code
+    if re.match(r'^[A-Z]{2}\d{2,4}$', q):
+        return "flight", q
+    
+    # Image file
+    if re.match(r'^.*\.(jpg|jpeg|png|gif|bmp|webp|tiff)$', q, re.I):
+        return "image", q
+    
+    # Domain
+    if re.match(r'^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(?:com|org|net|io|gov|edu|co|me|ai|app|dev|xyz|info|biz)$', q, re.I):
+        return "domain", q.lower()
+    
+    # Breach keywords
+    if any(kw in q.lower() for kw in ['breach', 'leaked', 'passwords', 'data dump']):
+        return "breach", q
+    
+    # Company indicators
+    if any(suffix in q for suffix in ['Inc', 'LLC', 'Ltd', 'Corp', 'GmbH', 'SA', 'AG', 'Group']):
+        return "company", q
+    
+    # Default: person
+    if ' ' in q and len(q.split()) >= 2:
+        return "person", q
+    
+    # Single word — could be username or company
+    return "person", q
+
+
+def select_agents(target_type: str) -> list[AgentRole]:
+    """Select which agents to dispatch for a given target type."""
+    mapping = {
+        "domain": [AgentRole.RECON, AgentRole.CORPORATE, AgentRole.PEOPLE],
+        "email": [AgentRole.SOCIAL, AgentRole.DARK, AgentRole.RECON],
+        "ip": [AgentRole.RECON, AgentRole.GEO],
+        "crypto": [AgentRole.CRYPTO, AgentRole.DARK],
+        "gps": [AgentRole.GEO, AgentRole.MEDIA],
+        "image": [AgentRole.MEDIA, AgentRole.GEO],
+        "onion": [AgentRole.DARK, AgentRole.RECON],
+        "company": [AgentRole.CORPORATE, AgentRole.RECON, AgentRole.SOCIAL],
+        "person": [AgentRole.SOCIAL, AgentRole.PEOPLE, AgentRole.RECON, AgentRole.CORPORATE],
+        "breach": [AgentRole.DARK, AgentRole.SOCIAL],
+        "flight": [AgentRole.GEO, AgentRole.RECON],
+    }
+    return mapping.get(target_type, [AgentRole.RECON, AgentRole.SOCIAL])

src/watson/auth/__init__.py

@@ -0,0 +1,68 @@
+"""Auth middleware — API key validation, workspace auth.
+
+Delegates to watson.web.middleware for actual API key checking.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+
+
+class AuthMiddleware(BaseHTTPMiddleware):
+    """FastAPI middleware: validate API keys from X-API-Key or Authorization header.
+
+    Delegates to watson.web.middleware.validate_api_key() when available,
+    falling back to WATSON_API_KEYS env var check.
+    """
+
+    _PUBLIC_PATHS: set[str] = {"/", "/health", "/docs", "/openapi.json", "/redoc",
+                                "/static", "/api/settings", "/api/agent/detect-intent"}
+
+    async def dispatch(self, request: Request, call_next) -> Response:
+        path = request.url.path
+
+        # Allow public paths
+        if path in self._PUBLIC_PATHS or path.startswith("/static"):
+            return await call_next(request)
+
+        # Extract API key
+        api_key = request.headers.get("X-API-Key", "")
+        if not api_key:
+            auth_header = request.headers.get("Authorization", "")
+            if auth_header.startswith("Bearer "):
+                api_key = auth_header[7:]
+
+        # Validate
+        valid = self._validate(api_key)
+        if not valid:
+            return JSONResponse(
+                {"error": "Unauthorized — provide API key via X-API-Key or Bearer token"},
+                status_code=401,
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+
+        return await call_next(request)
+
+    @staticmethod
+    def _validate(api_key: str) -> bool:
+        # Try delegating to existing watson.web.middleware
+        try:
+            from watson.web.middleware import validate_api_key
+            return validate_api_key(api_key)
+        except ImportError:
+            pass
+
+        # Fallback: check env
+        valid_keys_raw = os.environ.get("WATSON_API_KEYS", "")
+        if not valid_keys_raw:
+            return True  # Dev mode — no keys configured
+        valid_keys = {k.strip() for k in valid_keys_raw.split(",") if k.strip()}
+        hermes_key = os.environ.get("HERMES_API_KEY", "")
+        if hermes_key:
+            valid_keys.add(hermes_key)
+        return api_key in valid_keys

src/watson/auth/store.py

@@ -0,0 +1,145 @@
+"""Auth store — workspace/user management backed by a JSON file.
+
+Minimal implementation for Phase B auth endpoints.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+
+class UserRole(str, Enum):
+    ADMIN = "admin"
+    ANALYST = "analyst"
+    VIEWER = "viewer"
+
+
+@dataclass
+class User:
+    user_id: str
+    workspace_id: str
+    email: str
+    role: UserRole = UserRole.ANALYST
+    created_at: str = ""
+
+
+@dataclass
+class Workspace:
+    workspace_id: str
+    name: str
+    created_at: str = ""
+
+
+class AuthStore:
+    """In-memory JSON-backed auth store."""
+
+    def __init__(self, path: str | Path | None = None):
+        if path is None:
+            path = Path.home() / ".watson" / "auth_store.json"
+        self._path = Path(path)
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._data: dict = self._load()
+
+    def _load(self) -> dict:
+        if self._path.exists():
+            try:
+                return json.loads(self._path.read_text())
+            except (json.JSONDecodeError, OSError):
+                pass
+        return {"workspaces": {}, "users": {}, "api_keys": {}, "tokens": {}}
+
+    def _save(self):
+        self._path.write_text(json.dumps(self._data, indent=2, default=str))
+
+    def create_workspace(self, name: str, email: str) -> tuple[Workspace, User]:
+        ws_id = f"ws-{uuid.uuid4().hex[:8]}"
+        user_id = f"usr-{uuid.uuid4().hex[:8]}"
+        now = datetime.now(timezone.utc).isoformat()
+
+        ws = Workspace(workspace_id=ws_id, name=name, created_at=now)
+        user = User(user_id=user_id, workspace_id=ws_id, email=email,
+                     role=UserRole.ADMIN, created_at=now)
+
+        self._data["workspaces"][ws_id] = {"workspace_id": ws_id, "name": name, "created_at": now}
+        self._data["users"][user_id] = {"user_id": user_id, "workspace_id": ws_id,
+                                         "email": email, "role": "admin", "created_at": now}
+        self._save()
+        return ws, user
+
+    def generate_api_key(self, user_id: str) -> str:
+        key = f"watson-{uuid.uuid4().hex}"
+        self._data["api_keys"][key] = user_id
+        self._save()
+        return key
+
+    def validate_api_key(self, api_key: str) -> Optional[User]:
+        user_id = self._data["api_keys"].get(api_key)
+        if user_id:
+            return self.get_user(user_id)
+        # Fallback: check env keys
+        valid = os.environ.get("WATSON_API_KEYS", "").split(",")
+        if api_key in [k.strip() for k in valid if k.strip()]:
+            # Return a synthetic user for env-based keys
+            return User(user_id="env-user", workspace_id="default",
+                        email="admin@watson.local", role=UserRole.ADMIN)
+        return None
+
+    def create_token(self, user: User) -> "Token":
+        token_str = f"tok-{uuid.uuid4().hex}"
+        expires = (datetime.now(timezone.utc) + timedelta(hours=24)).isoformat()
+        self._data["tokens"][token_str] = {"user_id": user.user_id, "expires_at": expires}
+        self._save()
+        return Token(token=token_str, expires_at=expires, user_id=user.user_id)
+
+    def get_user(self, user_id: str) -> Optional[User]:
+        u = self._data["users"].get(user_id)
+        if u:
+            return User(user_id=u["user_id"], workspace_id=u["workspace_id"],
+                        email=u.get("email", ""), role=UserRole(u.get("role", "analyst")),
+                        created_at=u.get("created_at", ""))
+        return None
+
+    def get_user_by_email(self, email: str, workspace_id: str) -> Optional[User]:
+        for u in self._data["users"].values():
+            if u.get("email") == email and u.get("workspace_id") == workspace_id:
+                return User(**{k: v for k, v in u.items() if k in
+                               ("user_id", "workspace_id", "email", "role", "created_at")})
+        return None
+
+    def get_workspace(self, workspace_id: str) -> Optional[Workspace]:
+        w = self._data["workspaces"].get(workspace_id)
+        if w:
+            return Workspace(workspace_id=w["workspace_id"],
+                             name=w.get("name", ""), created_at=w.get("created_at", ""))
+        return None
+
+    def list_users(self, workspace_id: str) -> list[User]:
+        return [User(**{k: v for k, v in u.items() if k in
+                        ("user_id", "workspace_id", "email", "role", "created_at")})
+                for u in self._data["users"].values()
+                if u.get("workspace_id") == workspace_id]
+
+
+@dataclass
+class Token:
+    token: str
+    expires_at: str
+    user_id: str = ""
+
+
+# Singleton
+_auth_store: Optional[AuthStore] = None
+
+
+def get_auth_store() -> AuthStore:
+    global _auth_store
+    if _auth_store is None:
+        _auth_store = AuthStore()
+    return _auth_store

src/watson/conversation.py

@@ -0,0 +1,68 @@
+"""Conversation Agent — chat interface with investigation capabilities."""
+
+from __future__ import annotations
+import logging
+from typing import Any, Callable, Optional
+
+logger = logging.getLogger("watson.conversation")
+
+
+class ConversationAgent:
+    """Chat agent that can investigate and converse."""
+    
+    def __init__(self, api_key: str = ""):
+        self.api_key = api_key
+        self._history: list[dict] = []
+    
+    async def chat(
+        self,
+        message: str,
+        findings: list | None = None,
+        history: list | None = None,
+        on_event: Optional[Callable[[str, Any], None]] = None,
+    ) -> dict:
+        """Process a chat message. Returns response dict with events."""
+        
+        events = []
+        
+        # Determine if this is an investigation request
+        investigation_keywords = [
+            "investigate", "look up", "search", "find", "who is",
+            "what is", "research", "dig into", "analyze", "check",
+        ]
+        
+        msg_lower = message.lower().strip()
+        is_investigation = any(kw in msg_lower for kw in investigation_keywords)
+        
+        if is_investigation:
+            # Extract the target from the message
+            target = message
+            for kw in investigation_keywords:
+                idx = msg_lower.find(kw)
+                if idx >= 0:
+                    target = message[idx + len(kw):].strip().rstrip(".!?")
+                    break
+            
+            if target:
+                try:
+                    from .agent import OSINTAgent
+                    agent = OSINTAgent(depth=2)
+                    inv_events = await agent.investigate(
+                        query=target,
+                        on_event=on_event,
+                    )
+                    events.extend(inv_events)
+                except Exception as e:
+                    logger.warning("conversation_investigation_failed: %s", e)
+                    events.append({
+                        "event": "message",
+                        "data": {"content": f"I tried to investigate '{target}' but encountered an error: {e}"},
+                    })
+        
+        if not events:
+            events.append({
+                "event": "message",
+                "data": {"content": f"I can help you investigate. Try asking me to investigate a person, company, or domain."},
+            })
+        
+        return {"events": events, "investigation_triggered": is_investigation}