osintengine 1.0.2__py3-none-any.whl

src/watson/infra/retry.py

@@ -0,0 +1,186 @@
+"""Retry & circuit breaker infrastructure."""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import random
+import threading
+import time
+from typing import Any, Callable, Dict, Tuple, Type
+
+
+# ─────────────────────────────────────────────────────────────────
+# Retryable exception classification
+# ─────────────────────────────────────────────────────────────────
+
+# TimeoutError, ConnectionError, ConnectionRefusedError, ConnectionResetError
+# are all subclasses of OSError, so (OSError,) covers them all.
+_RETRYABLE_EXCS: Tuple[Type[BaseException], ...] = (
+    TimeoutError,
+    ConnectionError,
+    ConnectionRefusedError,
+    ConnectionResetError,
+    OSError,
+)
+
+
+def _is_retryable(exc: BaseException) -> bool:
+    """Return True if *exc* is a transient/network failure worth retrying."""
+    return isinstance(exc, _RETRYABLE_EXCS)
+
+
+# ─────────────────────────────────────────────────────────────────
+# Circuit breaker
+# ─────────────────────────────────────────────────────────────────
+
+
+class CircuitOpenError(Exception):
+    """Raised when an operation is attempted on an open circuit."""
+
+    def __init__(self, name: str = "", message: str = ""):
+        self.name = name
+        super().__init__(message or f"circuit '{name}' is open")
+
+
+class CircuitBreaker:
+    """Simple circuit breaker — opens after *failure_threshold* failures.
+
+    States:
+      * CLOSED  — normal operation, failures counted
+      * OPEN    — requests short-circuit and raise :class:`CircuitOpenError`
+      * HALF-OPEN — after *reset_timeout* elapses, one probe is allowed
+    """
+
+    def __init__(
+        self,
+        name: str = "",
+        failure_threshold: int = 5,
+        reset_timeout: float = 30.0,
+    ):
+        self.name = name
+        self.failure_threshold = failure_threshold
+        self.reset_timeout = reset_timeout
+        self._failures = 0
+        self._last_failure: float = 0.0
+        self._open_since: float = 0.0
+        self._lock = threading.Lock()
+
+    # -- state -----------------------------------------------------
+    @property
+    def is_open(self) -> bool:
+        with self._lock:
+            if self._failures < self.failure_threshold:
+                return False
+            # Half-open: allow a probe after reset_timeout
+            if time.monotonic() - self._open_since > self.reset_timeout:
+                return False
+            return True
+
+    # -- transitions -----------------------------------------------
+    def success(self) -> None:
+        with self._lock:
+            self._failures = 0
+            self._open_since = 0.0
+
+    def failure(self) -> None:
+        with self._lock:
+            self._failures += 1
+            self._last_failure = time.monotonic()
+            if self._failures >= self.failure_threshold:
+                self._open_since = time.monotonic()
+
+    # -- context manager ------------------------------------------
+    def __enter__(self) -> "CircuitBreaker":
+        if self.is_open:
+            raise CircuitOpenError(self.name)
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        if exc is None:
+            self.success()
+        else:
+            self.failure()
+        return False  # never suppress
+
+
+# Registry of named circuit breakers (singletons)
+_circuits: Dict[str, CircuitBreaker] = {}
+_circuits_lock = threading.Lock()
+
+
+def get_circuit(name: str, **kwargs: Any) -> CircuitBreaker:
+    """Get-or-create a singleton :class:`CircuitBreaker` by name."""
+    with _circuits_lock:
+        cb = _circuits.get(name)
+        if cb is None:
+            cb = CircuitBreaker(name=name, **kwargs)
+            _circuits[name] = cb
+        return cb
+
+
+# ─────────────────────────────────────────────────────────────────
+# Retry decorator
+# ─────────────────────────────────────────────────────────────────
+
+
+def retry(
+    max_attempts: int = 3,
+    base_delay: float = 1.0,
+    jitter: bool = True,
+    exceptions: Tuple[Type[BaseException], ...] = (Exception,),
+) -> Callable[[Callable], Callable]:
+    """Retry a callable with exponential backoff.
+
+    Only retryable exceptions (per :func:`_is_retryable`) are retried; any
+    other exception is re-raised immediately without consuming an attempt.
+
+    Works for both synchronous and ``async`` functions.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        if asyncio.iscoroutinefunction(func):
+
+            @functools.wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                last_exc: BaseException | None = None
+                for attempt in range(max_attempts):
+                    try:
+                        return await func(*args, **kwargs)
+                    except BaseException as e:  # noqa: BLE001
+                        last_exc = e
+                        if not _is_retryable(e):
+                            raise
+                        if attempt >= max_attempts - 1:
+                            raise
+                        delay = base_delay * (2 ** attempt)
+                        if jitter:
+                            delay = delay * random.uniform(0.5, 1.5)
+                        await asyncio.sleep(delay)
+                if last_exc is not None:  # pragma: no cover
+                    raise last_exc
+
+            return async_wrapper
+
+        @functools.wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            last_exc: BaseException | None = None
+            for attempt in range(max_attempts):
+                try:
+                    return func(*args, **kwargs)
+                except BaseException as e:  # noqa: BLE001
+                    last_exc = e
+                    if not _is_retryable(e):
+                        raise
+                    if attempt >= max_attempts - 1:
+                        raise
+                    delay = base_delay * (2 ** attempt)
+                    if jitter:
+                        delay = delay * random.uniform(0.5, 1.5)
+                    time.sleep(delay)
+            if last_exc is not None:  # pragma: no cover
+                raise last_exc
+
+        return sync_wrapper
+
+    return decorator

src/watson/metrics.py

@@ -0,0 +1,128 @@
+"""Metrics — Prometheus-compatible metrics endpoint and decorators."""
+
+from __future__ import annotations
+
+import functools
+import threading
+import time
+from typing import Any, Callable, Dict
+
+
+# ── Simple counter/gauge for Prometheus text format ─────────────────
+
+class _Counter:
+    """Thread-safe counter."""
+    def __init__(self, name: str, help_: str = ""):
+        self.name = name
+        self.help = help_
+        self._value = 0
+        self._lock = threading.Lock()
+
+    def inc(self, n: int = 1):
+        with self._lock:
+            self._value += n
+
+    def value(self) -> int:
+        with self._lock:
+            return self._value
+
+
+class _Gauge:
+    """Thread-safe gauge."""
+    def __init__(self, name: str, help_: str = ""):
+        self.name = name
+        self.help = help_
+        self._value = 0.0
+        self._lock = threading.Lock()
+
+    def set(self, v: float):
+        with self._lock:
+            self._value = v
+
+    def value(self) -> float:
+        with self._lock:
+            return self._value
+
+
+class _Histogram:
+    """Simple histogram."""
+    def __init__(self, name: str, help_: str = "", buckets=None):
+        self.name = name
+        self.help = help_
+        self.buckets = buckets or [0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0]
+        self._values: list[float] = []
+        self._lock = threading.Lock()
+
+    def observe(self, v: float):
+        with self._lock:
+            self._values.append(v)
+
+    def values(self) -> list[float]:
+        with self._lock:
+            return list(self._values)
+
+
+# ── Core metrics ────────────────────────────────────────────────────
+
+investigations_total = _Counter("watson_investigations_total", "Total investigations started")
+findings_total = _Counter("watson_findings_total", "Total findings collected")
+findings_confirmed = _Counter("watson_findings_confirmed", "Total confirmed findings")
+investigation_duration = _Histogram("watson_investigation_duration_seconds",
+                                     "Investigation duration in seconds")
+api_requests_total = _Counter("watson_api_requests_total", "Total API requests")
+api_errors_total = _Counter("watson_api_errors_total", "Total API errors")
+
+_registered_metrics = [investigations_total, findings_total, findings_confirmed,
+                       investigation_duration, api_requests_total, api_errors_total]
+
+
+# ── Prometheus text output ──────────────────────────────────────────
+
+def prometheus_endpoint() -> str:
+    """Generate Prometheus-compatible metrics text."""
+    lines: list[str] = []
+
+    for metric in _registered_metrics:
+        if isinstance(metric, (_Counter, _Gauge)):
+            lines.append(f"# HELP {metric.name} {metric.help}")
+            lines.append(f"# TYPE {metric.name} {'counter' if isinstance(metric, _Counter) else 'gauge'}")
+            lines.append(f"{metric.name} {metric.value()}")
+
+    # Add some system-level info
+    import os
+    import sys
+    lines.append(f"# HELP watson_info Watson version info")
+    lines.append(f"# TYPE watson_info gauge")
+    lines.append(f'watson_info{{version="0.3.0",python="{sys.version.split()[0]}"}} 1')
+
+    return "\n".join(lines) + "\n"
+
+
+# ── Track investigation decorator ───────────────────────────────────
+
+def track_investigation(func: Callable) -> Callable:
+    """Decorator to count investigations and track duration."""
+    @functools.wraps(func)
+    async def async_wrapper(*args, **kwargs):
+        investigations_total.inc()
+        start = time.monotonic()
+        try:
+            result = await func(*args, **kwargs)
+            return result
+        finally:
+            investigation_duration.observe(time.monotonic() - start)
+
+    @functools.wraps(func)
+    def sync_wrapper(*args, **kwargs):
+        investigations_total.inc()
+        start = time.monotonic()
+        try:
+            result = func(*args, **kwargs)
+            return result
+        finally:
+            investigation_duration.observe(time.monotonic() - start)
+
+    import asyncio
+    if asyncio.iscoroutinefunction(func):
+        return async_wrapper
+    return sync_wrapper

src/watson/opsec/__init__.py

@@ -0,0 +1,290 @@
+"""OpSec Proxy Firewall — enterprise HTTP layer with proxy chaining, domain-
+specific rate limiting, and connection pooling.
+
+Extends BaseHTTPClient from watson.utils.http with:
+  - SOCKS5 / HTTP / HTTPS proxy support
+  - Per-domain rate limiting (token bucket per host)
+  - Configurable proxy chain (multiple hops)
+  - TLS fingerprint rotation
+  - Real browser header profiles per domain category
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import random
+import time
+from typing import Optional
+from urllib.parse import urlparse
+
+import httpx
+
+from watson.utils.http import BaseHTTPClient, USER_AGENTS
+
+logger = logging.getLogger("watson.opsec")
+
+# ── Proxy configuration ──────────────────────────────────────
+
+_DEFAULT_PROXY = os.environ.get("WATSON_PROXY", "")  # socks5://user:pass@host:port
+_PROXY_CHAIN = os.environ.get("WATSON_PROXY_CHAIN", "")  # comma-separated SOCKS5 URLs
+
+
+def _parse_proxy_url(url: str) -> str | None:
+    """Parse a proxy URL into httpx format. Returns None if empty/invalid."""
+    if not url or not url.strip():
+        return None
+    parsed = urlparse(url.strip())
+    if parsed.scheme in ("socks5", "socks5h"):
+        # httpx: socks5://user:pass@host:port
+        netloc = parsed.hostname or ""
+        if parsed.port:
+            netloc += f":{parsed.port}"
+        auth = f"{parsed.username}:{parsed.password}@" if parsed.username else ""
+        return f"socks5://{auth}{netloc}"
+    if parsed.scheme in ("http", "https"):
+        return url.strip()
+    logger.warning("opsec: unknown proxy scheme %s", parsed.scheme)
+    return None
+
+
+# ── Per-domain rate limiter ───────────────────────────────────
+
+class DomainRateLimiter:
+    """Token bucket rate limiter per domain."""
+
+    def __init__(self, default_rps: float = 1.0):
+        self.default_rps = default_rps
+        self._buckets: dict[str, tuple[float, float]] = {}  # domain → (tokens, last_refill)
+        # Harsher limits for known rate-limit-happy domains
+        self._overrides: dict[str, float] = {
+            "crt.sh": 0.5,
+            "opencorporates.com": 0.5,
+            "google.com": 0.3,
+            "linkedin.com": 0.2,
+            "api.github.com": 2.0,
+        }
+
+    def _rps_for(self, domain: str) -> float:
+        for pattern, rps in self._overrides.items():
+            if pattern in domain:
+                return rps
+        return self.default_rps
+
+    async def acquire(self, domain: str) -> None:
+        rate = self._rps_for(domain)
+        while True:
+            now = time.monotonic()
+            tokens, last = self._buckets.get(domain, (rate, now))
+            elapsed = now - last
+            tokens = min(rate, tokens + elapsed * rate)
+            self._buckets[domain] = (tokens, now)
+
+            if tokens >= 1.0:
+                self._buckets[domain] = (tokens - 1.0, now)
+                return
+            await asyncio.sleep(0.1)
+
+
+# ── Domain-specific header profiles ───────────────────────────
+
+_DOMAIN_HEADERS: dict[str, dict[str, str]] = {
+    "google.com": {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    },
+    "opencorporates.com": {
+        "Accept": "application/json, text/html, */*",
+        "Accept-Language": "en-US,en;q=0.9",
+    },
+    "wikipedia.org": {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    },
+    "linkedin.com": {
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+    },
+}
+
+
+# ── OpSec HTTP Client ─────────────────────────────────────────
+
+class OpSecClient(BaseHTTPClient):
+    """Enterprise HTTP client: proxy chaining, per-domain rate limiting,
+    real browser header profiles, and rotating TLS fingerprints.
+
+    Usage:
+        client = OpSecClient()
+        resp = await client.get("https://crt.sh/?q=%.example.com")
+        data = resp.json()
+    """
+
+    def __init__(
+        self,
+        proxy: str | None = None,
+        proxy_chain: list[str] | None = None,
+        rate_limit: float = 1.0,
+        max_retries: int = 3,
+        timeout: float = 20.0,
+        rotate_fingerprints: bool = True,
+    ):
+        super().__init__(rate_limit=rate_limit, max_retries=max_retries, timeout=timeout)
+        self._proxy = proxy or _parse_proxy_url(_DEFAULT_PROXY)
+        self._proxy_chain = proxy_chain or [
+            p for u in _PROXY_CHAIN.split(",") if (p := _parse_proxy_url(u))
+        ] if _PROXY_CHAIN else []
+        self._domain_limiter = DomainRateLimiter()
+        self._rotate_fingerprints = rotate_fingerprints
+        self._session_counter = 0
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        """Create a new client with proxy and domain-appropriate headers."""
+        # Build proxy mount
+        proxy_mounts = {}
+        if self._proxy:
+            proxy_mounts["http://"] = self._proxy
+            proxy_mounts["https://"] = self._proxy
+            logger.debug("opsec: using proxy %s", self._proxy)
+
+        # Build headers with domain-appropriate profile
+        headers = {
+            "User-Agent": random.choice(USER_AGENTS),
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Language": "en-US,en;q=0.9",
+        }
+
+        client = httpx.AsyncClient(
+            timeout=httpx.Timeout(self.timeout, connect=10.0),
+            follow_redirects=True,
+            headers=headers,
+            mounts=proxy_mounts if proxy_mounts else None,
+        )
+        return client
+
+    def _domain_for(self, url: str) -> str:
+        parsed = urlparse(url)
+        return parsed.hostname or ""
+
+    def _per_domain_headers(self, domain: str) -> dict[str, str]:
+        """Get domain-specific header overrides."""
+        for pattern, hdrs in _DOMAIN_HEADERS.items():
+            if pattern in domain:
+                return hdrs
+        return {}
+
+    async def get(self, url: str, **kwargs) -> httpx.Response:
+        """GET with per-domain rate limiting, proxy, and retry."""
+        domain = self._domain_for(url)
+        await self._domain_limiter.acquire(domain)
+
+        # Inject domain-specific headers
+        domain_hdrs = self._per_domain_headers(domain)
+        if domain_hdrs:
+            existing = kwargs.get("headers", {})
+            merged = {**domain_hdrs, **existing}
+            kwargs["headers"] = merged
+
+        # Rotate UA per request for fingerprint diversity
+        client = await self._get_client()
+        if self._rotate_fingerprints:
+            client.headers["User-Agent"] = random.choice(USER_AGENTS)
+
+        last_err = ""
+        for attempt in range(self.max_retries + 1):
+            try:
+                response = await client.get(url, **kwargs)
+                response.raise_for_status()
+                return response
+            except httpx.HTTPStatusError as e:
+                last_err = f"HTTP {e.response.status_code} from {url}"
+                if e.response.status_code == 429:
+                    retry_after = float(e.response.headers.get("Retry-After", 10))
+                    logger.debug("opsec: 429 from %s, waiting %ss", domain, retry_after)
+                    await asyncio.sleep(retry_after)
+                    continue
+                if e.response.status_code in (403, 451):
+                    # Legal/censorship block — rotate proxy if available
+                    if self._proxy_chain:
+                        self._proxy = random.choice(self._proxy_chain)
+                        client = await self._get_client()
+                        logger.info("opsec: rotated proxy for %s after %d", domain, e.response.status_code)
+                        continue
+                if attempt == self.max_retries:
+                    self._last_error = last_err
+                    raise
+                await asyncio.sleep(2 ** attempt)
+            except (httpx.RequestError, httpx.TimeoutException) as e:
+                last_err = f"Connection error for {url}: {e}"
+                if attempt == self.max_retries:
+                    self._last_error = last_err
+                    raise
+                # Rotate proxy on connection failure
+                if self._proxy_chain:
+                    self._proxy = random.choice(self._proxy_chain)
+                    client = await self._get_client()
+                    logger.info("opsec: rotated proxy after connection failure to %s", domain)
+                await asyncio.sleep(2 ** attempt)
+
+        self._last_error = last_err
+        raise RuntimeError(f"Failed to fetch {url} after {self.max_retries + 1} attempts")
+
+    async def post(self, url: str, **kwargs) -> httpx.Response:
+        """POST with per-domain rate limiting and proxy."""
+        domain = self._domain_for(url)
+        await self._domain_limiter.acquire(domain)
+        client = await self._get_client()
+
+        last_err = ""
+        for attempt in range(self.max_retries + 1):
+            try:
+                response = await client.post(url, **kwargs)
+                response.raise_for_status()
+                return response
+            except httpx.HTTPStatusError as e:
+                last_err = f"HTTP {e.response.status_code} from {url}"
+                if e.response.status_code == 429:
+                    await asyncio.sleep(10)
+                    continue
+                if attempt == self.max_retries:
+                    self._last_error = last_err
+                    raise
+                await asyncio.sleep(2 ** attempt)
+            except (httpx.RequestError, httpx.TimeoutException) as e:
+                last_err = f"Connection error for {url}: {e}"
+                if self._proxy_chain:
+                    self._proxy = random.choice(self._proxy_chain)
+                    client = await self._get_client()
+                if attempt == self.max_retries:
+                    self._last_error = last_err
+                    raise
+                await asyncio.sleep(2 ** attempt)
+
+        self._last_error = last_err
+        raise RuntimeError(f"Failed to POST {url} after {self.max_retries + 1} attempts")
+
+    def stats(self) -> dict:
+        """Return OpSec statistics for monitoring."""
+        return {
+            "proxy": self._proxy or "direct",
+            "proxy_chain_size": len(self._proxy_chain),
+            "rate_limiter": {
+                "default_rps": self.rate_limiter.rate,
+                "active_domains": len(self._domain_limiter._buckets),
+            },
+            "errors": self._last_error,
+        }
+
+
+# ── Global instance (lazy) ────────────────────────────────────
+
+_opsec_client: Optional[OpSecClient] = None
+
+
+def get_opsec_client() -> OpSecClient:
+    """Get or create the global OpSec client instance."""
+    global _opsec_client
+    if _opsec_client is None:
+        _opsec_client = OpSecClient()
+    return _opsec_client

src/watson/orchestration/__init__.py

@@ -0,0 +1,23 @@
+"""Watson Orchestration — the intelligence pipeline that makes Watson more than a search engine.
+
+Layers on top of the 7-phase sequential engine with:
+  classify → surface → pivot → deep → dark → analyze → report
+"""
+
+from __future__ import annotations
+from .engine import OrchestrationEngine, get_engine
+from .resolution import (
+    build_intelligence_picture,
+    resolve_entities,
+    propagate_confidence,
+    cross_reference_advanced,
+)
+
+__all__ = [
+    "OrchestrationEngine",
+    "get_engine",
+    "build_intelligence_picture",
+    "resolve_entities",
+    "propagate_confidence",
+    "cross_reference_advanced",
+]