osintkit 0.1.1__py3-none-any.whl

osintkit/config.py

@@ -0,0 +1,151 @@
+"""Runtime settings, on-disk paths, and the response cache."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+APP_HOME_ENV = "OSINTKIT_HOME"
+DEFAULT_BASE_URL = "https://api.venice.ai/api/v1"
+DEFAULT_MODEL = "deepseek-v4-1-flash"
+DEFAULT_TEMPERATURE = 0.3
+DEFAULT_MAX_ATTEMPTS = 3
+DEFAULT_TIMEOUT = 30.0
+DEFAULT_CACHE_TTL = 24 * 60 * 60
+DEFAULT_RESOLVERS = ("1.1.1.1", "8.8.8.8", "9.9.9.9")
+
+PROVIDERS: dict[str, tuple[str, str]] = {
+    "venice": (DEFAULT_BASE_URL, DEFAULT_MODEL),
+    "openrouter": ("https://openrouter.ai/api/v1", "deepseek/deepseek-v4.1-flash-1017"),
+    "openai": ("https://api.openai.com/v1", "gpt-4.1-mini"),
+}
+KEY_PREFIXES: dict[str, str] = {"sk-or-": "openrouter", "sk-proj-": "openai"}
+
+
+def provider_from_key(api_key: str | None) -> str | None:
+    """Infer a provider name from a recognisable key prefix."""
+    if not api_key:
+        return None
+    for prefix, provider in KEY_PREFIXES.items():
+        if api_key.startswith(prefix):
+            return provider
+    return None
+
+
+def app_home() -> Path:
+    override = os.environ.get(APP_HOME_ENV)
+    if override:
+        return Path(override).expanduser()
+    return Path.home() / ".osintkit"
+
+
+def cache_dir() -> Path:
+    return app_home() / "cache"
+
+
+def log_dir() -> Path:
+    return app_home() / "logs"
+
+
+def reports_dir() -> Path:
+    return app_home() / "reports"
+
+
+def ensure_dirs() -> None:
+    for path in (cache_dir(), log_dir()):
+        path.mkdir(parents=True, exist_ok=True)
+
+
+@dataclass(slots=True)
+class Settings:
+    """Everything the run needs, resolved once from flags and the environment."""
+
+    target: str = ""
+    target_type: str = ""
+    base_url: str = DEFAULT_BASE_URL
+    model: str = DEFAULT_MODEL
+    api_key: str | None = None
+    temperature: float = DEFAULT_TEMPERATURE
+    max_tokens: int = 2400
+    timeout: float = DEFAULT_TIMEOUT
+    max_connections: int = 16
+    max_attempts: int = DEFAULT_MAX_ATTEMPTS
+    use_cache: bool = True
+    cache_ttl: int = DEFAULT_CACHE_TTL
+    resolvers: list[str] = field(default_factory=lambda: list(DEFAULT_RESOLVERS))
+    dry_run: bool = False
+    raw: bool = False
+    output_json: bool = False
+    output_markdown: bool = False
+    verbose: bool = False
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def hibp_api_key(self) -> str | None:
+        return os.environ.get("HIBP_API_KEY") or None
+
+    @property
+    def github_token(self) -> str | None:
+        return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") or None
+
+    def resolved_api_key(self) -> str | None:
+        return self.api_key or os.environ.get("API_KEY") or os.environ.get("OPENAI_API_KEY")
+
+
+def _cache_key(namespace: str, key: str) -> Path:
+    digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:32]
+    safe_ns = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in namespace)
+    return cache_dir() / safe_ns / f"{digest}.json"
+
+
+def cache_read(namespace: str, key: str, ttl: int = DEFAULT_CACHE_TTL) -> Any | None:
+    path = _cache_key(namespace, key)
+    try:
+        stat = path.stat()
+    except OSError:
+        return None
+    if ttl > 0 and time.time() - stat.st_mtime > ttl:
+        return None
+    try:
+        return json.loads(path.read_text("utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+def cache_write(namespace: str, key: str, value: Any) -> None:
+    path = _cache_key(namespace, key)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(value, ensure_ascii=False, default=str), "utf-8")
+        tmp.replace(path)
+    except OSError:
+        return
+
+
+def cache_clear() -> int:
+    root = cache_dir()
+    if not root.exists():
+        return 0
+    removed = 0
+    for path in root.rglob("*.json"):
+        try:
+            path.unlink()
+            removed += 1
+        except OSError:
+            continue
+    return removed
+
+
+def cache_stats() -> tuple[int, int]:
+    root = cache_dir()
+    if not root.exists():
+        return (0, 0)
+    files = list(root.rglob("*.json"))
+    size = sum(p.stat().st_size for p in files if p.exists())
+    return (len(files), size)

osintkit/diff.py

@@ -0,0 +1,96 @@
+"""Compare two archived reports."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .models import Report
+
+SEVERITY = ("low", "medium", "high")
+
+
+def _names(rows: list[dict[str, Any]], field: str = "name") -> dict[str, dict[str, Any]]:
+    out: dict[str, dict[str, Any]] = {}
+    for row in rows:
+        key = str(row.get(field) or "").strip().lower()
+        if key:
+            out[key] = row
+    return out
+
+
+def _set_diff(old: list[str], new: list[str]) -> dict[str, list[str]]:
+    before = {item.strip().lower(): item.strip() for item in old if item.strip()}
+    after = {item.strip().lower(): item.strip() for item in new if item.strip()}
+    return {
+        "added": sorted(after[key] for key in after.keys() - before.keys()),
+        "removed": sorted(before[key] for key in before.keys() - after.keys()),
+    }
+
+
+def compare(old: Report, new: Report) -> dict[str, Any]:
+    old_analysis = old.analysis
+    new_analysis = new.analysis
+    old_breaches = _names([b.model_dump() for b in old_analysis.breaches]) if old_analysis else {}
+    new_breaches = _names([b.model_dump() for b in new_analysis.breaches]) if new_analysis else {}
+
+    delta = _risk_delta(old, new)
+    old_status = {c.collector: (c.skipped, c.ok) for c in old.collectors}
+    new_status = {c.collector: (c.skipped, c.ok) for c in new.collectors}
+
+    return {
+        "old_target": old.target,
+        "new_target": new.target,
+        "target_changed": old.target != new.target,
+        "old_generated": old.generated_at,
+        "new_generated": new.generated_at,
+        "risk": {
+            "old": old_analysis.risk_level if old_analysis else None,
+            "new": new_analysis.risk_level if new_analysis else None,
+        },
+        "breaches": {
+            "added": sorted(new_breaches[k]["name"] for k in new_breaches.keys() - old_breaches.keys()),
+            "removed": sorted(old_breaches[k]["name"] for k in old_breaches.keys() - new_breaches.keys()),
+            "unchanged": len(old_breaches.keys() & new_breaches.keys()),
+        },
+        "subdomains": _set_diff(
+            old_analysis.subdomains if old_analysis else [],
+            new_analysis.subdomains if new_analysis else [],
+        ),
+        "collectors": {
+            "started_working": sorted(
+                name
+                for name, state in new_status.items()
+                if state[1] and not state[0] and old_status.get(name, (False, False))[1] is False
+            ),
+            "started_failing": sorted(
+                name
+                for name, state in new_status.items()
+                if not state[1] and not state[0] and old_status.get(name, (False, True))[1] is True
+            ),
+        },
+        "risk_rose": delta > 0,
+        "risk_fell": delta < 0,
+    }
+
+
+def _risk_delta(old: Report, new: Report) -> int:
+    if not old.analysis or not new.analysis:
+        return 0
+    before = SEVERITY.index(old.analysis.risk_level) if old.analysis.risk_level in SEVERITY else 0
+    after = SEVERITY.index(new.analysis.risk_level) if new.analysis.risk_level in SEVERITY else 0
+    return after - before
+
+
+def is_empty(diff: dict[str, Any]) -> bool:
+    return not any(
+        (
+            diff["breaches"]["added"],
+            diff["breaches"]["removed"],
+            diff["subdomains"]["added"],
+            diff["subdomains"]["removed"],
+            diff["collectors"]["started_working"],
+            diff["collectors"]["started_failing"],
+            diff["risk_rose"],
+            diff["risk_fell"],
+        )
+    )

osintkit/doctor.py

@@ -0,0 +1,119 @@
+"""Environment checks behind `osintkit doctor`."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from .config import Settings, cache_dir, ensure_dirs, log_dir
+from .http import HttpClient
+
+
+@dataclass(slots=True)
+class Check:
+    name: str
+    status: str
+    detail: str
+
+
+def _writable(path: Path) -> bool:
+    try:
+        ensure_dirs()
+        path.mkdir(parents=True, exist_ok=True)
+        with tempfile.NamedTemporaryFile(dir=path, delete=True):
+            return True
+    except OSError:
+        return False
+
+
+async def _reach_endpoint(settings: Settings, deep: bool = False) -> Check:
+    key = settings.resolved_api_key()
+    if not key:
+        return Check("model endpoint", "warn", "no API key; synthesis will fail, dry runs still work")
+    base = settings.base_url.rstrip("/")
+    headers = {"Authorization": f"Bearer {key}"}
+    fresh = replace(settings, use_cache=False)
+
+    try:
+        async with HttpClient(fresh) as http:
+            response = await http.get(
+                f"{base}/models",
+                namespace="doctor",
+                headers=headers,
+                empty_on=tuple(range(400, 500)),
+                timeout=10.0,
+            )
+            if response.status_code >= 400 and response.status_code != 404:
+                return Check("model endpoint", "fail", f"HTTP {response.status_code} from {base}/models")
+
+            if not deep:
+                detail = (
+                    f"{base} answered; key not exercised, use --deep"
+                    if response.status_code < 400
+                    else f"HTTP {response.status_code} from {base}/models; use --deep"
+                )
+                return Check("model endpoint", "ok" if response.status_code < 400 else "warn", detail)
+
+            probe = await http.post_json(
+                f"{base}/chat/completions",
+                namespace="doctor",
+                headers=headers,
+                payload={
+                    "model": settings.model,
+                    "messages": [{"role": "user", "content": "ping"}],
+                    "max_tokens": 1,
+                    "temperature": 0,
+                },
+                timeout=20.0,
+                empty_on=tuple(range(400, 500)),
+            )
+    except Exception as exc:
+        return Check("model endpoint", "fail", f"{base}: {type(exc).__name__}")
+
+    if probe.status_code in (401, 403):
+        return Check("model endpoint", "fail", f"key rejected (HTTP {probe.status_code})")
+    if probe.status_code == 404:
+        return Check("model endpoint", "fail", f"model {settings.model!r} or route not found (404)")
+    if probe.status_code >= 400:
+        return Check("model endpoint", "fail", f"completion probe failed (HTTP {probe.status_code})")
+    return Check("model endpoint", "ok", f"{base} accepted a completion for {settings.model}")
+
+
+def collect_checks(settings: Settings) -> list[Check]:
+    key = settings.resolved_api_key()
+    checks = [
+        Check(
+            "api key",
+            "ok" if key else "warn",
+            "set" if key else "API_KEY is not set",
+        ),
+        Check(
+            "cache dir",
+            "ok" if _writable(cache_dir()) else "fail",
+            str(cache_dir()),
+        ),
+        Check(
+            "log dir",
+            "ok" if _writable(log_dir()) else "fail",
+            str(log_dir()),
+        ),
+    ]
+    for name, variable, why in (
+        ("Have I Been Pwned", "HIBP_API_KEY", "breach collectors will be skipped"),
+        ("GitHub", "GITHUB_TOKEN", "GitHub is limited to 60 requests/hour"),
+    ):
+        present = bool(os.environ.get(variable))
+        checks.append(Check(name, "ok" if present else "warn", "set" if present else why))
+    if key:
+        checks.append(Check("api key shape", "ok" if len(key) >= 20 else "warn", f"{len(key)} chars"))
+    return checks
+
+
+async def diagnose(settings: Settings, *, deep: bool = False) -> list[Check]:
+    return [*collect_checks(settings), await _reach_endpoint(settings, deep)]
+
+
+def failed(checks: list[Check]) -> bool:
+    return any(check.status == "fail" for check in checks)

osintkit/errors.py

@@ -0,0 +1,51 @@
+"""Errors and shared type aliases."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+TargetType = Literal["email", "username", "domain", "company"]
+
+
+class OsintKitError(Exception):
+    """Base error for osintkit."""
+
+
+class CollectorError(OsintKitError):
+    """A collector could not complete its work."""
+
+
+class RateLimited(OsintKitError):
+    """An upstream source asked us to slow down."""
+
+    def __init__(self, message: str, retry_after: float | None = None) -> None:
+        super().__init__(message)
+        self.retry_after = retry_after
+
+
+class SourceUnavailable(OsintKitError):
+    """An upstream source is down or blocked."""
+
+
+class SynthesisError(OsintKitError):
+    """The model returned something we could not validate."""
+
+
+class MissingAPIKey(OsintKitError):
+    """No API key was supplied and none was found in the environment."""
+
+
+class TargetParseError(OsintKitError):
+    """A target string could not be typed."""
+
+
+__all__ = [
+    "CollectorError",
+    "MissingAPIKey",
+    "OsintKitError",
+    "RateLimited",
+    "SourceUnavailable",
+    "SynthesisError",
+    "TargetParseError",
+    "TargetType",
+]

osintkit/http.py

@@ -0,0 +1,272 @@
+"""Resilient HTTP client: retries, exponential backoff, and a warm cache."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import random
+from collections.abc import Iterable, Mapping
+from typing import Any, Self
+
+import httpx
+
+from .config import Settings, cache_read, cache_write
+from .errors import RateLimited, SourceUnavailable
+
+USER_AGENT = "osintkit/0.1 (+public-data-osint; contact: local user)"
+RETRY_STATUS = {429, 500, 502, 503, 504}
+
+
+class HttpClient:
+    """Thin async wrapper with bounded retries and transparent caching.
+
+    Only GETs are needed by the collectors, so that is all this exposes.
+    """
+
+    def __init__(self, settings: Settings, client: httpx.AsyncClient | None = None) -> None:
+        self.settings = settings
+        self._inflight: dict[tuple[str, str], asyncio.Task[httpx.Response]] = {}
+        self._own_client = client is None
+        self._client = client or httpx.AsyncClient(
+            timeout=httpx.Timeout(settings.timeout),
+            limits=httpx.Limits(max_connections=settings.max_connections),
+            follow_redirects=True,
+            headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
+        )
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        if self._own_client:
+            await self._client.aclose()
+
+    @staticmethod
+    def _cache_key(url: str, params: Mapping[str, Any] | None) -> str:
+        if not params:
+            return url
+        return f"{url}?{json.dumps(params, sort_keys=True, default=str)}"
+
+    async def get(
+        self,
+        url: str,
+        *,
+        namespace: str,
+        params: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+        accept: str = "application/json",
+        empty_on: Iterable[int] = (404,),
+        timeout: float | None = None,
+    ) -> httpx.Response:
+        return await self._request(
+            "GET",
+            url,
+            namespace=namespace,
+            params=params,
+            headers=headers,
+            accept=accept,
+            empty_on=empty_on,
+            timeout=timeout,
+        )
+
+    async def post_json(
+        self,
+        url: str,
+        *,
+        namespace: str,
+        payload: Mapping[str, Any],
+        headers: Mapping[str, str] | None = None,
+        timeout: float | None = None,
+        empty_on: Iterable[int] = (),
+    ) -> httpx.Response:
+        return await self._request(
+            "POST",
+            url,
+            namespace=namespace,
+            payload=payload,
+            headers=headers,
+            empty_on=empty_on,
+            timeout=timeout,
+        )
+
+    async def _request(
+        self,
+        method: str,
+        url: str,
+        *,
+        namespace: str,
+        params: Mapping[str, Any] | None = None,
+        payload: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+        accept: str = "application/json",
+        empty_on: Iterable[int] = (),
+        timeout: float | None = None,
+    ) -> httpx.Response:
+        key = self._cache_key(url, payload if method != "GET" else params)
+        if self.settings.use_cache:
+            cached = cache_read(namespace, key, self.settings.cache_ttl)
+            if cached is not None:
+                return httpx.Response(
+                    status_code=cached["status"],
+                    headers=cached.get("headers", {}),
+                    content=cached.get("body", "").encode("utf-8"),
+                    request=httpx.Request(method, url),
+                )
+
+        slot = (namespace, key)
+        pending = self._inflight.get(slot)
+        if pending is not None and not pending.done():
+            return await asyncio.shield(pending)
+
+        task = asyncio.ensure_future(
+            self._perform(
+                method,
+                url,
+                namespace=namespace,
+                key=key,
+                params=params,
+                payload=payload,
+                headers=headers,
+                accept=accept,
+                empty_on=empty_on,
+                timeout=timeout,
+            )
+        )
+        self._inflight[slot] = task
+        try:
+            return await task
+        finally:
+            if self._inflight.get(slot) is task:
+                self._inflight.pop(slot, None)
+
+    async def _perform(
+        self,
+        method: str,
+        url: str,
+        *,
+        namespace: str,
+        key: str,
+        params: Mapping[str, Any] | None = None,
+        payload: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+        accept: str = "application/json",
+        empty_on: Iterable[int] = (),
+        timeout: float | None = None,
+    ) -> httpx.Response:
+        last_error: Exception | None = None
+        for attempt in range(1, self.settings.max_attempts + 1):
+            try:
+                request_headers = {"Accept": accept}
+                if headers:
+                    request_headers.update(headers)
+                if method == "GET":
+                    response = await self._client.get(
+                        url, params=params, headers=request_headers, timeout=timeout
+                    )
+                else:
+                    response = await self._client.post(
+                        url, json=dict(payload or {}), headers=request_headers, timeout=timeout
+                    )
+            except (httpx.TimeoutException, httpx.TransportError) as exc:
+                last_error = exc
+                if attempt >= self.settings.max_attempts:
+                    raise SourceUnavailable(f"{url}: {type(exc).__name__}") from exc
+                await self._sleep(attempt, None)
+                continue
+
+            if response.status_code in empty_on:
+                self._store(namespace, key, response)
+                return response
+
+            if response.status_code in RETRY_STATUS:
+                retry_after = _retry_after(response)
+                if attempt >= self.settings.max_attempts:
+                    raise RateLimited(
+                        f"{url}: HTTP {response.status_code} after {attempt} attempts",
+                        retry_after=retry_after,
+                    )
+                await self._sleep(attempt, retry_after)
+                continue
+
+            if response.status_code == 403 and _quota_exhausted(response):
+                if attempt >= self.settings.max_attempts:
+                    raise RateLimited(f"{url}: quota exhausted after {attempt} attempts")
+                await self._sleep(attempt, _retry_after(response))
+                continue
+
+            if response.status_code >= 400:
+                raise SourceUnavailable(f"{url}: HTTP {response.status_code}")
+
+            self._store(namespace, key, response)
+            return response
+
+        raise SourceUnavailable(f"{url}: {last_error}")
+
+    async def get_json(self, url: str, **kwargs: Any) -> Any:
+        response = await self.get(url, **kwargs)
+        if response.status_code in set(kwargs.get("empty_on") or (404,)):
+            return None
+        try:
+            return response.json()
+        except (json.JSONDecodeError, ValueError) as exc:  # pragma: no cover - malformed upstream
+            raise SourceUnavailable(f"{url}: invalid JSON") from exc
+
+    async def head_ok(self, url: str, *, namespace: str = "presence", **kwargs: Any) -> int:
+        """Return the final status code, treating any HTTP answer as a hit."""
+        response = await self.get(url, namespace=namespace, empty_on=tuple(range(400, 500)), **kwargs)
+        return response.status_code
+
+    def _store(self, namespace: str, key: str, response: httpx.Response) -> None:
+        if not self.settings.use_cache:
+            return
+        try:
+            body = response.text
+        except Exception:  # pragma: no cover - undecodable body
+            body = ""
+        headers = {
+            name: value
+            for name, value in response.headers.items()
+            if name.lower() not in {"content-encoding", "content-length", "transfer-encoding"}
+        }
+        cache_write(
+            namespace,
+            key,
+            {"status": response.status_code, "headers": headers, "body": body},
+        )
+
+    async def _sleep(self, attempt: int, retry_after: float | None) -> None:
+        if retry_after:
+            delay = min(retry_after, 30.0)
+        else:
+            delay = min(0.5 * (2 ** (attempt - 1)), 8.0)
+            delay += random.uniform(0, delay / 2)
+        await asyncio.sleep(delay)
+
+
+def _quota_exhausted(response: httpx.Response) -> bool:
+    return response.headers.get("x-ratelimit-remaining") == "0"
+
+
+def _retry_after(response: httpx.Response) -> float | None:
+    raw = response.headers.get("retry-after")
+    if not raw:
+        return None
+    try:
+        return float(raw)
+    except ValueError:
+        return None
+
+
+def cache_hit(settings: Settings, namespace: str, key: str, ttl: int | None = None) -> Any | None:
+    """Read a cached non-HTTP collector result (DNS, WHOIS, TCP probes)."""
+    if not settings.use_cache:
+        return None
+    return cache_read(namespace, key, settings.cache_ttl if ttl is None else ttl)
+
+
+def cache_put(settings: Settings, namespace: str, key: str, value: Any) -> None:
+    if settings.use_cache:
+        cache_write(namespace, key, value)