osintkit 0.1.1__py3-none-any.whl

osintkit/llm.py

@@ -0,0 +1,91 @@
+"""Minimal OpenAI-compatible chat client."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any, Self
+
+from .config import Settings
+from .errors import MissingAPIKey, SourceUnavailable, SynthesisError
+from .http import HttpClient
+
+
+class LLMClient:
+    def __init__(self, settings: Settings, http: HttpClient | None = None) -> None:
+        self.settings = settings
+        self._own_http = http is None
+        self._http = http or HttpClient(settings)
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        if self._own_http:
+            await self._http.aclose()
+
+    def endpoint(self) -> str:
+        return f"{self.settings.base_url.rstrip('/')}/chat/completions"
+
+    async def chat(
+        self,
+        messages: Sequence[Mapping[str, str]],
+        *,
+        temperature: float | None = None,
+        max_tokens: int = 2400,
+        response_format: Mapping[str, Any] | None = None,
+    ) -> str:
+        api_key = self.settings.resolved_api_key()
+        if not api_key:
+            raise MissingAPIKey("No API key. Set API_KEY in the environment or pass --api-key.")
+
+        payload: dict[str, Any] = {
+            "model": self.settings.model,
+            "messages": [dict(m) for m in messages],
+            "temperature": self.settings.temperature if temperature is None else temperature,
+            "max_tokens": max_tokens,
+            "stream": False,
+        }
+        if response_format:
+            payload["response_format"] = dict(response_format)
+
+        try:
+            response = await self._http.post_json(
+                self.endpoint(),
+                namespace="llm",
+                payload=payload,
+                headers={
+                    "Authorization": f"Bearer {api_key}",
+                    "Content-Type": "application/json",
+                },
+            )
+        except SourceUnavailable as exc:
+            if "HTTP 401" in str(exc) or "HTTP 403" in str(exc):
+                raise SynthesisError("API key rejected; run `osintkit doctor --deep` to confirm it") from exc
+            raise
+        try:
+            body = response.json()
+        except ValueError as exc:
+            raise SynthesisError("Model endpoint returned non-JSON") from exc
+        return extract_content(body)
+
+
+def extract_content(body: Any) -> str:
+    if not isinstance(body, dict):
+        raise SynthesisError("Unexpected model response shape")
+    choices = body.get("choices")
+    if not isinstance(choices, list) or not choices:
+        error = body.get("error")
+        if isinstance(error, dict) and error.get("message"):
+            raise SynthesisError(str(error["message"]))
+        raise SynthesisError("Model response had no choices")
+    message = choices[0].get("message") or {}
+    content = message.get("content")
+    if isinstance(content, list):
+        parts = [c.get("text", "") for c in content if isinstance(c, dict)]
+        content = "".join(parts)
+    if not isinstance(content, str) or not content.strip():
+        raise SynthesisError("Model returned empty content")
+    return content

osintkit/logging_store.py

@@ -0,0 +1,206 @@
+"""Timestamped JSONL run logs and the report archive behind `history`."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from .config import ensure_dirs, log_dir, reports_dir
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat(timespec="milliseconds")
+
+
+def _slug(target: str) -> str:
+    slug = re.sub(r"[^A-Za-z0-9._-]+", "-", target).strip("-")
+    return slug[:60] or "target"
+
+
+class RunLogger:
+    """Appends JSONL lines. One file per day, shared by all runs."""
+
+    def __init__(self, run_id: str, target: str = "", target_type: str = "") -> None:
+        ensure_dirs()
+        self.run_id = run_id
+        self.target = target
+        self.target_type = target_type
+        self.path = log_dir() / f"{datetime.now(UTC):%Y-%m-%d}.jsonl"
+
+    def event(self, kind: str, **fields: Any) -> None:
+        record = {
+            "ts": _now(),
+            "run_id": self.run_id,
+            "kind": kind,
+            "target": self.target,
+            "target_type": self.target_type,
+            **fields,
+        }
+        line = json.dumps(record, ensure_ascii=False, default=str)
+        try:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(line + "\n")
+                handle.flush()
+                os.fsync(handle.fileno())
+        except OSError:
+            return
+
+    def collector(
+        self,
+        *,
+        collector: str,
+        ok: bool,
+        skipped: bool,
+        note: str | None,
+        error: str | None,
+        elapsed_ms: int,
+        size: int,
+    ) -> None:
+        self.event(
+            "collector",
+            collector=collector,
+            ok=ok,
+            skipped=skipped,
+            note=note,
+            error=error,
+            elapsed_ms=elapsed_ms,
+            payload_size=size,
+        )
+
+    def request(
+        self,
+        *,
+        base_url: str,
+        model: str,
+        attempt: int,
+        dry_run: bool,
+        messages: list[dict[str, str]],
+        max_tokens: int,
+    ) -> None:
+        self.event(
+            "model_request",
+            base_url=base_url,
+            model=model,
+            attempt=attempt,
+            dry_run=dry_run,
+            max_tokens=max_tokens,
+            messages=messages,
+        )
+
+    def response(
+        self, *, attempt: int, ok: bool, error: str | None, elapsed_ms: int, content: str | None
+    ) -> None:
+        self.event(
+            "model_response",
+            attempt=attempt,
+            ok=ok,
+            error=error,
+            elapsed_ms=elapsed_ms,
+            content_length=len(content or ""),
+            content=content,
+        )
+
+
+def archive_report(report: dict[str, Any]) -> Path | None:
+    """Persist a full report so `history --show` can replay it."""
+    try:
+        ensure_dirs()
+        root = reports_dir()
+        root.mkdir(parents=True, exist_ok=True)
+        stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S")
+        path = (
+            root / f"{stamp}-{_slug(str(report.get('target', 'target')))}-{int(time.time() * 1000) % 1000}.json"
+        )
+        path.write_text(json.dumps(report, indent=2, ensure_ascii=False, default=str), "utf-8")
+        return path
+    except OSError:
+        return None
+
+
+def iter_log_files() -> list[Path]:
+    root = log_dir()
+    if not root.exists():
+        return []
+    return sorted(root.glob("*.jsonl"), reverse=True)
+
+
+def read_runs(limit: int = 20) -> list[dict[str, Any]]:
+    """Collapse log lines into one summary per run, newest first."""
+    runs: dict[str, dict[str, Any]] = {}
+    order: list[str] = []
+    for path in iter_log_files():
+        try:
+            lines = path.read_text("utf-8").splitlines()
+        except OSError:
+            continue
+        for line in lines:
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            run_id = record.get("run_id")
+            if not run_id:
+                continue
+            if run_id not in runs:
+                runs[run_id] = {
+                    "run_id": run_id,
+                    "started": record.get("ts"),
+                    "target": record.get("target"),
+                    "target_type": record.get("target_type"),
+                    "report": None,
+                    "collectors_ok": 0,
+                    "collectors_failed": 0,
+                    "collectors_skipped": 0,
+                    "risk": None,
+                    "model": None,
+                }
+                order.append(run_id)
+            entry = runs[run_id]
+            kind = record.get("kind")
+            if kind == "collector":
+                if record.get("skipped"):
+                    entry["collectors_skipped"] += 1
+                elif record.get("ok"):
+                    entry["collectors_ok"] += 1
+                else:
+                    entry["collectors_failed"] += 1
+            elif kind == "run_end":
+                entry["report"] = record.get("report_path")
+                entry["risk"] = record.get("risk_level")
+                entry["model"] = record.get("model")
+                entry["ended"] = record.get("ts")
+    summaries = list(reversed(order))[:limit]
+    return [runs[r] for r in summaries]
+
+
+def find_report_for_run(run_id: str) -> str | None:
+    """Return the archived report path recorded for a run id, if any."""
+    for path in iter_log_files():
+        try:
+            lines = path.read_text("utf-8").splitlines()
+        except OSError:
+            continue
+        for line in lines:
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if record.get("run_id") == run_id and record.get("kind") == "run_end":
+                return record.get("report_path")
+    return None
+
+
+def run_exists(run_id: str) -> bool:
+    return any(run["run_id"] == run_id for run in read_runs(limit=500))
+
+
+def read_report(path: str | Path) -> dict[str, Any] | None:
+    try:
+        return json.loads(Path(path).read_text("utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None

osintkit/models.py

@@ -0,0 +1,228 @@
+"""Pydantic schemas for collector output, model output, and reports."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+RiskLevel = Literal["low", "medium", "high"]
+
+
+def _utcnow_iso() -> str:
+    return datetime.now(UTC).isoformat(timespec="seconds")
+
+
+def _as_str_list(value: Any) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        parts = [p.strip(" -•") for p in value.replace(";", "\n").split("\n")]
+        return [p for p in parts if p]
+    if isinstance(value, dict):
+        return [f"{k}: {v}" if v else str(k) for k, v in value.items()]
+    if isinstance(value, (list, tuple, set)):
+        out: list[str] = []
+        for item in value:
+            if isinstance(item, str):
+                if item.strip():
+                    out.append(item.strip())
+            elif isinstance(item, dict):
+                out.extend(_as_str_list(item.get("value") or item.get("name") or item))
+            elif item is not None:
+                out.append(str(item))
+        return out
+    return [str(value)]
+
+
+class CollectorResult(BaseModel):
+    """One collector's outcome for one target. Never raises."""
+
+    model_config = ConfigDict(extra="allow")
+
+    collector: str
+    target: str
+    target_type: str
+    ok: bool = True
+    skipped: bool = False
+    note: str | None = None
+    error: str | None = None
+    data: Any = None
+    elapsed_ms: int = 0
+    cached: bool = False
+
+    @property
+    def has_data(self) -> bool:
+        if self.skipped or not self.ok or self.data is None:
+            return False
+        if isinstance(self.data, (list, dict, str)):
+            return len(self.data) > 0
+        return True
+
+
+class Breach(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    name: str
+    date: str | None = None
+    data_types: list[str] = Field(default_factory=list)
+    source: str | None = None
+
+    @field_validator("name", "date", "source", mode="before")
+    @classmethod
+    def _stringify(cls, value: Any) -> Any:
+        if value is None:
+            return None
+        if isinstance(value, (list, tuple)):
+            return ", ".join(str(v) for v in value)
+        return str(value)
+
+    @field_validator("data_types", mode="before")
+    @classmethod
+    def _types(cls, value: Any) -> list[str]:
+        return _as_str_list(value)
+
+
+class AssociatedAccount(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    platform: str
+    url: str | None = None
+    handle: str | None = None
+    note: str | None = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def _normalize(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return {"platform": value}
+        return value
+
+    @field_validator("platform", "url", "handle", "note", mode="before")
+    @classmethod
+    def _stringify(cls, value: Any) -> Any:
+        if value is None:
+            return None
+        if isinstance(value, (list, tuple)):
+            return ", ".join(str(v) for v in value)
+        return str(value)
+
+
+class Analysis(BaseModel):
+    """The strict shape we demand back from the model."""
+
+    model_config = ConfigDict(extra="allow")
+
+    summary: str = ""
+    breaches: list[Breach] = Field(default_factory=list)
+    associated_accounts: list[AssociatedAccount] = Field(default_factory=list)
+    infrastructure_notes: list[str] = Field(default_factory=list)
+    subdomains: list[str] = Field(default_factory=list)
+    risk_level: RiskLevel = "low"
+    recommendations: list[str] = Field(default_factory=list)
+
+    @field_validator("summary", mode="before")
+    @classmethod
+    def _summary(cls, value: Any) -> str:
+        if value is None:
+            return ""
+        if isinstance(value, (list, dict)):
+            return "\n".join(_as_str_list(value))
+        return str(value).strip()
+
+    @field_validator("subdomains", "recommendations", "infrastructure_notes", mode="before")
+    @classmethod
+    def _lists(cls, value: Any) -> list[str]:
+        return _as_str_list(value)
+
+    @field_validator("breaches", mode="before")
+    @classmethod
+    def _breaches(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return [{"name": value}] if value.strip() else []
+        return value or []
+
+    @field_validator("associated_accounts", mode="before")
+    @classmethod
+    def _accounts(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return [{"platform": value}] if value.strip() else []
+        return value or []
+
+    @field_validator("subdomains")
+    @classmethod
+    def _dedupe(cls, value: list[str]) -> list[str]:
+        seen: set[str] = set()
+        out: list[str] = []
+        for item in value:
+            key = item.strip().lower()
+            if key and key not in seen:
+                seen.add(key)
+                out.append(item.strip())
+        return out
+
+    @field_validator("risk_level", mode="before")
+    @classmethod
+    def _risk(cls, value: Any) -> Any:
+        if value is None:
+            return "low"
+        text = str(value).strip().lower()
+        for level in ("high", "medium", "low"):
+            if level in text:
+                return level
+        return "low"
+
+
+class Report(BaseModel):
+    """A rendered-ready record of one target lookup."""
+
+    schema_version: str = "1"
+    target: str
+    target_type: str
+    generated_at: str = Field(default_factory=_utcnow_iso)
+    model: str | None = None
+    base_url: str | None = None
+    elapsed_ms: int = 0
+    collectors: list[CollectorResult] = Field(default_factory=list)
+    analysis: Analysis | None = None
+    notes: list[str] = Field(default_factory=list)
+
+    @property
+    def ok_collectors(self) -> list[CollectorResult]:
+        return [c for c in self.collectors if c.has_data]
+
+
+class PromptBundle(BaseModel):
+    """The package handed to the model."""
+
+    target: str
+    target_type: str
+    collected_at: str = Field(default_factory=_utcnow_iso)
+    results: list[CollectorResult] = Field(default_factory=list)
+    signals: dict[str, Any] = Field(default_factory=dict)
+
+    def to_prompt_json(self) -> str:
+        payload = {
+            "target": self.target,
+            "target_type": self.target_type,
+            "signals": self.signals,
+            "sources": [
+                {
+                    "source": r.collector,
+                    "status": "ok" if r.ok else ("skipped" if r.skipped else "error"),
+                    "note": r.note,
+                    "data": r.data,
+                }
+                for r in self.results
+                if r.ok and not r.skipped
+            ],
+            "unavailable_sources": [
+                {"source": r.collector, "reason": r.note or r.error or "unknown"}
+                for r in self.results
+                if not r.ok or r.skipped
+            ],
+        }
+        import json
+
+        return json.dumps(payload, indent=2, ensure_ascii=False, default=str)

osintkit/normalise.py

@@ -0,0 +1,131 @@
+"""Collapse duplicate breach records across sources before prompting."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from .models import CollectorResult
+
+BREACH_SOURCES = frozenset({"hibp_breaches", "xposed_or_not", "leakcheck_public"})
+KEY_RE = re.compile(r"[^a-z0-9]+")
+CREDENTIAL_RE = re.compile(r"pass|hash|credential|secret|token|api[ _-]?key", re.I)
+SEVERITY = ("low", "medium", "high")
+
+
+def signals_for(merged: list[dict[str, Any]]) -> dict[str, Any]:
+    """Compute risk signals the model can check itself against."""
+    credential = [
+        entry
+        for entry in merged
+        if any(CREDENTIAL_RE.search(str(kind)) for kind in entry.get("data_types", []))
+    ]
+    count = len(merged)
+    return {
+        "breach_count": count,
+        "credential_bearing": len(credential),
+        "risk_floor": _floor(count, len(credential)),
+    }
+
+
+def _floor(count: int, credential: int) -> str:
+    if credential >= 3 or count >= 20:
+        return "high"
+    if credential >= 1 or count >= 5:
+        return "medium"
+    return "low"
+
+
+def apply_floor(chosen: str, floor: str) -> str:
+    """Never let the model rank below the deterministic floor."""
+    rank = {name: index for index, name in enumerate(SEVERITY)}
+    return chosen if rank.get(chosen, 0) >= rank.get(floor, 0) else floor
+
+
+def _key(name: str) -> str:
+    return KEY_RE.sub("", name.lower())
+
+
+def _strings(value: Any) -> list[str]:
+    if isinstance(value, str):
+        return [part.strip() for part in re.split(r"[;,]", value) if part.strip()]
+    if isinstance(value, list):
+        return [str(item).strip() for item in value if str(item).strip()]
+    return []
+
+
+def _records(result: CollectorResult) -> list[dict[str, Any]]:
+    data = result.data
+    if isinstance(data, list):
+        return [entry for entry in data if isinstance(entry, dict)]
+    if isinstance(data, dict):
+        for field in ("breaches", "sources"):
+            entries = data.get(field)
+            if isinstance(entries, list):
+                return [entry for entry in entries if isinstance(entry, dict)]
+    return []
+
+
+def merge_breaches(results: list[CollectorResult]) -> tuple[list[CollectorResult], int]:
+    """Merge duplicate breach names across sources.
+
+    Returns the results to prompt with (breach sources replaced by one index)
+    and the number of duplicate records folded away.
+    """
+    breach_results = [r for r in results if r.collector in BREACH_SOURCES and r.has_data]
+    if not breach_results:
+        return results, 0
+
+    merged: dict[str, dict[str, Any]] = {}
+    duplicates = 0
+    for result in breach_results:
+        records = _records(result)
+        if not records:
+            continue
+        source = str(records[0].get("source") or result.collector)
+        for entry in records:
+            name = str(entry.get("name") or "").strip()
+            if not name:
+                continue
+            key = _key(name)
+            existing = merged.get(key)
+            if existing is None:
+                merged[key] = {
+                    "name": name,
+                    "date": entry.get("date"),
+                    "data_types": set(_strings(entry.get("data_types"))),
+                    "sources": {source},
+                }
+                continue
+            duplicates += 1
+            if not existing["date"] and entry.get("date"):
+                existing["date"] = entry["date"]
+            existing["data_types"].update(_strings(entry.get("data_types")))
+            existing["sources"].add(source)
+
+    if not merged:
+        return results, duplicates
+
+    rendered = [
+        {
+            "name": item["name"],
+            "date": item["date"],
+            "data_types": sorted(item["data_types"]),
+            "sources": sorted(item["sources"]),
+            "source": ", ".join(sorted(item["sources"])),
+        }
+        for item in merged.values()
+    ]
+    rendered.sort(key=lambda item: (-len(item["data_types"]), item["name"].lower()))
+
+    index = CollectorResult(
+        collector="breach_index",
+        target=breach_results[0].target,
+        target_type=breach_results[0].target_type,
+        data=rendered,
+    )
+    kept = [r for r in results if r.collector not in BREACH_SOURCES]
+    kept.extend(r for r in results if r.collector in BREACH_SOURCES and r.skipped)
+    position = next((i for i, r in enumerate(results) if r.collector in BREACH_SOURCES), len(kept))
+    kept.insert(min(position, len(kept)), index)
+    return kept, duplicates