voidaccess 1.3.0__py3-none-any.whl

cli/config.py

@@ -0,0 +1,180 @@
+"""
+cli/config.py — Persistent config for the voidaccess CLI.
+
+Stores LLM provider/model, API keys, Tor proxy settings, and output dir
+in ~/.voidaccess/config.json. Exposes helpers and an apply_env() function
+that pushes the saved config into os.environ before any voidaccess module
+is imported (the existing modules read API keys from env at import time).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Optional
+
+CLI_HOME = Path(os.path.expanduser("~/.voidaccess"))
+CONFIG_PATH = CLI_HOME / "config.json"
+DB_PATH = CLI_HOME / "investigations.db"
+DEFAULT_OUTPUT_DIR = CLI_HOME / "results"
+
+ENRICHMENT_KEYS = [
+    "OTX_API_KEY",
+    "VT_API_KEY",
+    "ABUSEIPDB_API_KEY",
+    "GREYNOISE_API_KEY",
+    "URLSCAN_API_KEY",
+    "SECURITYTRAILS_API_KEY",
+    "GITHUB_TOKEN",
+    "GITLAB_TOKEN",
+    "HYBRID_ANALYSIS_API_KEY",
+    "HIBP_API_KEY",
+    "EMAILREP_API_KEY",
+    "SHODAN_API_KEY",
+    "BLOCKCYPHER_TOKEN",
+    "ETHERSCAN_API_KEY",
+    "DEEPL_API_KEY",
+    "DARKSEARCH_API_KEY",
+]
+
+PROVIDER_ENV = {
+    "openrouter": "OPENROUTER_API_KEY",
+    "groq":       "GROQ_API_KEY",
+    "google":     "GOOGLE_API_KEY",
+    "openai":     "OPENAI_API_KEY",
+    "anthropic":  "ANTHROPIC_API_KEY",
+    "ollama":     None,
+}
+
+DEFAULT_CONFIG: dict[str, Any] = {
+    "llm": {
+        "provider": "openrouter",
+        "model": "openrouter/deepseek/deepseek-chat",
+        "api_key": "",
+    },
+    "enrichment_keys": {k: "" for k in ENRICHMENT_KEYS},
+    "tor": {
+        "host": "127.0.0.1",
+        "port": 9050,
+    },
+    "output_dir": str(DEFAULT_OUTPUT_DIR),
+}
+
+
+def _ensure_home() -> None:
+    CLI_HOME.mkdir(parents=True, exist_ok=True)
+
+
+def load_config() -> dict[str, Any]:
+    """Return saved config or DEFAULT_CONFIG if none exists."""
+    _ensure_home()
+    if not CONFIG_PATH.exists():
+        return json.loads(json.dumps(DEFAULT_CONFIG))
+    try:
+        cfg = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
+    except Exception:
+        return json.loads(json.dumps(DEFAULT_CONFIG))
+    # Merge with defaults so missing keys don't crash
+    merged = json.loads(json.dumps(DEFAULT_CONFIG))
+    merged["llm"].update(cfg.get("llm", {}))
+    merged["tor"].update(cfg.get("tor", {}))
+    merged["enrichment_keys"].update(cfg.get("enrichment_keys", {}))
+    if cfg.get("output_dir"):
+        merged["output_dir"] = cfg["output_dir"]
+    return merged
+
+
+def save_config(config: dict[str, Any]) -> None:
+    _ensure_home()
+    CONFIG_PATH.write_text(
+        json.dumps(config, indent=2, sort_keys=True),
+        encoding="utf-8",
+    )
+
+
+def is_configured() -> bool:
+    if not CONFIG_PATH.exists():
+        return False
+    cfg = load_config()
+    provider = cfg.get("llm", {}).get("provider", "")
+    api_key = cfg.get("llm", {}).get("api_key", "")
+    if provider == "ollama":
+        return True
+    return bool(provider and api_key)
+
+
+def get_llm_key(config: Optional[dict[str, Any]] = None) -> str:
+    cfg = config or load_config()
+    return cfg.get("llm", {}).get("api_key", "") or ""
+
+
+def get_llm_model(config: Optional[dict[str, Any]] = None) -> str:
+    cfg = config or load_config()
+    return cfg.get("llm", {}).get("model", "") or ""
+
+
+def get_llm_provider(config: Optional[dict[str, Any]] = None) -> str:
+    cfg = config or load_config()
+    return cfg.get("llm", {}).get("provider", "") or ""
+
+
+def get_tor_proxy(config: Optional[dict[str, Any]] = None) -> str:
+    cfg = config or load_config()
+    host = cfg.get("tor", {}).get("host", "127.0.0.1")
+    port = cfg.get("tor", {}).get("port", 9050)
+    return f"socks5://{host}:{port}"
+
+
+def get_output_dir(config: Optional[dict[str, Any]] = None) -> Path:
+    cfg = config or load_config()
+    p = Path(os.path.expanduser(cfg.get("output_dir") or str(DEFAULT_OUTPUT_DIR)))
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def db_url() -> str:
+    """SQLite URL used by db.session via DATABASE_URL env var."""
+    _ensure_home()
+    return f"sqlite:///{DB_PATH.as_posix()}"
+
+
+def apply_env(config: Optional[dict[str, Any]] = None) -> None:
+    """
+    Push saved config into os.environ so that the existing voidaccess
+    modules (config.py, llm.py, sources/*) pick up the values at import.
+
+    Must be called BEFORE any voidaccess module is imported.
+    """
+    cfg = config or load_config()
+
+    os.environ.setdefault("DATABASE_URL", db_url())
+    os.environ.setdefault("JWT_SECRET", "voidaccess-cli-local-no-auth")
+    os.environ.setdefault("DISABLE_RATE_LIMIT", "true")
+    os.environ.setdefault("PLAYWRIGHT_ENABLED", "false")
+
+    def _set_env_if_present(key: str, value: Any, *, clear_if_empty: bool = False) -> None:
+        text = str(value).strip() if value is not None else ""
+        if text:
+            os.environ[key] = text
+        elif clear_if_empty:
+            os.environ.pop(key, None)
+
+    # Tor proxy
+    _set_env_if_present("TOR_PROXY_HOST", cfg.get("tor", {}).get("host", "127.0.0.1"))
+    _set_env_if_present("TOR_PROXY_PORT", cfg.get("tor", {}).get("port", 9050))
+
+    # LLM provider key (push under its canonical env var name)
+    provider = cfg.get("llm", {}).get("provider", "")
+    api_key = cfg.get("llm", {}).get("api_key", "")
+    env_name = PROVIDER_ENV.get(provider)
+    if env_name:
+        _set_env_if_present(env_name, api_key, clear_if_empty=True)
+
+    # Default model
+    default_model = cfg.get("llm", {}).get("model", "")
+    _set_env_if_present("DEFAULT_MODEL", default_model)
+
+    # Enrichment keys
+    for k, v in (cfg.get("enrichment_keys") or {}).items():
+        _set_env_if_present(k, v, clear_if_empty=True)

cli/display.py

@@ -0,0 +1,212 @@
+"""
+cli/display.py — Rich live display for investigations.
+
+Three-zone layout:
+    title bar     — query + elapsed timer
+    step table    — pipeline stages with status icons
+    activity line — current URL / sub-task detail
+
+Status icons:
+    pending  · gray dot
+    active   ⠹ spinner (cycles in tick())
+    ok       ✓ green
+    fail     ✗ red
+    skip     ↷ yellow
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Optional
+
+from rich.console import Console, Group
+from rich.live import Live
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+
+SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
+
+STATUS_GLYPH = {
+    "pending": ("·", "grey50"),
+    "active":  ("⠹", "cyan"),
+    "ok":      ("✓", "green"),
+    "fail":    ("✗", "red"),
+    "skip":    ("↷", "yellow"),
+}
+
+
+@dataclass
+class StepRow:
+    name: str
+    status: str = "pending"
+    detail: str = ""
+    substeps: list[tuple[str, str]] = field(default_factory=list)  # (label, status)
+
+
+class InvestigationDisplay:
+    """Live terminal display driven by .update_step() and .update_current_url()."""
+
+    def __init__(self, console: Optional[Console] = None, quiet: bool = False):
+        self.console = console or Console()
+        self.quiet = quiet
+        self._live: Optional[Live] = None
+        self._query: str = ""
+        self._start_ts: float = time.monotonic()
+        self._steps: list[StepRow] = []
+        self._current_url: str = ""
+        self._spinner_index = 0
+        self._final_summary: Optional[dict] = None
+        self._error: Optional[str] = None
+
+    # -- lifecycle ----------------------------------------------------------
+
+    def start(self, query: str, steps: Optional[list[str]] = None) -> None:
+        self._query = query
+        self._start_ts = time.monotonic()
+        names = steps or [
+            "Refining query",
+            "Searching dark web",
+            "Filtering results",
+            "Scraping pages",
+            "Extracting entities",
+            "Enriching intelligence",
+            "Building graph",
+            "Generating summary",
+            "Finalizing results",
+        ]
+        self._steps = [StepRow(name=n) for n in names]
+        if self.quiet:
+            self.console.print(f"[bold]VoidAccess[/bold] — {query}")
+            return
+        self._live = Live(
+            self._render(),
+            console=self.console,
+            refresh_per_second=8,
+            transient=False,
+        )
+        self._live.start()
+
+    def stop(self) -> None:
+        if self._live is not None:
+            self._live.update(self._render(), refresh=True)
+            self._live.stop()
+            self._live = None
+
+    # -- updates ------------------------------------------------------------
+
+    def update_step(self, step_name: str, status: str, detail: str = "") -> None:
+        row = self._find_step(step_name)
+        if row is None:
+            row = StepRow(name=step_name)
+            self._steps.append(row)
+        row.status = status
+        if detail:
+            row.detail = detail
+        self._refresh()
+        if self.quiet:
+            icon = STATUS_GLYPH.get(status, ("·", "grey50"))[0]
+            d = f" — {detail}" if detail else ""
+            self.console.print(f"  {icon} {step_name}{d}")
+
+    def update_substep(self, step_name: str, label: str, status: str) -> None:
+        row = self._find_step(step_name)
+        if row is None:
+            return
+        for idx, (existing, _) in enumerate(row.substeps):
+            if existing == label:
+                row.substeps[idx] = (label, status)
+                self._refresh()
+                return
+        row.substeps.append((label, status))
+        self._refresh()
+
+    def update_current_url(self, url: str) -> None:
+        self._current_url = url
+        self._refresh()
+
+    def complete(self, summary: dict) -> None:
+        self._final_summary = summary
+        self.stop()
+        self._print_completion(summary)
+
+    def error(self, msg: str) -> None:
+        self._error = msg
+        self.stop()
+        self.console.print(f"[bold red]Investigation failed:[/bold red] {msg}")
+
+    # -- render -------------------------------------------------------------
+
+    def _refresh(self) -> None:
+        self._spinner_index = (self._spinner_index + 1) % len(SPINNER_FRAMES)
+        if self._live is not None:
+            self._live.update(self._render(), refresh=True)
+
+    def _find_step(self, name: str) -> Optional[StepRow]:
+        for row in self._steps:
+            if row.name == name:
+                return row
+        return None
+
+    def _render(self) -> Panel:
+        elapsed = time.monotonic() - self._start_ts
+        title = Text()
+        title.append("VoidAccess", style="bold magenta")
+        title.append(f" — \"{self._query}\"", style="bold white")
+        title.append(f"   Elapsed: {self._fmt_elapsed(elapsed)}", style="grey50")
+
+        table = Table.grid(padding=(0, 1))
+        table.add_column(width=2)
+        table.add_column(no_wrap=False)
+        for row in self._steps:
+            glyph, colour = STATUS_GLYPH.get(row.status, ("·", "grey50"))
+            if row.status == "active":
+                glyph = SPINNER_FRAMES[self._spinner_index]
+            line = Text()
+            line.append(f"{glyph} ", style=colour)
+            line.append(row.name, style="white" if row.status != "pending" else "grey50")
+            if row.detail:
+                line.append(f"  ({row.detail})", style="grey62")
+            table.add_row("", line)
+            for sub_label, sub_status in row.substeps:
+                sg, sc = STATUS_GLYPH.get(sub_status, ("·", "grey50"))
+                sub = Text(f"   {sg} {sub_label}", style=sc)
+                table.add_row("", sub)
+
+        activity = Text()
+        if self._current_url:
+            activity.append("Fetching: ", style="bold")
+            activity.append(self._current_url, style="cyan")
+        else:
+            activity.append("", style="grey50")
+
+        body = Group(title, Text(""), table, Text(""), activity)
+        return Panel(body, border_style="magenta", padding=(1, 2))
+
+    @staticmethod
+    def _fmt_elapsed(secs: float) -> str:
+        m, s = divmod(int(secs), 60)
+        return f"{m}m {s:02d}s"
+
+    def _print_completion(self, summary: dict) -> None:
+        table = Table.grid(padding=(0, 2))
+        table.add_column(style="bold")
+        table.add_column()
+        table.add_row("Entities", str(summary.get("entity_count", "—")))
+        table.add_row("Pages", str(summary.get("page_count", "—")))
+        if "c2_ips" in summary:
+            table.add_row("C2 IPs", f"{summary['c2_ips']} confirmed")
+        table.add_row("Sources", str(summary.get("sources_used", "—")))
+        if summary.get("report_path"):
+            table.add_row("Report", str(summary["report_path"]))
+        if summary.get("data_path"):
+            table.add_row("Data", str(summary["data_path"]))
+
+        panel = Panel(
+            Group(Text("✓ Investigation complete", style="bold green"), Text(""), table),
+            border_style="green",
+            padding=(1, 2),
+        )
+        self.console.print(panel)

cli/main.py

@@ -0,0 +1,154 @@
+"""
+cli/main.py — typer entry point exposed as the `voidaccess` script.
+
+Defined as the [project.scripts] target in pyproject.toml:
+    voidaccess = "cli.main:app"
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+
+# Force UTF-8 on Windows consoles so rich glyphs render reliably
+if sys.platform == "win32":
+    os.environ.setdefault("PYTHONIOENCODING", "utf-8")
+    try:
+        sys.stdout.reconfigure(encoding="utf-8")  # type: ignore[attr-defined]
+        sys.stderr.reconfigure(encoding="utf-8")  # type: ignore[attr-defined]
+    except Exception:
+        pass
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from cli import __version__
+from cli import config as cli_config
+from cli.commands import configure, enrich, export, investigate, show
+
+console = Console()
+
+app = typer.Typer(
+    name="voidaccess",
+    help="Dark web OSINT — query to intelligence report.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+# Sub-commands
+app.add_typer(configure.app, name="configure", help="Configure the CLI (LLM, keys, Tor).")
+app.command("investigate", help="Run a new investigation.")(investigate.run)
+app.command("show", help="Open the entity browser TUI.")(show.run)
+app.command("export", help="Export an investigation to STIX/MISP/Sigma/CSV/MD/JSON.")(export.run)
+app.command("enrich", help="Re-enrich a stored investigation against current feeds.")(enrich.run)
+
+
+def _ensure_first_run() -> None:
+    """Auto-launch wizard on first invocation when no config exists."""
+    if cli_config.CONFIG_PATH.exists():
+        return
+    console.print(
+        "[bold magenta]Welcome to voidaccess.[/bold magenta] "
+        "Let's get you configured first."
+    )
+    # Invoke wizard via Typer
+    try:
+        configure.configure_default(ctx=typer.Context(configure.app))
+    except Exception:
+        pass
+
+
+@app.command("status")
+def status() -> None:
+    """Show current config, Tor status, and detected API keys."""
+    from cli.tor_detect import detect_tor
+    cli_config.apply_env()
+    cfg = cli_config.load_config()
+
+    table = Table(title="voidaccess status", show_lines=False)
+    table.add_column("Setting", style="bold")
+    table.add_column("Value")
+    table.add_row("Version", __version__)
+    table.add_row("Config path", str(cli_config.CONFIG_PATH))
+    table.add_row("DB path", str(cli_config.DB_PATH))
+    table.add_row("Output dir", str(cli_config.get_output_dir(cfg)))
+
+    llm = cfg.get("llm", {})
+    table.add_row("LLM provider", llm.get("provider") or "—")
+    table.add_row("LLM model", llm.get("model") or "—")
+    table.add_row("LLM key", "[green]set[/green]" if llm.get("api_key") else "[red]missing[/red]")
+
+    table.add_row("Tor host", cfg.get("tor", {}).get("host", "—"))
+    table.add_row("Tor port", str(cfg.get("tor", {}).get("port", "—")))
+
+    tor_status = detect_tor()
+    if tor_status.proxy_url:
+        table.add_row("Tor reachable", f"[green]{tor_status.source}[/green] at {tor_status.proxy_url}")
+    else:
+        table.add_row("Tor reachable", "[red]no proxy responded[/red]")
+
+    try:
+        import spacy
+
+        spacy.load("en_core_web_sm")
+        spacy_status = "ready"
+    except Exception:
+        spacy_status = "not installed"
+    table.add_row("spaCy NER", spacy_status)
+
+    keys = cfg.get("enrichment_keys", {})
+    set_count = sum(1 for v in keys.values() if v)
+    table.add_row("Enrichment keys", f"{set_count}/{len(keys)} set")
+
+    console.print(table)
+
+
+@app.command("list")
+def list_investigations(
+    limit: int = typer.Option(20, "--limit", "-n", help="Number of rows"),
+    as_json: bool = typer.Option(False, "--json", help="JSON output"),
+) -> None:
+    """List saved investigations."""
+    cli_config.apply_env()
+    from cli.adapters import sqlite as sqlite_adapter
+    sqlite_adapter.init_db()
+    rows = sqlite_adapter.list_investigations(limit=limit)
+    if as_json:
+        console.print_json(json.dumps(rows, default=str))
+        return
+    if not rows:
+        console.print("[grey50]No saved investigations.[/grey50]")
+        return
+    table = Table(title="Saved investigations")
+    table.add_column("Id", style="cyan")
+    table.add_column("Query")
+    table.add_column("Status")
+    table.add_column("Entities", justify="right")
+    table.add_column("Created")
+    for r in rows:
+        table.add_row(
+            r["id"][:8],
+            (r["query"] or "")[:60],
+            r["status"] or "",
+            str(r["entity_count"]),
+            (r["created_at"] or "")[:19],
+        )
+    console.print(table)
+
+
+@app.command("version")
+def version() -> None:
+    """Print the installed version."""
+    console.print(f"voidaccess {__version__}")
+
+
+@app.callback()
+def _global(ctx: typer.Context) -> None:
+    """Set env vars before sub-commands import voidaccess modules."""
+    cli_config.apply_env()
+
+
+if __name__ == "__main__":
+    app()

cli/tor_detect.py

@@ -0,0 +1,71 @@
+"""
+cli/tor_detect.py — Locate a running Tor SOCKS5 proxy.
+
+Probe order:
+    1. 127.0.0.1:9050  (system tor)
+    2. 127.0.0.1:9150  (Tor Browser bundle)
+    3. config override (cli/config.py)
+
+Tests by performing a SOCKS5 handshake — no full HTTP round-trip required,
+so detection stays fast even when the wider Tor network is slow.
+"""
+
+from __future__ import annotations
+
+import socket
+import struct
+from dataclasses import dataclass
+from typing import Optional
+
+from cli.config import load_config
+
+
+@dataclass
+class TorStatus:
+    proxy_url: Optional[str]
+    source: str  # "system_tor" | "tor_browser" | "config" | "none"
+    host: Optional[str] = None
+    port: Optional[int] = None
+
+
+def _socks5_handshake(host: str, port: int, timeout: float = 2.0) -> bool:
+    """Open TCP, send a SOCKS5 NO-AUTH greeting, expect 0x05 0x00 back."""
+    try:
+        with socket.create_connection((host, port), timeout=timeout) as sock:
+            sock.sendall(b"\x05\x01\x00")
+            resp = sock.recv(2)
+            return len(resp) == 2 and resp[0] == 0x05 and resp[1] == 0x00
+    except Exception:
+        return False
+
+
+def detect_tor() -> TorStatus:
+    cfg = load_config()
+    cfg_host = cfg.get("tor", {}).get("host", "127.0.0.1")
+    cfg_port = int(cfg.get("tor", {}).get("port", 9050))
+
+    probes = [
+        ("system_tor", "127.0.0.1", 9050),
+        ("tor_browser", "127.0.0.1", 9150),
+    ]
+    if (cfg_host, cfg_port) not in {(h, p) for _, h, p in probes}:
+        probes.append(("config", cfg_host, cfg_port))
+
+    for source, host, port in probes:
+        if _socks5_handshake(host, port):
+            return TorStatus(
+                proxy_url=f"socks5://{host}:{port}",
+                source=source,
+                host=host,
+                port=port,
+            )
+
+    return TorStatus(proxy_url=None, source="none")
+
+
+def tor_unavailable_message() -> str:
+    return (
+        "Tor not found. Install: https://torproject.org\n"
+        "Or run Tor Browser before investigating.\n"
+        "Use --no-tor to skip Tor entirely (clearnet sources only)."
+    )