rekos 1.3.0__py3-none-any.whl

rekos/__init__.py

@@ -0,0 +1,5 @@
+"""REKOS core package."""
+
+__all__ = ["__version__"]
+
+__version__ = "1.3.0"

rekos/adapters/__init__.py

@@ -0,0 +1,18 @@
+"""Passive OSINT source adapters."""
+
+from .base import AdapterResult, BaseSourceAdapter, SourceRunResult
+from .maigret import MaigretAdapter
+from .sherlock import SherlockAdapter, SherlockUsernameAdapter
+from .web_osint import DnsDomainAdapter
+from .wmn import WmnUsernameAdapter
+
+__all__ = [
+    "AdapterResult",
+    "BaseSourceAdapter",
+    "DnsDomainAdapter",
+    "MaigretAdapter",
+    "SherlockAdapter",
+    "SherlockUsernameAdapter",
+    "SourceRunResult",
+    "WmnUsernameAdapter",
+]

rekos/adapters/base.py

@@ -0,0 +1,104 @@
+"""Base interface for passive OSINT source adapters."""
+
+from __future__ import annotations
+
+import shutil
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from rekos.storage import CaseStore
+
+
+@dataclass(frozen=True)
+class AdapterResult:
+    source: str
+    target: str
+    url: str
+    platform: str
+    confidence: str
+    raw_reference: str
+
+
+@dataclass(frozen=True)
+class SourceRunResult:
+    source: str
+    target: str
+    raw_output: str
+    results: list[AdapterResult]
+    artifacts: list[Path]
+    skipped: bool = False
+
+
+class BaseSourceAdapter:
+    name: str = ""
+    description: str = ""
+    supported_target_types: tuple[str, ...] = ()
+    passive_only: bool = True
+    external_dependencies: tuple[str, ...] = ()
+
+    def dependency_status(self) -> dict[str, bool]:
+        return {
+            dependency: shutil.which(dependency) is not None
+            for dependency in self.external_dependencies
+        }
+
+    def missing_dependencies(self) -> list[str]:
+        return [
+            dependency
+            for dependency, available in self.dependency_status().items()
+            if not available
+        ]
+
+    def execute(self, case: str, target: str, store: CaseStore) -> SourceRunResult:
+        missing = self.missing_dependencies()
+        if missing:
+            from rekos.errors import ExternalToolMissingError
+
+            raise ExternalToolMissingError(
+                f"Missing dependencies for {self.name}: {', '.join(missing)}."
+            )
+        raw_output = self.run(case, target)
+        artifact_path = self._write_source_output(case, target, store, raw_output)
+        results = self.parse_results(target, raw_output)
+        store.add_adapter_results(case, results)
+        store.add_timeline_event(case, "source.run", f"Ran source {self.name} for {target}")
+        return SourceRunResult(
+            source=self.name,
+            target=target,
+            raw_output=raw_output,
+            results=results,
+            artifacts=[artifact_path],
+        )
+
+    def run(self, case: str, target: str) -> str:
+        raise NotImplementedError
+
+    def parse_results(self, target: str, raw_output: str) -> list[AdapterResult]:
+        raise NotImplementedError
+
+    def _write_source_output(
+        self,
+        case: str,
+        target: str,
+        store: CaseStore,
+        raw_output: str,
+    ) -> Path:
+        sources_folder = store.exports_folder(case) / "sources"
+        sources_folder.mkdir(exist_ok=True)
+        stem = f"{int(time.time())}-{self.name}-{_safe_export_name(target)}"
+        path = sources_folder / f"{stem}.txt"
+        counter = 2
+        while path.exists():
+            path = sources_folder / f"{stem}-{counter}.txt"
+            counter += 1
+        path.write_text(raw_output, encoding="utf-8")
+        return path
+
+
+def _safe_export_name(value: str) -> str:
+    cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "-", value.strip()).strip(".-")
+    return (cleaned or "target")[:80]

rekos/adapters/http_snapshot.py

@@ -0,0 +1,104 @@
+"""HTTP snapshot passive URL adapter."""
+
+from __future__ import annotations
+
+import json
+from urllib.parse import urlparse
+
+from rekos.snapshots import fetch_public_url, normalize_public_url, snapshot_url
+
+from .base import AdapterResult, BaseSourceAdapter, SourceRunResult
+
+
+class HttpSnapshotAdapter(BaseSourceAdapter):
+    name = "http_snapshot"
+    description = "Capture a public HTTP(S) URL snapshot as a local evidence artifact."
+    supported_target_types = ("url",)
+    passive_only = True
+    external_dependencies: tuple[str, ...] = ()
+
+    def execute(self, case: str, target: str, store) -> SourceRunResult:
+        result = snapshot_url(case, target, store)
+        raw_output = json.dumps(
+            {
+                "url": result.url,
+                "skipped": result.skipped,
+                "headers_path": str(result.headers_path or ""),
+                "body_path": str(result.body_path or ""),
+                "screenshot_path": str(result.screenshot_path or ""),
+                "error": result.error or "",
+            },
+            indent=2,
+            sort_keys=True,
+        )
+        adapter_result = AdapterResult(
+            source=self.name,
+            target=result.url,
+            url=result.url,
+            platform=_platform_from_url(result.url),
+            confidence="high",
+            raw_reference=raw_output,
+        )
+        source_artifact = self._write_source_output(case, result.url, store, raw_output)
+        store.add_adapter_results(case, [adapter_result])
+        store.add_timeline_event(case, "source.run", f"Ran source {self.name} for {result.url}")
+        artifacts = [
+            path
+            for path in (
+                source_artifact,
+                result.headers_path,
+                result.body_path,
+                result.screenshot_path,
+            )
+            if path is not None
+        ]
+        return SourceRunResult(
+            source=self.name,
+            target=result.url,
+            raw_output=raw_output,
+            results=[adapter_result],
+            artifacts=artifacts,
+            skipped=result.skipped,
+        )
+
+    def run(self, case: str, target: str) -> str:
+        normalized_url = normalize_public_url(target)
+        capture = fetch_public_url(normalized_url)
+        return json.dumps(
+            {
+                "url": normalized_url,
+                "status_code": capture.status_code,
+                "headers": capture.headers,
+                "body": capture.body,
+            },
+            indent=2,
+            sort_keys=True,
+        )
+
+    def parse_results(self, target: str, raw_output: str) -> list[AdapterResult]:
+        try:
+            parsed = json.loads(raw_output)
+        except json.JSONDecodeError:
+            parsed = {}
+        url = str(parsed.get("url") or normalize_public_url(target))
+        status = parsed.get("status_code", "unknown")
+        return [
+            AdapterResult(
+                source=self.name,
+                target=url,
+                url=url,
+                platform=_platform_from_url(url),
+                confidence="high",
+                raw_reference=f"HTTP status: {status}",
+            )
+        ]
+
+
+def _platform_from_url(url: str) -> str:
+    host = urlparse(url).hostname or ""
+    if not host:
+        return "unknown"
+    parts = host.split(".")
+    if len(parts) >= 2:
+        return parts[-2]
+    return host

rekos/adapters/maigret.py

@@ -0,0 +1,99 @@
+"""Maigret passive username adapter."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+from urllib.parse import urlparse
+
+from rekos.errors import ExternalToolMissingError
+from rekos.osint import _run_command
+
+from .base import AdapterResult, BaseSourceAdapter
+
+
+URL_RE = re.compile(r"https?://[^\s<>'\"]+")
+MAIGRET_EXECUTABLE_NAMES = ("maigret", "maigret.py")
+
+
+class MaigretAdapter(BaseSourceAdapter):
+    name = "maigret_username"
+    description = "Run Maigret against a username and parse public profile URLs."
+    supported_target_types = ("username",)
+    passive_only = True
+    external_dependencies = ("maigret",)
+
+    def dependency_status(self) -> dict[str, bool]:
+        return {"maigret": _resolve_maigret_command() is not None}
+
+    def run(self, case: str, target: str) -> str:
+        command = _resolve_maigret_command()
+        if command is None:
+            raise ExternalToolMissingError("Missing username investigation tool: install maigret.")
+        output = _run_command([*command, "--print-found", "--", target], "maigret")
+        return _raw_output(output.stdout, output.stderr)
+
+    def parse_results(self, target: str, raw_output: str) -> list[AdapterResult]:
+        results: list[AdapterResult] = []
+        seen: set[str] = set()
+        for match in URL_RE.finditer(raw_output):
+            url = match.group(0).rstrip(").,];")
+            if url in seen:
+                continue
+            seen.add(url)
+            results.append(
+                AdapterResult(
+                    source=self.name,
+                    target=target,
+                    url=url,
+                    platform=_platform_from_url(url),
+                    confidence="medium",
+                    raw_reference=url,
+                )
+            )
+        return results
+
+
+def _platform_from_url(url: str) -> str:
+    host = urlparse(url).hostname or ""
+    if not host:
+        return "unknown"
+    parts = host.split(".")
+    if len(parts) >= 2:
+        return parts[-2]
+    return host
+
+
+def _raw_output(stdout: str, stderr: str) -> str:
+    parts = [stdout.strip()]
+    if stderr.strip():
+        parts.extend(["", "[stderr]", stderr.strip()])
+    return "\n".join(part for part in parts if part).rstrip() + "\n"
+
+
+def _resolve_maigret_command() -> list[str] | None:
+    for executable_name in MAIGRET_EXECUTABLE_NAMES:
+        executable = shutil.which(executable_name)
+        if executable:
+            return [executable]
+
+    python_bin = Path(sys.executable).resolve().parent
+    for executable_name in MAIGRET_EXECUTABLE_NAMES:
+        candidate = python_bin / executable_name
+        if candidate.is_file() and os.access(candidate, os.X_OK):
+            return [str(candidate)]
+
+    if _module_runner_available("maigret"):
+        return [sys.executable, "-m", "maigret"]
+    return None
+
+
+def _module_runner_available(module_name: str) -> bool:
+    try:
+        return importlib.util.find_spec(f"{module_name}.__main__") is not None
+    except (ImportError, AttributeError, ValueError):
+        return False

rekos/adapters/registry.py

@@ -0,0 +1,72 @@
+"""Registry for passive OSINT source adapters."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from rekos.errors import RekosError
+
+from .base import BaseSourceAdapter
+from .http_snapshot import HttpSnapshotAdapter
+from .maigret import MaigretAdapter
+from .sherlock import SherlockUsernameAdapter
+from .web_osint import (
+    CrtshDomainAdapter,
+    DnsDomainAdapter,
+    RdapDomainAdapter,
+    WaybackUrlAdapter,
+    WebDomainAdapter,
+)
+from .wmn import WmnUsernameAdapter
+
+
+class SourceNotFoundError(RekosError):
+    pass
+
+
+@dataclass(frozen=True)
+class SourceDependencyCheck:
+    source: str
+    dependencies: dict[str, bool]
+
+
+class SourceAdapterRegistry:
+    def __init__(self, adapters: list[BaseSourceAdapter]) -> None:
+        self._adapters = {adapter.name: adapter for adapter in adapters}
+
+    def list(self) -> list[BaseSourceAdapter]:
+        return [self._adapters[name] for name in sorted(self._adapters)]
+
+    def get(self, name: str) -> BaseSourceAdapter:
+        try:
+            return self._adapters[name]
+        except KeyError as exc:
+            available = ", ".join(sorted(self._adapters))
+            raise SourceNotFoundError(
+                f"Unknown source '{name}'. Available sources: {available}."
+            ) from exc
+
+    def check_dependencies(self) -> list[SourceDependencyCheck]:
+        return [
+            SourceDependencyCheck(
+                source=adapter.name,
+                dependencies=adapter.dependency_status(),
+            )
+            for adapter in self.list()
+        ]
+
+
+def default_registry() -> SourceAdapterRegistry:
+    return SourceAdapterRegistry(
+        [
+            CrtshDomainAdapter(),
+            DnsDomainAdapter(),
+            HttpSnapshotAdapter(),
+            MaigretAdapter(),
+            RdapDomainAdapter(),
+            SherlockUsernameAdapter(),
+            WebDomainAdapter(),
+            WaybackUrlAdapter(),
+            WmnUsernameAdapter(),
+        ]
+    )

rekos/adapters/sherlock.py

@@ -0,0 +1,70 @@
+"""Sherlock passive username adapter."""
+
+from __future__ import annotations
+
+import re
+import shutil
+from urllib.parse import urlparse
+
+from rekos.errors import ExternalToolMissingError
+from rekos.osint import _run_tool
+
+from .base import AdapterResult, BaseSourceAdapter
+
+
+URL_RE = re.compile(r"https?://[^\s<>'\"]+")
+
+
+class SherlockAdapter(BaseSourceAdapter):
+    name = "sherlock"
+    description = "Run Sherlock against a username and parse public profile URLs."
+    supported_target_types = ("username",)
+    passive_only = True
+    external_dependencies = ("sherlock",)
+
+    def run(self, case: str, target: str) -> str:
+        if not shutil.which("sherlock"):
+            raise ExternalToolMissingError("Missing username investigation tool: install sherlock.")
+        output = _run_tool("sherlock", ["--print-found", "--", target])
+        return _raw_output(output.stdout, output.stderr)
+
+    def parse_results(self, target: str, raw_output: str) -> list[AdapterResult]:
+        results: list[AdapterResult] = []
+        seen: set[str] = set()
+        for match in URL_RE.finditer(raw_output):
+            url = match.group(0).rstrip(").,];")
+            if url in seen:
+                continue
+            seen.add(url)
+            results.append(
+                AdapterResult(
+                    source=self.name,
+                    target=target,
+                    url=url,
+                    platform=_platform_from_url(url),
+                    confidence="medium",
+                    raw_reference=url,
+                )
+            )
+        return results
+
+
+def _platform_from_url(url: str) -> str:
+    host = urlparse(url).hostname or ""
+    if not host:
+        return "unknown"
+    parts = host.split(".")
+    if len(parts) >= 2:
+        return parts[-2]
+    return host
+
+
+def _raw_output(stdout: str, stderr: str) -> str:
+    parts = [stdout.strip()]
+    if stderr.strip():
+        parts.extend(["", "[stderr]", stderr.strip()])
+    return "\n".join(part for part in parts if part).rstrip() + "\n"
+
+
+class SherlockUsernameAdapter(SherlockAdapter):
+    name = "sherlock_username"