mailaccess 0.1.0__py3-none-any.whl

backend/Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Prevent Python from writing pyc files to disc
+ENV PYTHONDONTWRITEBYTECODE=1
+# Prevent Python from buffering stdout and stderr
+ENV PYTHONUNBUFFERED=1
+
+# Copy required files for installation
+COPY pyproject.toml README.md ./
+COPY backend/ ./backend/
+COPY cli/ ./cli/
+
+# WeasyPrint runtime libraries
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgobject-2.0-0 libcairo2 libpango-1.0-0 libpangocairo-1.0-0 \
+    libgdk-pixbuf-xlib-2.0-0 libffi-dev shared-mime-info \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install dependencies (using --no-cache-dir to reduce image size)
+RUN pip install --no-cache-dir .
+
+# Create data directory for SQLite persistence
+RUN mkdir -p /app/data
+
+EXPOSE 8000
+
+# Default command (used in prod, overridden in dev)
+CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/api/middleware/auth.py

@@ -0,0 +1,49 @@
+import contextvars
+import uuid
+from typing import Awaitable, Callable
+
+from fastapi import Request
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from backend.config import settings
+
+request_id_contextvar = contextvars.ContextVar("request_id", default="-")
+
+
+class RequestIDMiddleware(BaseHTTPMiddleware):
+    async def dispatch(
+        self, request: Request, call_next: Callable[[Request], Awaitable]
+    ):
+        req_id = str(uuid.uuid4())
+        request_id_contextvar.set(req_id)
+        
+        response = await call_next(request)
+        response.headers["X-Request-ID"] = req_id
+        return response
+
+
+class APIKeyMiddleware(BaseHTTPMiddleware):
+    async def dispatch(
+        self, request: Request, call_next: Callable[[Request], Awaitable]
+    ):
+        path = request.url.path
+        
+        # Bypass authentication for /health, websocket, and Maltego transform routes
+        if path.startswith("/health") or path.startswith("/ws/") or path.startswith("/maltego/"):
+            return await call_next(request)
+
+        # If API key is not configured, bypass authentication entirely
+        if not settings.mailaccess_api_key:
+            return await call_next(request)
+
+        # For /api/ routes, enforce the API key header
+        if path.startswith("/api/"):
+            api_key = request.headers.get("X-API-Key")
+            if not api_key or api_key != settings.mailaccess_api_key:
+                return JSONResponse(
+                    status_code=401,
+                    content={"error": "unauthorized"}
+                )
+
+        return await call_next(request)

backend/api/queue_registry.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import asyncio
+
+_queues: dict[str, asyncio.Queue] = {}
+
+
+def put(investigation_id: str, queue: asyncio.Queue) -> None:
+    _queues[investigation_id] = queue
+
+
+def pop(investigation_id: str) -> asyncio.Queue | None:
+    return _queues.pop(investigation_id, None)

backend/api/router.py

@@ -0,0 +1,11 @@
+from fastapi import APIRouter
+
+from .routes.investigations import router as investigations_router
+from .routes.modules import router as modules_router
+from .websocket import router as ws_router
+
+api_router = APIRouter()
+api_router.include_router(investigations_router, tags=["investigations"])
+api_router.include_router(modules_router, prefix="/modules", tags=["modules"])
+
+__all__ = ["api_router", "ws_router"]

backend/api/routes/health.py

@@ -0,0 +1,28 @@
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from backend.db.database import get_db
+from backend.modules import get_all_modules
+
+router = APIRouter()
+
+
+@router.get("/health")
+async def health_check(session: AsyncSession = Depends(get_db)):
+    db_status = "error"
+    try:
+        # Simple query to check if the DB is reachable
+        await session.execute(text("SELECT 1"))
+        db_status = "connected"
+    except Exception:
+        pass
+
+    modules_loaded = [mod.name for mod in get_all_modules()]
+
+    return {
+        "status": "ok",
+        "version": "0.1.0",
+        "modules_loaded": modules_loaded,
+        "db": db_status
+    }

backend/api/routes/investigations.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import asyncio
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.responses import Response
+from pydantic import BaseModel
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ...core.service import InvestigationService, enrich_report
+from ...db.database import get_db
+from ...exporters import EXPORTERS
+from .. import queue_registry
+
+router = APIRouter()
+
+async def _cleanup_queue(investigation_id: str, delay: float = 300.0) -> None:
+    await asyncio.sleep(delay)
+    queue_registry.pop(investigation_id)
+
+
+# ---------------------------------------------------------------------------
+# Schemas
+# ---------------------------------------------------------------------------
+
+
+class InvestigateRequest(BaseModel):
+    email: str
+    modules: list[str] | None = None
+
+
+class InvestigateResponse(BaseModel):
+    id: str
+    status: str
+    created_at: str
+
+
+class InvestigationSummary(BaseModel):
+    id: str
+    email: str
+    status: str
+    exposure_score: int | None
+    created_at: str
+    completed_at: str | None
+
+
+class PaginatedInvestigations(BaseModel):
+    total: int
+    page: int
+    page_size: int
+    pages: int
+    items: list[InvestigationSummary]
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+
+@router.post("/investigate", response_model=InvestigateResponse, status_code=202)
+async def start_investigation(
+    body: InvestigateRequest,
+    session: AsyncSession = Depends(get_db),
+) -> InvestigateResponse:
+    """Create a new investigation and kick off the engine in the background."""
+    service = InvestigationService(session)
+    investigation_id, created_at, queue = await service.create_investigation(
+        body.email, body.modules
+    )
+    queue_registry.put(investigation_id, queue)
+    # Release the queue from memory after 5 minutes if no WS consumer arrives.
+    asyncio.create_task(_cleanup_queue(investigation_id, delay=300.0))
+    return InvestigateResponse(
+        id=investigation_id,
+        status="pending",
+        created_at=created_at.isoformat(),
+    )
+
+
+@router.get("/report/{investigation_id}")
+async def get_report(
+    investigation_id: str,
+    session: AsyncSession = Depends(get_db),
+) -> dict:
+    """Return the full enriched investigation report."""
+    service = InvestigationService(session)
+    data = await service.get_investigation(investigation_id)
+
+    if data is None:
+        raise HTTPException(status_code=404, detail="Investigation not found")
+
+    return enrich_report(data)
+
+
+@router.get("/report/{investigation_id}/export")
+async def export_report(
+    investigation_id: str,
+    format: str = Query("json", pattern="^(json|csv|markdown|pdf|stix|maltego)$"),
+    session: AsyncSession = Depends(get_db),
+) -> Response:
+    """Export the investigation report in the requested format."""
+
+    service = InvestigationService(session)
+    data = await service.get_investigation(investigation_id)
+    if data is None:
+        raise HTTPException(status_code=404, detail="Investigation not found")
+
+    data = enrich_report(data)
+    email = data.get("email", "unknown")
+
+    exporter = EXPORTERS[format]()
+    if format == "pdf":
+        from ...exporters.pdf_exporter import PdfExporter
+        assert isinstance(exporter, PdfExporter)
+        content = await exporter.generate(investigation_id, data)
+    else:
+        content = exporter.export(investigation_id, data)
+    return Response(
+        content=content,
+        media_type=exporter.content_type,
+        headers={
+            "Content-Disposition": (
+                f'attachment; filename="mailaccess_{email}_{investigation_id}'
+                f'.{"stix.json" if format == "stix" else "maltego.csv" if format == "maltego" else format}"'
+            )
+        },
+    )
+
+
+@router.get("/investigations", response_model=PaginatedInvestigations)
+async def list_investigations(
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    session: AsyncSession = Depends(get_db),
+) -> PaginatedInvestigations:
+    """Paginated list of past investigations, newest first."""
+    service = InvestigationService(session)
+    result = await service.list_investigations(page=page, page_size=page_size)
+    return PaginatedInvestigations(
+        total=result["total"],
+        page=result["page"],
+        page_size=result["page_size"],
+        pages=result["pages"],
+        items=[
+            InvestigationSummary(
+                id=item["id"],
+                email=item["email"],
+                status=item["status"],
+                exposure_score=item.get("exposure_score"),
+                created_at=item["created_at"],
+                completed_at=item.get("completed_at"),
+            )
+            for item in result["items"]
+        ],
+    )
+
+
+@router.delete("/investigation/{investigation_id}", status_code=204)
+async def delete_investigation(
+    investigation_id: str,
+    session: AsyncSession = Depends(get_db),
+) -> None:
+    """Hard-delete an investigation and all associated findings."""
+    service = InvestigationService(session)
+    deleted = await service.delete_investigation(investigation_id)
+    if not deleted:
+        raise HTTPException(status_code=404, detail="Investigation not found")

backend/api/routes/maltego.py

@@ -0,0 +1,83 @@
+"""Maltego local transform server endpoint.
+
+Implements the TRX protocol (XML over HTTP POST) so Maltego Desktop can run
+email investigations directly without any API key.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import Response
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ...core.service import InvestigationService, enrich_report
+from ...db.database import get_db
+from ...integrations.maltego_transform import build_error_response, build_response, parse_request
+from .. import queue_registry
+
+logger = logging.getLogger("mailaccess.maltego")
+
+router = APIRouter()
+
+_TRANSFORM_TIMEOUT = 55.0
+_XML = "application/xml"
+
+
+@router.post("/email_investigate")
+async def email_investigate(
+    request: Request,
+    session: AsyncSession = Depends(get_db),
+) -> Response:
+    """TRX transform: run a full email investigation and return Maltego entities."""
+    body = await request.body()
+
+    try:
+        email = parse_request(body)
+    except Exception as exc:
+        return Response(
+            content=build_error_response(f"Invalid TRX request: {exc}"),
+            media_type=_XML,
+        )
+
+    service = InvestigationService(session)
+    try:
+        investigation_id, _created_at, queue = await service.create_investigation(email)
+        queue_registry.put(investigation_id, queue)
+    except Exception as exc:
+        logger.exception("Failed to start investigation for %s", email)
+        return Response(
+            content=build_error_response(f"Failed to start investigation: {exc}"),
+            media_type=_XML,
+        )
+
+    # Drain the event queue until the engine pushes the None sentinel or we hit 55s.
+    partial = False
+    loop = asyncio.get_running_loop()
+    deadline = loop.time() + _TRANSFORM_TIMEOUT
+
+    while True:
+        remaining = deadline - loop.time()
+        if remaining <= 0:
+            partial = True
+            break
+        try:
+            event = await asyncio.wait_for(queue.get(), timeout=remaining)
+            if event is None:  # completion sentinel from the engine
+                break
+        except asyncio.TimeoutError:
+            partial = True
+            break
+
+    data = await service.get_investigation(investigation_id)
+    if data is None:
+        return Response(
+            content=build_error_response("Investigation record not found"),
+            media_type=_XML,
+        )
+
+    return Response(
+        content=build_response(enrich_report(data), partial=partial),
+        media_type=_XML,
+    )

backend/api/routes/modules.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+from ...modules import get_all_modules
+
+router = APIRouter()
+
+
+class ModuleInfo(BaseModel):
+    name: str
+    description: str
+    requires_key: bool
+
+
+@router.get("/", response_model=list[ModuleInfo])
+async def list_modules() -> list[ModuleInfo]:
+    return [
+        ModuleInfo(
+            name=m.name,
+            description=m.description,
+            requires_key=m.requires_key,
+        )
+        for m in get_all_modules()
+    ]

backend/api/websocket.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from ..core.engine import QueueEvent
+from ..db.database import AsyncSessionLocal
+from ..db.models import Investigation
+from . import queue_registry
+
+router = APIRouter()
+
+
+def _risk_level(score: int | None) -> str:
+    if score is None:
+        return "unknown"
+    if score <= 20:
+        return "low"
+    if score <= 50:
+        return "medium"
+    if score <= 80:
+        return "high"
+    return "critical"
+
+
+@router.websocket("/ws/investigate/{investigation_id}")
+async def ws_investigate(investigation_id: str, websocket: WebSocket) -> None:
+    """
+    Stream investigation events in real time.
+
+    Connect immediately after POST /api/investigate. The server pushes one
+    event per module as it starts and completes, then a final
+    "investigation_complete" frame when all modules are done and the DB is
+    persisted.
+
+    Event frames::
+
+        { "type": "module_start",  "module": "hibp", "timestamp": "..." }
+        { "type": "module_result", "module": "hibp", "findings": [...], "status": "success" }
+        { "type": "module_error",  "module": "social", "error": "...", "status": "failed" }
+        { "type": "investigation_complete", "exposure_score": 72, "risk_level": "high" }
+    """
+    await websocket.accept()
+
+    queue = queue_registry.pop(investigation_id)
+    if queue is None:
+        await websocket.send_json(
+            {"type": "error", "error": "investigation not found or already streaming"}
+        )
+        await websocket.close(code=1008)
+        return
+
+    try:
+        while True:
+            item: QueueEvent | None = await queue.get()
+
+            if item is None:
+                # Sentinel: engine finished and persisted — fetch final score from DB.
+                async with AsyncSessionLocal() as db:
+                    inv = await db.get(Investigation, investigation_id)
+                score = inv.exposure_score if inv else None
+                await websocket.send_json(
+                    {
+                        "type": "investigation_complete",
+                        "exposure_score": score,
+                        "risk_level": _risk_level(score),
+                    }
+                )
+                break
+
+            if item.type == "module_start":
+                await websocket.send_json(
+                    {
+                        "type": "module_start",
+                        "module": item.module_name,
+                        "timestamp": datetime.now(timezone.utc).isoformat(),
+                    }
+                )
+            elif item.type == "module_result":
+                assert item.result is not None
+                await websocket.send_json(
+                    {
+                        "type": "module_result",
+                        "module": item.module_name,
+                        "findings": item.result.findings,
+                        "status": item.result.status.value,
+                    }
+                )
+            elif item.type == "module_error":
+                assert item.result is not None
+                await websocket.send_json(
+                    {
+                        "type": "module_error",
+                        "module": item.module_name,
+                        "error": ", ".join(item.result.errors or ["unknown error"]),
+                        "status": "failed",
+                    }
+                )
+
+    except WebSocketDisconnect:
+        # Drain the queue so the engine's background task isn't blocked.
+        asyncio.create_task(_drain_silently(queue))
+
+
+async def _drain_silently(queue: asyncio.Queue) -> None:
+    while True:
+        item = await queue.get()
+        if item is None:
+            break

backend/config.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import json
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # Database
+    database_url: str = "sqlite+aiosqlite:///./data/mailaccess.db"
+
+    # Application
+    debug: bool = False
+    log_level: str = "INFO"
+    cors_origins: list[str] = ["http://localhost:5173", "http://localhost:3000"]
+
+    # Worker
+    max_concurrent_modules: int = 10
+    module_timeout_seconds: int = 30
+    # Per-module timeout overrides: MODULE_TIMEOUT_OVERRIDES={"whatsmyname": 120}
+    module_timeout_overrides: dict[str, int] = {}
+
+    # Account discovery (opt-in — probes 120+ platforms via Holehe, can be noisy)
+    enable_account_discovery: bool = False
+
+    # WhatsMyName (opt-in — username enumeration across 700+ platforms, takes 60–90s)
+    enable_whatsmyname: bool = False
+
+    # Permutation discovery (opt-in — generates email variations from recovered names,
+    # then probes each with HIBP + Hudson Rock; adds 30–60s and up to 120 API calls)
+    enable_permutation_discovery: bool = False
+
+    # GHunt (opt-in — requires ghunt>=2.3 installed and a valid creds file from `ghunt login`)
+    # Cookies expire periodically and require manual refresh via `ghunt login`.
+    enable_ghunt: bool = False
+    ghunt_creds_path: str | None = None
+
+    # Webhooks
+    slack_webhook_url: str | None = None
+    discord_webhook_url: str | None = None
+    integration_webhook_url: str | None = None
+    integration_webhook_secret: str | None = None
+
+    # API keys (all optional — modules skip themselves when their key is absent)
+    mailaccess_api_key: str | None = None
+    haveibeenpwned_api_key: str | None = None
+    hibp_api_key: str | None = None
+    hunter_io_api_key: str | None = None
+    emailrep_api_key: str | None = None
+    shodan_api_key: str | None = None
+    serpapi_key: str | None = None
+
+    # Proxy
+    proxy_url: str | None = None
+    proxy_enabled: bool = False
+
+    # Rate limiting
+    rate_limit_enabled: bool = True
+    request_delay_ms: int = 1000
+    # Per-domain overrides (ms): RATE_LIMIT_OVERRIDES={"api.github.com": 500}
+    rate_limit_overrides: dict[str, int] = {}
+    # Legacy per-domain delays (seconds): RATE_LIMIT_DELAYS={"haveibeenpwned.com": 1.5}
+    rate_limit_delays: dict[str, float] = {}
+
+    @field_validator("rate_limit_overrides", mode="before")
+    @classmethod
+    def _parse_overrides(cls, v: str | dict) -> dict[str, int]:
+        if isinstance(v, str):
+            return json.loads(v) if v else {}
+        return v
+
+    @field_validator("module_timeout_overrides", mode="before")
+    @classmethod
+    def _parse_timeout_overrides(cls, v: str | dict) -> dict[str, int]:
+        if isinstance(v, str):
+            return json.loads(v) if v else {}
+        return v
+
+    @field_validator("rate_limit_delays", mode="before")
+    @classmethod
+    def _parse_delays(cls, v: str | dict) -> dict[str, float]:
+        if isinstance(v, str):
+            return json.loads(v) if v else {}
+        return v
+
+    @field_validator("cors_origins", mode="before")
+    @classmethod
+    def _split_cors(cls, v: str | list[str]) -> list[str]:
+        if isinstance(v, str):
+            return [origin.strip() for origin in v.split(",")]
+        return v
+
+
+settings = Settings()

backend/core/__init__.py

@@ -0,0 +1,24 @@
+from .aggregator import ResultAggregator
+from .engine import InvestigationEngine
+from .http_client import build_client
+from .proxy import ProxyConnectionError, ProxyConfig, proxy_config
+from .rate_limiter import DomainRateLimiter, RateLimiter, rate_limiter
+from .result_aggregator import ProfileAggregator, UnifiedProfile
+from .scheduler import Scheduler
+from .service import InvestigationService
+
+__all__ = [
+    "DomainRateLimiter",
+    "InvestigationEngine",
+    "InvestigationService",
+    "ProfileAggregator",
+    "ProxyConfig",
+    "ProxyConnectionError",
+    "RateLimiter",
+    "ResultAggregator",
+    "Scheduler",
+    "UnifiedProfile",
+    "build_client",
+    "proxy_config",
+    "rate_limiter",
+]